This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch CAMEL-26211-djl-body-types in repository https://gitbox.apache.org/repos/asf/camel.git
commit c642def1a8eb736408469254253efabe606312dc Author: Claus Ibsen <[email protected]> AuthorDate: Tue Nov 3 00:34:30 2026 +0100 CAMEL-26211: camel-djl - a cv or audio body without a converter fails as not supported, nlp predictors take bytes and streams, and a custom model is looked up once The cv and audio predictors read the body with getBody(Image/Audio), which returns null for a String, Map, BufferedImage or null body, so the predictor failed later with "Could not process input or output". They now use getMandatoryBody and fail with "Data type is not supported". The nlp predictors accepted only a String body. They now convert the body to String, so a byte[] or InputStream body works. The custom predictors looked up the model and translator for every message, and a wrong name gave a NullPointerException. They are now looked up once, at the first message, and a missing bean fails with a NoSuchBeanException that names it. A missing model or translator option fails the start of the producer. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../org/apache/camel/catalog/components/djl.json | 2 +- .../apache/camel/catalog/docs/djl-component.adoc | 39 ++- components/camel-ai/camel-djl/pom.xml | 6 + .../org/apache/camel/component/djl/djl.json | 2 +- .../camel-djl/src/main/docs/djl-component.adoc | 39 ++- .../apache/camel/component/djl/DJLEndpoint.java | 22 +- .../apache/camel/component/djl/DJLProducer.java | 6 + .../djl/model/AbstractCustomPredictor.java | 75 ++++++ .../component/djl/model/AbstractPredictor.java | 8 + .../djl/model/audio/CustomAudioPredictor.java | 31 +-- .../djl/model/audio/ZooAudioPredictor.java | 13 +- .../djl/model/cv/AbstractCvZooPredictor.java | 13 +- .../component/djl/model/cv/CustomCvPredictor.java | 31 +-- .../model/cv/CustomImageGenerationPredictor.java | 18 +- .../djl/model/nlp/AbstractNlpZooPredictor.java | 14 +- .../djl/model/nlp/CustomNlpPredictor.java | 32 +-- .../model/nlp/CustomQuestionAnswerPredictor.java | 18 +- .../djl/model/tabular/CustomTabularPredictor.java | 18 +- .../timeseries/CustomForecastingPredictor.java | 18 +- .../component/djl/model/PredictorBodyTest.java | 280 +++++++++++++++++++++ .../ROOT/pages/camel-4x-upgrade-guide-4_24.adoc | 10 + 21 files changed, 523 insertions(+), 172 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/djl.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/djl.json index 368f90157bb4..c537ad8a68f2 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/djl.json +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/djl.json @@ -32,7 +32,7 @@ "CamelDjlFileType": { "index": 1, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The file type of the message body data. It is used when the body is converted to bytes.", "constantName": "org.apache.camel.component.djl.DJLConstants#FILE_TYPE" } }, "bodyTypes": { - "producerIn": { "carriers": [ "bytes", "stream", "file", "pojo", "text", "list" ], "byOption": [ { "option": "application", "value": "cv\/image_classification", "carriers": [ "bytes", "stream", "file", "pojo" ] }, { "option": "application", "value": "cv\/object_detection", "carriers": [ "bytes", "stream", "file", "pojo" ] }, { "option": "application", "value": "cv\/semantic_segmentation", "carriers": [ "bytes", "stream", "file", "pojo" ] }, { "option": "application", "value": "cv\/in [...] + "producerIn": { "carriers": [ "bytes", "stream", "file", "pojo", "text", "list" ], "byOption": [ { "option": "application", "value": "cv\/image_classification", "carriers": [ "bytes", "stream", "file", "pojo" ] }, { "option": "application", "value": "cv\/object_detection", "carriers": [ "bytes", "stream", "file", "pojo" ] }, { "option": "application", "value": "cv\/semantic_segmentation", "carriers": [ "bytes", "stream", "file", "pojo" ] }, { "option": "application", "value": "cv\/in [...] "producerOut": { "carriers": [ "pojo", "text", "list", "bytes" ], "byOption": [ { "option": "application", "value": "cv\/image_classification", "carriers": [ "pojo" ], "javaType": "ai.djl.modality.Classifications" }, { "option": "application", "value": "cv\/object_detection", "carriers": [ "pojo" ], "javaType": "ai.djl.modality.cv.output.DetectedObjects" }, { "option": "application", "value": "cv\/semantic_segmentation", "carriers": [ "pojo" ], "javaType": "ai.djl.modality.cv.output. [...] }, "properties": { diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/djl-component.adoc b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/djl-component.adoc index 3f30bd095040..c96a2b6752ff 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/djl-component.adoc +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/djl-component.adoc @@ -106,7 +106,9 @@ File` | `ai.djl.modality.cv.Image` | `nlp/fill_mask` -| `String` +| `String + +byte[] + +InputStream` | `String[]` | `nlp/question_answer` @@ -115,35 +117,51 @@ String[]` | `String` | `nlp/text_classification` -| `String` +| `String + +byte[] + +InputStream` | `ai.djl.modality.Classifications` | `nlp/sentiment_analysis` -| `String` +| `String + +byte[] + +InputStream` | `ai.djl.modality.Classifications` | `nlp/token_classification` -| `String` +| `String + +byte[] + +InputStream` | `ai.djl.modality.Classifications` | `nlp/text_generation` -| `String` +| `String + +byte[] + +InputStream` | `String` | `nlp/machine_translation` -| `String` +| `String + +byte[] + +InputStream` | `String` | `nlp/multiple_choice` -| `String` +| `String + +byte[] + +InputStream` | `String` | `nlp/text_embedding` -| `String` +| `String + +byte[] + +InputStream` | `ai.djl.ndarray.NDArray` | `nlp/word_embedding` (custom model only) -| `String` +| `String + +byte[] + +InputStream` | `byte[]` (the encoded `ai.djl.ndarray.NDList`) | `audio` @@ -584,3 +602,6 @@ from("file:/data/mnist/0/10.png") .to("djl:cv/image_classification?model=MyModel&translator=MyTranslator"); ---- +The `model` and `translator` beans are looked up in the registry once, at the first message, and are kept until +the route is stopped. A missing bean fails the message with a `NoSuchBeanException` that names the bean. + diff --git a/components/camel-ai/camel-djl/pom.xml b/components/camel-ai/camel-djl/pom.xml index bce1790a98fd..2a6e46228ac0 100644 --- a/components/camel-ai/camel-djl/pom.xml +++ b/components/camel-ai/camel-djl/pom.xml @@ -140,6 +140,12 @@ <version>${djl-version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <version>${mockito-version}</version> + <scope>test</scope> + </dependency> </dependencies> </project> diff --git a/components/camel-ai/camel-djl/src/generated/resources/META-INF/org/apache/camel/component/djl/djl.json b/components/camel-ai/camel-djl/src/generated/resources/META-INF/org/apache/camel/component/djl/djl.json index 368f90157bb4..c537ad8a68f2 100644 --- a/components/camel-ai/camel-djl/src/generated/resources/META-INF/org/apache/camel/component/djl/djl.json +++ b/components/camel-ai/camel-djl/src/generated/resources/META-INF/org/apache/camel/component/djl/djl.json @@ -32,7 +32,7 @@ "CamelDjlFileType": { "index": 1, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The file type of the message body data. It is used when the body is converted to bytes.", "constantName": "org.apache.camel.component.djl.DJLConstants#FILE_TYPE" } }, "bodyTypes": { - "producerIn": { "carriers": [ "bytes", "stream", "file", "pojo", "text", "list" ], "byOption": [ { "option": "application", "value": "cv\/image_classification", "carriers": [ "bytes", "stream", "file", "pojo" ] }, { "option": "application", "value": "cv\/object_detection", "carriers": [ "bytes", "stream", "file", "pojo" ] }, { "option": "application", "value": "cv\/semantic_segmentation", "carriers": [ "bytes", "stream", "file", "pojo" ] }, { "option": "application", "value": "cv\/in [...] + "producerIn": { "carriers": [ "bytes", "stream", "file", "pojo", "text", "list" ], "byOption": [ { "option": "application", "value": "cv\/image_classification", "carriers": [ "bytes", "stream", "file", "pojo" ] }, { "option": "application", "value": "cv\/object_detection", "carriers": [ "bytes", "stream", "file", "pojo" ] }, { "option": "application", "value": "cv\/semantic_segmentation", "carriers": [ "bytes", "stream", "file", "pojo" ] }, { "option": "application", "value": "cv\/in [...] "producerOut": { "carriers": [ "pojo", "text", "list", "bytes" ], "byOption": [ { "option": "application", "value": "cv\/image_classification", "carriers": [ "pojo" ], "javaType": "ai.djl.modality.Classifications" }, { "option": "application", "value": "cv\/object_detection", "carriers": [ "pojo" ], "javaType": "ai.djl.modality.cv.output.DetectedObjects" }, { "option": "application", "value": "cv\/semantic_segmentation", "carriers": [ "pojo" ], "javaType": "ai.djl.modality.cv.output. [...] }, "properties": { diff --git a/components/camel-ai/camel-djl/src/main/docs/djl-component.adoc b/components/camel-ai/camel-djl/src/main/docs/djl-component.adoc index 3f30bd095040..c96a2b6752ff 100644 --- a/components/camel-ai/camel-djl/src/main/docs/djl-component.adoc +++ b/components/camel-ai/camel-djl/src/main/docs/djl-component.adoc @@ -106,7 +106,9 @@ File` | `ai.djl.modality.cv.Image` | `nlp/fill_mask` -| `String` +| `String + +byte[] + +InputStream` | `String[]` | `nlp/question_answer` @@ -115,35 +117,51 @@ String[]` | `String` | `nlp/text_classification` -| `String` +| `String + +byte[] + +InputStream` | `ai.djl.modality.Classifications` | `nlp/sentiment_analysis` -| `String` +| `String + +byte[] + +InputStream` | `ai.djl.modality.Classifications` | `nlp/token_classification` -| `String` +| `String + +byte[] + +InputStream` | `ai.djl.modality.Classifications` | `nlp/text_generation` -| `String` +| `String + +byte[] + +InputStream` | `String` | `nlp/machine_translation` -| `String` +| `String + +byte[] + +InputStream` | `String` | `nlp/multiple_choice` -| `String` +| `String + +byte[] + +InputStream` | `String` | `nlp/text_embedding` -| `String` +| `String + +byte[] + +InputStream` | `ai.djl.ndarray.NDArray` | `nlp/word_embedding` (custom model only) -| `String` +| `String + +byte[] + +InputStream` | `byte[]` (the encoded `ai.djl.ndarray.NDList`) | `audio` @@ -584,3 +602,6 @@ from("file:/data/mnist/0/10.png") .to("djl:cv/image_classification?model=MyModel&translator=MyTranslator"); ---- +The `model` and `translator` beans are looked up in the registry once, at the first message, and are kept until +the route is stopped. A missing bean fails the message with a `NoSuchBeanException` that names the bean. + diff --git a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/DJLEndpoint.java b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/DJLEndpoint.java index 461044dc7168..5588aaf3489c 100644 --- a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/DJLEndpoint.java +++ b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/DJLEndpoint.java @@ -64,23 +64,23 @@ import org.apache.camel.support.DefaultEndpoint; @BodyType.When(option = "application", value = "cv/image_generation", carriers = Carrier.LIST, javaType = "int[]"), @BodyType.When(option = "application", value = "nlp/fill_mask", - carriers = Carrier.TEXT, javaType = "java.lang.String"), + carriers = { Carrier.TEXT, Carrier.BYTES, Carrier.STREAM }), @BodyType.When(option = "application", value = "nlp/text_classification", - carriers = Carrier.TEXT, javaType = "java.lang.String"), + carriers = { Carrier.TEXT, Carrier.BYTES, Carrier.STREAM }), @BodyType.When(option = "application", value = "nlp/sentiment_analysis", - carriers = Carrier.TEXT, javaType = "java.lang.String"), + carriers = { Carrier.TEXT, Carrier.BYTES, Carrier.STREAM }), @BodyType.When(option = "application", value = "nlp/token_classification", - carriers = Carrier.TEXT, javaType = "java.lang.String"), + carriers = { Carrier.TEXT, Carrier.BYTES, Carrier.STREAM }), @BodyType.When(option = "application", value = "nlp/word_embedding", - carriers = Carrier.TEXT, javaType = "java.lang.String"), + carriers = { Carrier.TEXT, Carrier.BYTES, Carrier.STREAM }), @BodyType.When(option = "application", value = "nlp/text_generation", - carriers = Carrier.TEXT, javaType = "java.lang.String"), + carriers = { Carrier.TEXT, Carrier.BYTES, Carrier.STREAM }), @BodyType.When(option = "application", value = "nlp/machine_translation", - carriers = Carrier.TEXT, javaType = "java.lang.String"), + carriers = { Carrier.TEXT, Carrier.BYTES, Carrier.STREAM }), @BodyType.When(option = "application", value = "nlp/multiple_choice", - carriers = Carrier.TEXT, javaType = "java.lang.String"), + carriers = { Carrier.TEXT, Carrier.BYTES, Carrier.STREAM }), @BodyType.When(option = "application", value = "nlp/text_embedding", - carriers = Carrier.TEXT, javaType = "java.lang.String"), + carriers = { Carrier.TEXT, Carrier.BYTES, Carrier.STREAM }), @BodyType.When(option = "application", value = "nlp/question_answer", carriers = { Carrier.POJO, Carrier.LIST }), @BodyType.When(option = "application", value = "audio", @@ -96,8 +96,8 @@ import org.apache.camel.support.DefaultEndpoint; description = "Depends on the application. The cv applications and audio take an image" + " or audio clip as bytes, a stream, a File or a Path, or a DJL Image or" + " Audio (a BufferedImage or a file name as text is not converted);" - + " cv/image_generation an int[] of seeds. The nlp applications take a String" - + " only (bytes or a stream fail: convert to text first), nlp/question_answer" + + " cv/image_generation an int[] of seeds. The nlp applications take text: a" + + " String, bytes or a stream, nlp/question_answer" + " a QAInput or a String[] of the question and the paragraph," + " timeseries/forecasting a TimeSeriesData. No body fails"), producerOut = @BodyType(value = { Carrier.POJO, Carrier.TEXT, Carrier.LIST, Carrier.BYTES }, diff --git a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/DJLProducer.java b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/DJLProducer.java index e0552a868850..835a890ba558 100644 --- a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/DJLProducer.java +++ b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/DJLProducer.java @@ -37,6 +37,12 @@ public class DJLProducer extends DefaultProducer { this.predictor.process(exchange); } + @Override + protected void doStart() throws Exception { + super.doStart(); + this.predictor.start(); + } + @Override protected void doStop() throws Exception { super.doStop(); diff --git a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/AbstractCustomPredictor.java b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/AbstractCustomPredictor.java new file mode 100644 index 000000000000..7bcb1e6fff55 --- /dev/null +++ b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/AbstractCustomPredictor.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.djl.model; + +import ai.djl.Model; +import ai.djl.inference.Predictor; +import ai.djl.translate.Translator; +import org.apache.camel.component.djl.DJLEndpoint; +import org.apache.camel.support.CamelContextHelper; +import org.apache.camel.util.ObjectHelper; + +/** + * A predictor of a custom model: the model and translator beans named by the model and translator options of the + * endpoint. + */ +public abstract class AbstractCustomPredictor<I, O> extends AbstractPredictor { + + protected final String modelName; + protected final String translatorName; + private volatile Model model; + private volatile Translator<I, O> translator; + + protected AbstractCustomPredictor(DJLEndpoint endpoint) { + super(endpoint); + this.modelName = endpoint.getModel(); + this.translatorName = endpoint.getTranslator(); + } + + @Override + public void start() { + // the beans may be bound after the producer is started, so only the options are checked here + ObjectHelper.notNull(modelName, "model"); + ObjectHelper.notNull(translatorName, "translator"); + } + + @Override + public void close() { + // the model belongs to the user, so it is not closed; it is looked up again when the producer starts again + model = null; + translator = null; + } + + /** + * Creates a predictor of the model and translator. They are looked up in the registry once, at the first message, + * and a missing bean fails with a {@link org.apache.camel.NoSuchBeanException} that names it. + */ + @SuppressWarnings("unchecked") + protected Predictor<I, O> newPredictor() { + Model m = model; + if (m == null) { + m = CamelContextHelper.mandatoryLookup(getEndpoint().getCamelContext(), modelName, Model.class); + model = m; + } + Translator<I, O> t = translator; + if (t == null) { + t = CamelContextHelper.mandatoryLookup(getEndpoint().getCamelContext(), translatorName, Translator.class); + translator = t; + } + return m.newPredictor(t); + } +} diff --git a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/AbstractPredictor.java b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/AbstractPredictor.java index ea7b93cf2b73..012c03f6ac3d 100644 --- a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/AbstractPredictor.java +++ b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/AbstractPredictor.java @@ -29,6 +29,14 @@ public abstract class AbstractPredictor { public abstract void process(Exchange exchange) throws Exception; + /** + * Prepares this predictor, such as checking its options. Called when the owning producer is started. The default + * implementation does nothing. + */ + public void start() { + // no-op by default + } + /** * Releases any resources held by this predictor, such as a model loaded from the DJL model zoo. Called when the * owning producer is stopped. The default implementation does nothing; predictors that keep a long-lived model diff --git a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/audio/CustomAudioPredictor.java b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/audio/CustomAudioPredictor.java index 5a10e928d53e..befcaa24f840 100644 --- a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/audio/CustomAudioPredictor.java +++ b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/audio/CustomAudioPredictor.java @@ -16,49 +16,38 @@ */ package org.apache.camel.component.djl.model.audio; -import ai.djl.Model; import ai.djl.inference.Predictor; import ai.djl.modality.audio.Audio; import ai.djl.translate.TranslateException; -import ai.djl.translate.Translator; import org.apache.camel.Exchange; +import org.apache.camel.InvalidPayloadException; import org.apache.camel.RuntimeCamelException; -import org.apache.camel.TypeConversionException; import org.apache.camel.component.djl.DJLConstants; import org.apache.camel.component.djl.DJLEndpoint; -import org.apache.camel.component.djl.model.AbstractPredictor; +import org.apache.camel.component.djl.model.AbstractCustomPredictor; -public class CustomAudioPredictor extends AbstractPredictor { - - protected final String modelName; - protected final String translatorName; +public class CustomAudioPredictor extends AbstractCustomPredictor<Audio, String> { public CustomAudioPredictor(DJLEndpoint endpoint) { super(endpoint); - this.modelName = endpoint.getModel(); - this.translatorName = endpoint.getTranslator(); } @Override public void process(Exchange exchange) throws Exception { + Audio audio; try { - Audio audio = exchange.getIn().getBody(Audio.class); - String result = predict(exchange, audio); - exchange.getIn().setBody(result); - } catch (TypeConversionException e) { + audio = exchange.getIn().getMandatoryBody(Audio.class); + } catch (InvalidPayloadException e) { throw new RuntimeCamelException( - "Data type is not supported. Body should be ai.djl.modality.audio.Audio, byte[], InputStream or File"); + "Data type is not supported. Body should be ai.djl.modality.audio.Audio, byte[], InputStream or File", e); } + String result = predict(exchange, audio); + exchange.getIn().setBody(result); } protected String predict(Exchange exchange, Audio audio) { - Model model = exchange.getContext().getRegistry().lookupByNameAndType(modelName, Model.class); - @SuppressWarnings("unchecked") - Translator<Audio, String> translator - = exchange.getContext().getRegistry().lookupByNameAndType(translatorName, Translator.class); - exchange.getIn().setHeader(DJLConstants.INPUT, audio); - try (Predictor<Audio, String> predictor = model.newPredictor(translator)) { + try (Predictor<Audio, String> predictor = newPredictor()) { return predictor.predict(audio); } catch (TranslateException e) { throw new RuntimeCamelException("Could not process input or output", e); diff --git a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/audio/ZooAudioPredictor.java b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/audio/ZooAudioPredictor.java index e35048a22a99..0828f7652b61 100644 --- a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/audio/ZooAudioPredictor.java +++ b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/audio/ZooAudioPredictor.java @@ -29,8 +29,8 @@ import ai.djl.repository.zoo.ZooModel; import ai.djl.training.util.ProgressBar; import ai.djl.translate.TranslateException; import org.apache.camel.Exchange; +import org.apache.camel.InvalidPayloadException; import org.apache.camel.RuntimeCamelException; -import org.apache.camel.TypeConversionException; import org.apache.camel.component.djl.DJLConstants; import org.apache.camel.component.djl.DJLEndpoint; import org.apache.camel.component.djl.model.AbstractPredictor; @@ -56,14 +56,15 @@ public class ZooAudioPredictor extends AbstractPredictor { @Override public void process(Exchange exchange) throws Exception { + Audio audio; try { - Audio audio = exchange.getIn().getBody(Audio.class); - String result = predict(exchange, audio); - exchange.getIn().setBody(result); - } catch (TypeConversionException e) { + audio = exchange.getIn().getMandatoryBody(Audio.class); + } catch (InvalidPayloadException e) { throw new RuntimeCamelException( - "Data type is not supported. Body should be ai.djl.modality.audio.Audio, byte[], InputStream or File"); + "Data type is not supported. Body should be ai.djl.modality.audio.Audio, byte[], InputStream or File", e); } + String result = predict(exchange, audio); + exchange.getIn().setBody(result); } protected String predict(Exchange exchange, Audio audio) { diff --git a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/cv/AbstractCvZooPredictor.java b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/cv/AbstractCvZooPredictor.java index f2bf3e617c9c..045efc755695 100644 --- a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/cv/AbstractCvZooPredictor.java +++ b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/cv/AbstractCvZooPredictor.java @@ -21,8 +21,8 @@ import ai.djl.modality.cv.Image; import ai.djl.repository.zoo.ZooModel; import ai.djl.translate.TranslateException; import org.apache.camel.Exchange; +import org.apache.camel.InvalidPayloadException; import org.apache.camel.RuntimeCamelException; -import org.apache.camel.TypeConversionException; import org.apache.camel.component.djl.DJLConstants; import org.apache.camel.component.djl.DJLEndpoint; import org.apache.camel.component.djl.model.AbstractPredictor; @@ -37,14 +37,15 @@ public abstract class AbstractCvZooPredictor<T> extends AbstractPredictor { @Override public void process(Exchange exchange) { + Image image; try { - Image image = exchange.getIn().getBody(Image.class); - T result = predict(exchange, image); - exchange.getIn().setBody(result); - } catch (TypeConversionException e) { + image = exchange.getIn().getMandatoryBody(Image.class); + } catch (InvalidPayloadException e) { throw new RuntimeCamelException( - "Data type is not supported. Body should be ai.djl.modality.cv.Image, byte[], InputStream or File"); + "Data type is not supported. Body should be ai.djl.modality.cv.Image, byte[], InputStream or File", e); } + T result = predict(exchange, image); + exchange.getIn().setBody(result); } protected T predict(Exchange exchange, Image image) { diff --git a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/cv/CustomCvPredictor.java b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/cv/CustomCvPredictor.java index ccf327e3cfe8..3c2b4d547010 100644 --- a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/cv/CustomCvPredictor.java +++ b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/cv/CustomCvPredictor.java @@ -16,53 +16,42 @@ */ package org.apache.camel.component.djl.model.cv; -import ai.djl.Model; import ai.djl.inference.Predictor; import ai.djl.modality.cv.Image; import ai.djl.translate.TranslateException; -import ai.djl.translate.Translator; import org.apache.camel.Exchange; +import org.apache.camel.InvalidPayloadException; import org.apache.camel.RuntimeCamelException; -import org.apache.camel.TypeConversionException; import org.apache.camel.component.djl.DJLConstants; import org.apache.camel.component.djl.DJLEndpoint; -import org.apache.camel.component.djl.model.AbstractPredictor; +import org.apache.camel.component.djl.model.AbstractCustomPredictor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CustomCvPredictor<T> extends AbstractPredictor { +public class CustomCvPredictor<T> extends AbstractCustomPredictor<Image, T> { private static final Logger LOG = LoggerFactory.getLogger(CustomCvPredictor.class); - protected final String modelName; - protected final String translatorName; - public CustomCvPredictor(DJLEndpoint endpoint) { super(endpoint); - this.modelName = endpoint.getModel(); - this.translatorName = endpoint.getTranslator(); } @Override public void process(Exchange exchange) throws Exception { + Image image; try { - Image image = exchange.getIn().getBody(Image.class); - T result = predict(exchange, image); - exchange.getIn().setBody(result); - } catch (TypeConversionException e) { + image = exchange.getIn().getMandatoryBody(Image.class); + } catch (InvalidPayloadException e) { throw new RuntimeCamelException( - "Data type is not supported. Body should be ai.djl.modality.cv.Image, byte[], InputStream or File"); + "Data type is not supported. Body should be ai.djl.modality.cv.Image, byte[], InputStream or File", e); } + T result = predict(exchange, image); + exchange.getIn().setBody(result); } protected T predict(Exchange exchange, Image image) { - Model model = exchange.getContext().getRegistry().lookupByNameAndType(modelName, Model.class); - @SuppressWarnings("unchecked") - Translator<Image, T> translator - = exchange.getContext().getRegistry().lookupByNameAndType(translatorName, Translator.class); - exchange.getIn().setHeader(DJLConstants.INPUT, image); - try (Predictor<Image, T> predictor = model.newPredictor(translator)) { + try (Predictor<Image, T> predictor = newPredictor()) { return predictor.predict(image); } catch (TranslateException e) { LOG.error("Could not process input or output", e); diff --git a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/cv/CustomImageGenerationPredictor.java b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/cv/CustomImageGenerationPredictor.java index a2f5197503d4..c543d6e577a9 100644 --- a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/cv/CustomImageGenerationPredictor.java +++ b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/cv/CustomImageGenerationPredictor.java @@ -16,26 +16,19 @@ */ package org.apache.camel.component.djl.model.cv; -import ai.djl.Model; import ai.djl.inference.Predictor; import ai.djl.modality.cv.Image; import ai.djl.translate.TranslateException; -import ai.djl.translate.Translator; import org.apache.camel.Exchange; import org.apache.camel.RuntimeCamelException; import org.apache.camel.component.djl.DJLConstants; import org.apache.camel.component.djl.DJLEndpoint; -import org.apache.camel.component.djl.model.AbstractPredictor; +import org.apache.camel.component.djl.model.AbstractCustomPredictor; -public class CustomImageGenerationPredictor extends AbstractPredictor { - - private final String modelName; - private final String translatorName; +public class CustomImageGenerationPredictor extends AbstractCustomPredictor<int[], Image[]> { public CustomImageGenerationPredictor(DJLEndpoint endpoint) { super(endpoint); - this.modelName = endpoint.getModel(); - this.translatorName = endpoint.getTranslator(); } @Override @@ -50,13 +43,8 @@ public class CustomImageGenerationPredictor extends AbstractPredictor { } protected Image[] predict(Exchange exchange, int[] seed) { - Model model = exchange.getContext().getRegistry().lookupByNameAndType(modelName, Model.class); - @SuppressWarnings("unchecked") - Translator<int[], Image[]> translator - = exchange.getContext().getRegistry().lookupByNameAndType(translatorName, Translator.class); - exchange.getIn().setHeader(DJLConstants.INPUT, seed); - try (Predictor<int[], Image[]> predictor = model.newPredictor(translator)) { + try (Predictor<int[], Image[]> predictor = newPredictor()) { return predictor.predict(seed); } catch (TranslateException e) { throw new RuntimeCamelException("Could not process input or output", e); diff --git a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/nlp/AbstractNlpZooPredictor.java b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/nlp/AbstractNlpZooPredictor.java index 557750e889db..9284aba7268f 100644 --- a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/nlp/AbstractNlpZooPredictor.java +++ b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/nlp/AbstractNlpZooPredictor.java @@ -20,6 +20,7 @@ import ai.djl.inference.Predictor; import ai.djl.repository.zoo.ZooModel; import ai.djl.translate.TranslateException; import org.apache.camel.Exchange; +import org.apache.camel.InvalidPayloadException; import org.apache.camel.RuntimeCamelException; import org.apache.camel.component.djl.DJLConstants; import org.apache.camel.component.djl.DJLEndpoint; @@ -35,13 +36,14 @@ public abstract class AbstractNlpZooPredictor<T> extends AbstractPredictor { @Override public void process(Exchange exchange) { - if (exchange.getIn().getBody() instanceof String) { - String input = exchange.getIn().getBody(String.class); - T result = predict(exchange, input); - exchange.getIn().setBody(result); - } else { - throw new RuntimeCamelException("Data type is not supported. Body should be String"); + String input; + try { + input = exchange.getIn().getMandatoryBody(String.class); + } catch (InvalidPayloadException e) { + throw new RuntimeCamelException("Data type is not supported. Body should be String, byte[] or InputStream", e); } + T result = predict(exchange, input); + exchange.getIn().setBody(result); } protected T predict(Exchange exchange, String input) { diff --git a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/nlp/CustomNlpPredictor.java b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/nlp/CustomNlpPredictor.java index dfe3a0a9c0ed..8ef43d75bb4d 100644 --- a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/nlp/CustomNlpPredictor.java +++ b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/nlp/CustomNlpPredictor.java @@ -16,46 +16,36 @@ */ package org.apache.camel.component.djl.model.nlp; -import ai.djl.Model; import ai.djl.inference.Predictor; import ai.djl.translate.TranslateException; -import ai.djl.translate.Translator; import org.apache.camel.Exchange; +import org.apache.camel.InvalidPayloadException; import org.apache.camel.RuntimeCamelException; import org.apache.camel.component.djl.DJLConstants; import org.apache.camel.component.djl.DJLEndpoint; -import org.apache.camel.component.djl.model.AbstractPredictor; +import org.apache.camel.component.djl.model.AbstractCustomPredictor; -public class CustomNlpPredictor<T> extends AbstractPredictor { - - protected final String modelName; - protected final String translatorName; +public class CustomNlpPredictor<T> extends AbstractCustomPredictor<String, T> { public CustomNlpPredictor(DJLEndpoint endpoint) { super(endpoint); - this.modelName = endpoint.getModel(); - this.translatorName = endpoint.getTranslator(); } @Override public void process(Exchange exchange) { - if (exchange.getIn().getBody() instanceof String) { - String input = exchange.getIn().getBody(String.class); - T result = predict(exchange, input); - exchange.getIn().setBody(result); - } else { - throw new RuntimeCamelException("Data type is not supported. Body should be String"); + String input; + try { + input = exchange.getIn().getMandatoryBody(String.class); + } catch (InvalidPayloadException e) { + throw new RuntimeCamelException("Data type is not supported. Body should be String, byte[] or InputStream", e); } + T result = predict(exchange, input); + exchange.getIn().setBody(result); } protected T predict(Exchange exchange, String input) { - Model model = exchange.getContext().getRegistry().lookupByNameAndType(modelName, Model.class); - @SuppressWarnings("unchecked") - Translator<String, T> translator - = exchange.getContext().getRegistry().lookupByNameAndType(translatorName, Translator.class); - exchange.getIn().setHeader(DJLConstants.INPUT, input); - try (Predictor<String, T> predictor = model.newPredictor(translator)) { + try (Predictor<String, T> predictor = newPredictor()) { return predictor.predict(input); } catch (TranslateException e) { throw new RuntimeCamelException("Could not process input or output", e); diff --git a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/nlp/CustomQuestionAnswerPredictor.java b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/nlp/CustomQuestionAnswerPredictor.java index e54d172fa288..adf7b1468499 100644 --- a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/nlp/CustomQuestionAnswerPredictor.java +++ b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/nlp/CustomQuestionAnswerPredictor.java @@ -16,26 +16,19 @@ */ package org.apache.camel.component.djl.model.nlp; -import ai.djl.Model; import ai.djl.inference.Predictor; import ai.djl.modality.nlp.qa.QAInput; import ai.djl.translate.TranslateException; -import ai.djl.translate.Translator; import org.apache.camel.Exchange; import org.apache.camel.RuntimeCamelException; import org.apache.camel.component.djl.DJLConstants; import org.apache.camel.component.djl.DJLEndpoint; -import org.apache.camel.component.djl.model.AbstractPredictor; +import org.apache.camel.component.djl.model.AbstractCustomPredictor; -public class CustomQuestionAnswerPredictor extends AbstractPredictor { - - private final String modelName; - private final String translatorName; +public class CustomQuestionAnswerPredictor extends AbstractCustomPredictor<QAInput, String> { public CustomQuestionAnswerPredictor(DJLEndpoint endpoint) { super(endpoint); - this.modelName = endpoint.getModel(); - this.translatorName = endpoint.getTranslator(); } @Override @@ -59,13 +52,8 @@ public class CustomQuestionAnswerPredictor extends AbstractPredictor { } protected String predict(Exchange exchange, QAInput input) { - Model model = exchange.getContext().getRegistry().lookupByNameAndType(modelName, Model.class); - @SuppressWarnings("unchecked") - Translator<QAInput, String> translator - = exchange.getContext().getRegistry().lookupByNameAndType(translatorName, Translator.class); - exchange.getIn().setHeader(DJLConstants.INPUT, input); - try (Predictor<QAInput, String> predictor = model.newPredictor(translator)) { + try (Predictor<QAInput, String> predictor = newPredictor()) { return predictor.predict(input); } catch (TranslateException e) { throw new RuntimeCamelException("Could not process input or output", e); diff --git a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/tabular/CustomTabularPredictor.java b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/tabular/CustomTabularPredictor.java index 1c999c5c52bd..4080b0ab373e 100644 --- a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/tabular/CustomTabularPredictor.java +++ b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/tabular/CustomTabularPredictor.java @@ -16,25 +16,18 @@ */ package org.apache.camel.component.djl.model.tabular; -import ai.djl.Model; import ai.djl.inference.Predictor; import ai.djl.translate.TranslateException; -import ai.djl.translate.Translator; import org.apache.camel.Exchange; import org.apache.camel.RuntimeCamelException; import org.apache.camel.component.djl.DJLConstants; import org.apache.camel.component.djl.DJLEndpoint; -import org.apache.camel.component.djl.model.AbstractPredictor; +import org.apache.camel.component.djl.model.AbstractCustomPredictor; -public class CustomTabularPredictor extends AbstractPredictor { - - protected final String modelName; - protected final String translatorName; +public class CustomTabularPredictor extends AbstractCustomPredictor<Object, Object> { public CustomTabularPredictor(DJLEndpoint endpoint) { super(endpoint); - this.modelName = endpoint.getModel(); - this.translatorName = endpoint.getTranslator(); } @Override @@ -44,14 +37,9 @@ public class CustomTabularPredictor extends AbstractPredictor { exchange.getIn().setBody(result); } - @SuppressWarnings({ "unchecked", "rawtypes" }) protected Object predict(Exchange exchange, Object input) { - Model model = exchange.getContext().getRegistry().lookupByNameAndType(modelName, Model.class); - Translator translator - = exchange.getContext().getRegistry().lookupByNameAndType(translatorName, Translator.class); - exchange.getIn().setHeader(DJLConstants.INPUT, input); - try (Predictor predictor = model.newPredictor(translator)) { + try (Predictor<Object, Object> predictor = newPredictor()) { return predictor.predict(input); } catch (TranslateException e) { throw new RuntimeCamelException("Could not process input or output", e); diff --git a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/timeseries/CustomForecastingPredictor.java b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/timeseries/CustomForecastingPredictor.java index 34c2ab36b4f3..a5ddfce09c33 100644 --- a/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/timeseries/CustomForecastingPredictor.java +++ b/components/camel-ai/camel-djl/src/main/java/org/apache/camel/component/djl/model/timeseries/CustomForecastingPredictor.java @@ -16,27 +16,20 @@ */ package org.apache.camel.component.djl.model.timeseries; -import ai.djl.Model; import ai.djl.inference.Predictor; import ai.djl.timeseries.Forecast; import ai.djl.timeseries.TimeSeriesData; import ai.djl.translate.TranslateException; -import ai.djl.translate.Translator; import org.apache.camel.Exchange; import org.apache.camel.RuntimeCamelException; import org.apache.camel.component.djl.DJLConstants; import org.apache.camel.component.djl.DJLEndpoint; -import org.apache.camel.component.djl.model.AbstractPredictor; +import org.apache.camel.component.djl.model.AbstractCustomPredictor; -public class CustomForecastingPredictor extends AbstractPredictor { - - protected final String modelName; - protected final String translatorName; +public class CustomForecastingPredictor extends AbstractCustomPredictor<TimeSeriesData, Forecast> { public CustomForecastingPredictor(DJLEndpoint endpoint) { super(endpoint); - this.modelName = endpoint.getModel(); - this.translatorName = endpoint.getTranslator(); } @Override @@ -51,13 +44,8 @@ public class CustomForecastingPredictor extends AbstractPredictor { } protected Forecast predict(Exchange exchange, TimeSeriesData input) { - Model model = exchange.getContext().getRegistry().lookupByNameAndType(modelName, Model.class); - @SuppressWarnings("unchecked") - Translator<TimeSeriesData, Forecast> translator - = exchange.getContext().getRegistry().lookupByNameAndType(translatorName, Translator.class); - exchange.getIn().setHeader(DJLConstants.INPUT, input); - try (Predictor<TimeSeriesData, Forecast> predictor = model.newPredictor(translator)) { + try (Predictor<TimeSeriesData, Forecast> predictor = newPredictor()) { return predictor.predict(input); } catch (TranslateException e) { throw new RuntimeCamelException("Could not process input or output", e); diff --git a/components/camel-ai/camel-djl/src/test/java/org/apache/camel/component/djl/model/PredictorBodyTest.java b/components/camel-ai/camel-djl/src/test/java/org/apache/camel/component/djl/model/PredictorBodyTest.java new file mode 100644 index 000000000000..1046b2783848 --- /dev/null +++ b/components/camel-ai/camel-djl/src/test/java/org/apache/camel/component/djl/model/PredictorBodyTest.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.djl.model; + +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.LinkedHashMap; +import java.util.Map; + +import ai.djl.Model; +import ai.djl.inference.Predictor; +import ai.djl.modality.audio.Audio; +import ai.djl.modality.cv.Image; +import ai.djl.repository.zoo.Criteria; +import ai.djl.repository.zoo.ModelZoo; +import ai.djl.repository.zoo.ZooModel; +import ai.djl.timeseries.TimeSeriesData; +import ai.djl.translate.Translator; +import org.apache.camel.Exchange; +import org.apache.camel.NoSuchBeanException; +import org.apache.camel.Producer; +import org.apache.camel.RuntimeCamelException; +import org.apache.camel.component.djl.DJLComponent; +import org.apache.camel.component.djl.DJLConstants; +import org.apache.camel.component.djl.DJLEndpoint; +import org.apache.camel.component.djl.model.audio.ZooAudioPredictor; +import org.apache.camel.component.djl.model.cv.AbstractCvZooPredictor; +import org.apache.camel.component.djl.model.nlp.AbstractNlpZooPredictor; +import org.apache.camel.spi.Registry; +import org.apache.camel.support.DefaultExchange; +import org.apache.camel.test.junit6.CamelTestSupport; +import org.apache.camel.util.ObjectHelper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The body handling of the predictors and the lookup of a custom model, with a mocked DJL model and predictor so no + * model is loaded or downloaded. + */ +class PredictorBodyTest extends CamelTestSupport { + + private static final String IMAGE_FILE = "src/test/resources/data/mnist/0/10.png"; + + private final Model model = mock(Model.class); + private final ZooModel<Object, Object> zooModel = mock(ZooModel.class); + private final Predictor<Object, Object> predictor = mock(Predictor.class); + private final Translator<Object, Object> translator = mock(Translator.class); + + @Override + protected void bindToRegistry(Registry registry) { + registry.bind("MyModel", model); + registry.bind("MyTranslator", translator); + } + + @BeforeEach + void setupPredictor() throws Exception { + when(model.newPredictor(any(Translator.class))).thenReturn(predictor); + when(zooModel.newPredictor()).thenReturn(predictor); + when(predictor.predict(any())).thenAnswer(invocation -> "predicted"); + } + + @Test + void customCvPredictorRejectsBodyWithoutConverter() throws Exception { + try (Producer producer = startProducer("djl:cv/image_classification?model=MyModel&translator=MyTranslator")) { + for (Object body : new Object[] { + "not an image", Map.of("a", 1), new BufferedImage(1, 1, BufferedImage.TYPE_INT_RGB), null }) { + assertNotSupported(producer, body, "ai.djl.modality.cv.Image, byte[], InputStream or File"); + } + verify(predictor, never()).predict(any()); + } + } + + @Test + void customCvPredictorConvertsImageBytes() throws Exception { + try (Producer producer = startProducer("djl:cv/image_classification?model=MyModel&translator=MyTranslator")) { + Exchange exchange = exchange(Files.readAllBytes(Paths.get(IMAGE_FILE))); + producer.process(exchange); + + assertEquals("predicted", exchange.getMessage().getBody()); + assertInstanceOf(Image.class, exchange.getMessage().getHeader(DJLConstants.INPUT)); + } + } + + @Test + void zooCvPredictorRejectsBodyWithoutConverter() throws Exception { + AbstractCvZooPredictor<Object> cv = new AbstractCvZooPredictor<>(endpoint("cv/image_classification")) { + { + model = mockZooModel(); + } + }; + for (Object body : new Object[] { + "not an image", Map.of("a", 1), new BufferedImage(1, 1, BufferedImage.TYPE_INT_RGB), null }) { + assertNotSupported(cv, body, "ai.djl.modality.cv.Image, byte[], InputStream or File"); + } + verify(predictor, never()).predict(any()); + } + + @Test + void customAudioPredictorRejectsBodyWithoutConverter() throws Exception { + try (Producer producer = startProducer("djl:audio?model=MyModel&translator=MyTranslator")) { + for (Object body : new Object[] { "not an audio clip", Map.of("a", 1), null }) { + assertNotSupported(producer, body, "ai.djl.modality.audio.Audio, byte[], InputStream or File"); + } + verify(predictor, never()).predict(any()); + + Audio audio = new Audio(new float[] { 0.1f, 0.2f }); + Exchange exchange = exchange(audio); + producer.process(exchange); + assertEquals("predicted", exchange.getMessage().getBody()); + verify(predictor).predict(audio); + } + } + + @Test + void zooAudioPredictorRejectsBodyWithoutConverter() throws Exception { + DJLEndpoint endpoint = endpoint("audio"); + endpoint.setArtifactId("ai.djl.test:audio:0.0.1"); + ZooAudioPredictor audio; + try (MockedStatic<ModelZoo> zoo = mockStatic(ModelZoo.class)) { + zoo.when(() -> ModelZoo.loadModel(any(Criteria.class))).thenReturn(zooModel); + audio = new ZooAudioPredictor(endpoint); + } + for (Object body : new Object[] { "not an audio clip", Map.of("a", 1), null }) { + assertNotSupported(audio, body, "ai.djl.modality.audio.Audio, byte[], InputStream or File"); + } + verify(predictor, never()).predict(any()); + } + + @Test + void customNlpPredictorTakesTextAsStringBytesOrStream() throws Exception { + try (Producer producer = startProducer("djl:nlp/sentiment_analysis?model=MyModel&translator=MyTranslator")) { + byte[] text = "I like Camel".getBytes(StandardCharsets.UTF_8); + for (Object body : new Object[] { "I like Camel", text, new ByteArrayInputStream(text) }) { + Exchange exchange = exchange(body); + producer.process(exchange); + assertEquals("predicted", exchange.getMessage().getBody()); + assertEquals("I like Camel", exchange.getMessage().getHeader(DJLConstants.INPUT)); + } + + assertNotSupported(producer, null, "String, byte[] or InputStream"); + } + } + + @Test + void zooNlpPredictorTakesTextAsStringBytesOrStream() throws Exception { + AbstractNlpZooPredictor<Object> nlp = new AbstractNlpZooPredictor<>(endpoint("nlp/sentiment_analysis")) { + { + model = mockZooModel(); + } + }; + byte[] text = "I like Camel".getBytes(StandardCharsets.UTF_8); + for (Object body : new Object[] { "I like Camel", text, new ByteArrayInputStream(text) }) { + Exchange exchange = exchange(body); + nlp.process(exchange); + assertEquals("predicted", exchange.getMessage().getBody()); + assertEquals("I like Camel", exchange.getMessage().getHeader(DJLConstants.INPUT)); + } + + assertNotSupported(nlp, null, "String, byte[] or InputStream"); + } + + @Test + void customPredictorNamesTheMissingBean() throws Exception { + Map<String, Object> bodies = new LinkedHashMap<>(); + bodies.put("cv/image_classification", Files.readAllBytes(Paths.get(IMAGE_FILE))); + bodies.put("cv/image_generation", new int[] { 1, 2 }); + bodies.put("nlp/question_answer", new String[] { "Who?", "Camel." }); + bodies.put("nlp/text_generation", "Camel"); + bodies.put("audio", new Audio(new float[] { 0.1f })); + bodies.put("tabular/linear_regression", "1,2,3"); + bodies.put("timeseries/forecasting", new TimeSeriesData(1)); + for (Map.Entry<String, Object> entry : bodies.entrySet()) { + String uri = "djl:" + entry.getKey() + "?model=NoSuchModel&translator=MyTranslator"; + try (Producer producer = startProducer(uri)) { + NoSuchBeanException e = assertThrows(NoSuchBeanException.class, + () -> producer.process(exchange(entry.getValue())), uri); + assertEquals("NoSuchModel", e.getName()); + } + + uri = "djl:" + entry.getKey() + "?model=MyModel&translator=NoSuchTranslator"; + try (Producer producer = startProducer(uri)) { + NoSuchBeanException e = assertThrows(NoSuchBeanException.class, + () -> producer.process(exchange(entry.getValue())), uri); + assertEquals("NoSuchTranslator", e.getName()); + } + } + verify(predictor, never()).predict(any()); + } + + @Test + void customPredictorLooksUpTheModelOnce() throws Exception { + try (Producer producer = startProducer("djl:nlp/text_generation?model=MyModel&translator=MyTranslator")) { + producer.process(exchange("first")); + + context.getRegistry().unbind("MyModel"); + context.getRegistry().unbind("MyTranslator"); + Exchange exchange = exchange("second"); + producer.process(exchange); + + assertEquals("predicted", exchange.getMessage().getBody()); + verify(model, times(2)).newPredictor(translator); + } + } + + @Test + void customPredictorFailsToStartWithoutModelOption() { + Exception e = assertThrows(Exception.class, + () -> startProducer("djl:cv/image_classification?translator=MyTranslator")); + IllegalArgumentException cause = ObjectHelper.getException(IllegalArgumentException.class, e); + assertNotNull(cause, e.toString()); + assertTrue(cause.getMessage().contains("model"), cause.getMessage()); + } + + @SuppressWarnings("unchecked") + private <I, O> ZooModel<I, O> mockZooModel() { + return (ZooModel<I, O>) zooModel; + } + + private Producer startProducer(String uri) throws Exception { + Producer producer = context.getEndpoint(uri).createProducer(); + producer.start(); + return producer; + } + + private DJLEndpoint endpoint(String application) { + DJLEndpoint endpoint + = new DJLEndpoint("djl:" + application, context.getComponent("djl", DJLComponent.class), application); + endpoint.setCamelContext(context); + return endpoint; + } + + private Exchange exchange(Object body) { + Exchange exchange = new DefaultExchange(context); + exchange.getMessage().setBody(body); + return exchange; + } + + private void assertNotSupported(Producer producer, Object body, String types) { + RuntimeCamelException e = assertThrows(RuntimeCamelException.class, () -> producer.process(exchange(body)), + "body: " + body); + assertEquals("Data type is not supported. Body should be " + types, e.getMessage()); + } + + private void assertNotSupported(AbstractPredictor predictor, Object body, String types) { + RuntimeCamelException e = assertThrows(RuntimeCamelException.class, () -> predictor.process(exchange(body)), + "body: " + body); + assertEquals("Data type is not supported. Body should be " + types, e.getMessage()); + } +} diff --git a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc index 10f5a47c1b89..214d559bcf1b 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc @@ -3809,6 +3809,16 @@ a Spring Boot or Quarkus app that `camel run` runs (`<name>.log`), also while th tools keep, and the history of the AI prompt. Directories are not counted, and `--clean` counts only the files it removed. +=== camel-djl - the model and translator of a custom model are looked up once + +With a custom model (the `model` and `translator` options), the model and translator beans are looked up in the +registry at the first message and kept until the route is stopped. Before, they were looked up for every message. A +bean bound again under the same name is used after the route is restarted. A missing bean fails the message with a +`NoSuchBeanException` that names the bean, instead of a `NullPointerException`, and an endpoint without the `model` +or `translator` option fails to start. + +The nlp applications take a `byte[]` or `InputStream` body as text, as well as a `String`. + === camel-google-storage - a listed object that cannot be got is skipped When the consumer lists the bucket, a failure to download one of the listed objects (for example access denied, or an
