This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 3fbf4bfa5c29 CAMEL-26342: camel-langchain4j-ingest - expose the
includeId/excludeId matching for reuse (#28571)
3fbf4bfa5c29 is described below
commit 3fbf4bfa5c29737cd45f38275f51bb37df22dc30
Author: Jiří Ondrušek <[email protected]>
AuthorDate: Fri Oct 30 17:31:41 2026 +0100
CAMEL-26342: camel-langchain4j-ingest - expose the includeId/excludeId
matching for reuse (#28571)
Expose the includeId/excludeId document id matching of the producer as
IngestIdFilter, so a route can check an id before it parses a document. A null
id fails fast, as the producer refuses a missing id.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../catalog/docs/langchain4j-ingest-component.adoc | 5 ++
.../main/docs/langchain4j-ingest-component.adoc | 5 ++
.../langchain4j/ingest/IngestIdFilter.java | 72 ++++++++++++++++++++
.../ingest/LangChain4jIngestProducer.java | 31 +--------
.../langchain4j/ingest/IngestIdFilterTest.java | 76 ++++++++++++++++++++++
5 files changed, 161 insertions(+), 28 deletions(-)
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-ingest-component.adoc
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-ingest-component.adoc
index 8ddfe5679c1d..0a8ec8012dae 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-ingest-component.adoc
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-ingest-component.adoc
@@ -193,6 +193,11 @@ YAML::
----
====
+`IngestIdFilter` applies the same id patterns outside the endpoint: a route
that parses
+documents first can check `IngestIdFilter.of(includeId,
excludeId).accepts(documentId)`,
+with the same values as the endpoint, before the parse, so a filtered document
is never
+parsed.
+
=== Deduplicating deliveries
With an `idempotentRepository` configured, the producer claims each document
id before
diff --git
a/components/camel-ai/camel-langchain4j-ingest/src/main/docs/langchain4j-ingest-component.adoc
b/components/camel-ai/camel-langchain4j-ingest/src/main/docs/langchain4j-ingest-component.adoc
index 8ddfe5679c1d..0a8ec8012dae 100644
---
a/components/camel-ai/camel-langchain4j-ingest/src/main/docs/langchain4j-ingest-component.adoc
+++
b/components/camel-ai/camel-langchain4j-ingest/src/main/docs/langchain4j-ingest-component.adoc
@@ -193,6 +193,11 @@ YAML::
----
====
+`IngestIdFilter` applies the same id patterns outside the endpoint: a route
that parses
+documents first can check `IngestIdFilter.of(includeId,
excludeId).accepts(documentId)`,
+with the same values as the endpoint, before the parse, so a filtered document
is never
+parsed.
+
=== Deduplicating deliveries
With an `idempotentRepository` configured, the producer claims each document
id before
diff --git
a/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/IngestIdFilter.java
b/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/IngestIdFilter.java
new file mode 100644
index 000000000000..f2431e37dade
--- /dev/null
+++
b/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/IngestIdFilter.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.langchain4j.ingest;
+
+import java.util.Arrays;
+
+import org.apache.camel.util.AntPathMatcher;
+import org.apache.camel.util.ObjectHelper;
+
+/**
+ * The {@code includeId}/{@code excludeId} document id filter the producer
applies. Code that sees the document id
+ * before the producer, such as a route that parses documents first, can apply
the same filter, so a filtered document
+ * is never parsed.
+ */
+public final class IngestIdFilter {
+
+ private final String[] includeIds;
+ private final String[] excludeIds;
+
+ private IngestIdFilter(String[] includeIds, String[] excludeIds) {
+ this.includeIds = includeIds;
+ this.excludeIds = excludeIds;
+ }
+
+ /**
+ * The filter for the given option values: comma-separated Ant-style
patterns, matched case-sensitively; null or
+ * blank means no patterns.
+ */
+ public static IngestIdFilter of(String includeId, String excludeId) {
+ return new IngestIdFilter(parsePatterns(includeId),
parsePatterns(excludeId));
+ }
+
+ /**
+ * Exclusion wins over inclusion; with includeId set, only matching ids
pass.
+ *
+ * @throws IllegalArgumentException if the id is null, as the producer
refuses a missing id
+ */
+ public boolean accepts(String documentId) {
+ ObjectHelper.notNull(documentId, "documentId");
+ if (AntPathMatcher.INSTANCE.anyMatch(excludeIds, documentId)) {
+ return false;
+ }
+ return includeIds == null ||
AntPathMatcher.INSTANCE.anyMatch(includeIds, documentId);
+ }
+
+ // normalised to null when no pattern remains: an all-blank includeId must
accept
+ // everything, while an empty array handed to anyMatch would reject
everything
+ private static String[] parsePatterns(String patterns) {
+ if (patterns == null || patterns.isBlank()) {
+ return null;
+ }
+ String[] parsed = Arrays.stream(patterns.split(","))
+ .map(String::trim)
+ .filter(pattern -> !pattern.isEmpty())
+ .toArray(String[]::new);
+ return parsed.length == 0 ? null : parsed;
+ }
+}
diff --git
a/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestProducer.java
b/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestProducer.java
index 50bea6afe60e..7ca5e67d325d 100644
---
a/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestProducer.java
+++
b/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestProducer.java
@@ -20,7 +20,6 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
-import java.util.Arrays;
import java.util.Locale;
import java.util.Set;
@@ -35,7 +34,6 @@ import org.apache.camel.Predicate;
import org.apache.camel.spi.IdempotentRepository;
import org.apache.camel.support.DefaultProducer;
import org.apache.camel.support.service.ServiceHelper;
-import org.apache.camel.util.AntPathMatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,8 +59,7 @@ public class LangChain4jIngestProducer extends
DefaultProducer {
/** One of the two is set at start, by modality. */
private IngestService textService;
private MediaIngestService mediaService;
- private String[] includeIds;
- private String[] excludeIds;
+ private IngestIdFilter idFilter;
private boolean media;
/** The configured contentType, normalised; null when unset. Content-type
headers are ignored on purpose. */
private String contentType;
@@ -143,8 +140,7 @@ public class LangChain4jIngestProducer extends
DefaultProducer {
+
configuration.getMaxDocumentSize() + ")");
}
- includeIds = parsePatterns(configuration.getIncludeId());
- excludeIds = parsePatterns(configuration.getExcludeId());
+ idFilter = IngestIdFilter.of(configuration.getIncludeId(),
configuration.getExcludeId());
if (configuration.getDocumentFilter() != null) {
// initialised and started, but - like the repository below -
never stopped: the
// bean may be shared, and stopping it here would tear it down
under other users
@@ -221,7 +217,7 @@ public class LangChain4jIngestProducer extends
DefaultProducer {
// before the claim and the body read: a filtered document never
occupies its id and
// never pays for materialising the payload
- if (!idAccepted(documentId)) {
+ if (!idFilter.accepts(documentId)) {
exchange.getMessage().setBody(filtered(documentId));
return;
}
@@ -409,27 +405,6 @@ public class LangChain4jIngestProducer extends
DefaultProducer {
return type.isEmpty() ? null : type;
}
- /** Exclusion wins over inclusion; with includeId set, only matching ids
pass. */
- private boolean idAccepted(String documentId) {
- if (AntPathMatcher.INSTANCE.anyMatch(excludeIds, documentId)) {
- return false;
- }
- return includeIds == null ||
AntPathMatcher.INSTANCE.anyMatch(includeIds, documentId);
- }
-
- // normalised to null when no pattern remains: an all-blank includeId must
accept
- // everything, while an empty array handed to anyMatch would reject
everything
- private static String[] parsePatterns(String patterns) {
- if (patterns == null || patterns.isBlank()) {
- return null;
- }
- String[] parsed = Arrays.stream(patterns.split(","))
- .map(String::trim)
- .filter(pattern -> !pattern.isEmpty())
- .toArray(String[]::new);
- return parsed.length == 0 ? null : parsed;
- }
-
/**
* The exchange property wins over the header: a route that parses
documents captures the id into the property
* before the parse, and a parser (Tika) copies document metadata over the
headers, so a header read here could be
diff --git
a/components/camel-ai/camel-langchain4j-ingest/src/test/java/org/apache/camel/component/langchain4j/ingest/IngestIdFilterTest.java
b/components/camel-ai/camel-langchain4j-ingest/src/test/java/org/apache/camel/component/langchain4j/ingest/IngestIdFilterTest.java
new file mode 100644
index 000000000000..f1fa3752f532
--- /dev/null
+++
b/components/camel-ai/camel-langchain4j-ingest/src/test/java/org/apache/camel/component/langchain4j/ingest/IngestIdFilterTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.langchain4j.ingest;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * The includeId/excludeId semantics on their own; {@link
LangChain4jIngestFilterTest} covers them in the producer.
+ */
+class IngestIdFilterTest {
+
+ /** An all-blank includeId accepts every id, instead of rejecting every id
for lack of a matching pattern. */
+ @Test
+ void noPatternsAcceptEveryId() {
+ assertThat(IngestIdFilter.of(null,
null).accepts("docs/guide.md")).isTrue();
+ assertThat(IngestIdFilter.of("", "
").accepts("docs/guide.md")).isTrue();
+ assertThat(IngestIdFilter.of(" , ,",
",").accepts("docs/guide.md")).isTrue();
+ }
+
+ @Test
+ void onlyIdsMatchingAnIncludePatternPass() {
+ // blank entries and the spaces around a pattern are ignored
+ IngestIdFilter filter = IngestIdFilter.of(" docs/** , ,*.md", null);
+
+ assertThat(filter.accepts("docs/guide.md")).isTrue();
+ assertThat(filter.accepts("notes.md")).isTrue();
+ assertThat(filter.accepts("notes.txt")).isFalse();
+ }
+
+ @Test
+ void exclusionWinsOverInclusion() {
+ IngestIdFilter filter = IngestIdFilter.of("docs/**", "**/draft-*");
+
+ assertThat(filter.accepts("docs/guide.md")).isTrue();
+ assertThat(filter.accepts("docs/draft-plan.md")).isFalse();
+ // without includeId, every id not excluded passes
+ assertThat(IngestIdFilter.of(null,
"**/draft-*").accepts("notes.txt")).isTrue();
+ }
+
+ @Test
+ void matchingIsCaseSensitiveAndNeedsTheSameLeadingSlash() {
+ IngestIdFilter filter = IngestIdFilter.of("docs/**", null);
+
+ assertThat(filter.accepts("Docs/guide.md")).isFalse();
+ assertThat(filter.accepts("/docs/guide.md")).isFalse();
+ assertThat(IngestIdFilter.of("/docs/**",
null).accepts("/docs/guide.md")).isTrue();
+ }
+
+ /** A null id fails with and without patterns, instead of passing without
them and an NPE with them. */
+ @Test
+ void nullIdFails() {
+ assertThatThrownBy(() -> IngestIdFilter.of(null, null).accepts(null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("documentId");
+ assertThatThrownBy(() -> IngestIdFilter.of("docs/**",
"**/draft-*").accepts(null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("documentId");
+ }
+}
