This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new fac55782529b CAMEL-25043: camel-core - Tokenize language: fix bugs
found in a deep review (#26923)
fac55782529b is described below
commit fac55782529bee72ad083cc56c7c41ab85f28b62
Author: Claus Ibsen <[email protected]>
AuthorDate: Mon Sep 28 09:26:32 2026 +0200
CAMEL-25043: camel-core - Tokenize language: fix bugs found in a deep
review (#26923)
- a message without a body has no tokens instead of failing with a
NullPointerException
- skipFirst on an empty body no longer fails with NoSuchElementException
- pair mode reads from source (header, variable or property) instead of
always the body
- skipFirst applies in xml mode and pair mode
- pair mode with group joins the pairs without the start token between them
- a pair with the same start and end token is refused when the route is
created
- xml wrap mode (inheritNamespaceTagName=*) handles a comment, DOCTYPE or
CDATA before the
root, and multi-byte characters before the first token
- ValueBuilder/MockValueBuilder.tokenize(token, null, skipFirst) no longer
fails at startup
- the group delimiter is encoded in the exchange charset
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Signed-off-by: Claus Ibsen <[email protected]>
---
.../camel/component/mock/MockValueBuilder.java | 4 +-
.../camel/language/tokenizer/TokenizeLanguage.java | 32 ++--
.../language/tokenizer/TokenizeEdgeCasesTest.java | 181 +++++++++++++++++++++
.../org/apache/camel/processor/SplitterTest.java | 9 +-
.../support/builder/RecordableInputStreamTest.java | 15 ++
.../org/apache/camel/support/ExchangeHelper.java | 3 +-
.../apache/camel/support/GroupTokenIterator.java | 2 +-
.../camel/support/builder/ExpressionBuilder.java | 15 +-
.../support/builder/RecordableInputStream.java | 20 ++-
.../builder/TokenPairExpressionIterator.java | 10 ++
.../builder/TokenXMLExpressionIterator.java | 39 +++++
.../apache/camel/support/builder/ValueBuilder.java | 4 +-
.../ROOT/pages/camel-4x-upgrade-guide-4_23.adoc | 13 ++
13 files changed, 318 insertions(+), 29 deletions(-)
diff --git
a/components/camel-mock/src/main/java/org/apache/camel/component/mock/MockValueBuilder.java
b/components/camel-mock/src/main/java/org/apache/camel/component/mock/MockValueBuilder.java
index 4420a6180fb9..a42918a795cd 100644
---
a/components/camel-mock/src/main/java/org/apache/camel/component/mock/MockValueBuilder.java
+++
b/components/camel-mock/src/main/java/org/apache/camel/component/mock/MockValueBuilder.java
@@ -504,7 +504,9 @@ public class MockValueBuilder implements Expression,
Predicate {
// wrap in skip first (if group then it has its own skip-first
logic)
newExp = ExpressionBuilder.skipFirstExpression(newExp);
}
- newExp = ExpressionBuilder.groupIteratorExpression(newExp, token,
group, skipFirst);
+ if (group != null) {
+ newExp = ExpressionBuilder.groupIteratorExpression(newExp, token,
group, skipFirst);
+ }
return onNewValueBuilder(newExp);
}
diff --git
a/core/camel-core-languages/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java
b/core/camel-core-languages/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java
index 59d16652403c..feb0da7aed7b 100644
---
a/core/camel-core-languages/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java
+++
b/core/camel-core-languages/src/main/java/org/apache/camel/language/tokenizer/TokenizeLanguage.java
@@ -71,32 +71,34 @@ public class TokenizeLanguage extends
SingleInputTypedLanguageSupport {
throw new IllegalArgumentException("The option includeTokens
requires endToken to be specified.");
}
- Expression answer = null;
+ Expression answer;
if (xml) {
answer = ExpressionBuilder.tokenizeXMLExpression(source, token,
inheritNamespaceTagName);
} else if (endToken != null) {
- answer = ExpressionBuilder.tokenizePairExpression(token, endToken,
includeTokens);
+ answer = ExpressionBuilder.tokenizePairExpression(source, token,
endToken, includeTokens);
+ } else if (regex) {
+ answer = ExpressionBuilder.regexTokenizeExpression(source, token);
+ } else {
+ answer = ExpressionBuilder.tokenizeExpression(source, token);
}
- if (answer == null) {
- // use the regular tokenizer
- if (regex) {
- answer = ExpressionBuilder.regexTokenizeExpression(source,
token);
- } else {
- answer = ExpressionBuilder.tokenizeExpression(source, token);
- }
- if (group == null && skipFirst) {
- // wrap in skip first (if group then it has its own skip first
logic)
- answer = ExpressionBuilder.skipFirstExpression(answer);
- }
+ if (group == null && skipFirst) {
+ // wrap in skip first (if group then it has its own skip first
logic)
+ answer = ExpressionBuilder.skipFirstExpression(answer);
}
// if group then wrap answer in group expression
if (group != null) {
if (xml) {
- answer = ExpressionBuilder.groupXmlIteratorExpression(answer,
group);
+ answer = ExpressionBuilder.groupXmlIteratorExpression(answer,
group, skipFirst);
} else {
- String delim = groupDelimiter != null ? groupDelimiter : token;
+ String delim = groupDelimiter;
+ if (delim == null) {
+ // pairs that include their tokens are joined without a
delimiter (as in xml mode), as the start
+ // token between them would make them look like an
unfinished pair; bare values keep the token
+ // as boundary
+ delim = endToken != null && includeTokens ? "" : token;
+ }
answer = ExpressionBuilder.groupIteratorExpression(answer,
delim, group, skipFirst);
}
}
diff --git
a/core/camel-core/src/test/java/org/apache/camel/language/tokenizer/TokenizeEdgeCasesTest.java
b/core/camel-core/src/test/java/org/apache/camel/language/tokenizer/TokenizeEdgeCasesTest.java
new file mode 100644
index 000000000000..92930a773ff7
--- /dev/null
+++
b/core/camel-core/src/test/java/org/apache/camel/language/tokenizer/TokenizeEdgeCasesTest.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.language.tokenizer;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.function.Consumer;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.Exchange;
+import org.apache.camel.Expression;
+import org.apache.camel.FailedToCreateRouteException;
+import org.apache.camel.builder.Builder;
+import org.apache.camel.builder.LanguageBuilderFactory;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.mock.MockEndpoint;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+public class TokenizeEdgeCasesTest extends ContextTestSupport {
+
+ private static final String ORDERS = "<?xml
version=\"1.0\"?>%s<root%s><order>1</order><order>2</order></root>";
+
+ @Override
+ public boolean isUseRouteBuilder() {
+ return false;
+ }
+
+ private static LanguageBuilderFactory lang() {
+ return new LanguageBuilderFactory();
+ }
+
+ private MockEndpoint split(Expression expression, Object body, String
header, Object value) throws Exception {
+ return split(expression, body, e -> {
+ if (header != null) {
+ e.getMessage().setHeader(header, value);
+ }
+ });
+ }
+
+ private MockEndpoint split(Expression expression, Object body,
Consumer<Exchange> setup) throws Exception {
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+ from("direct:start").split(expression).to("mock:split");
+ }
+ });
+ context.start();
+ Exchange out = template.send("direct:start", e -> {
+ e.getMessage().setBody(body);
+ setup.accept(e);
+ });
+ assertThat(out.getException()).isNull();
+ return getMockEndpoint("mock:split");
+ }
+
+ private static List<String> bodies(MockEndpoint mock) {
+ return mock.getReceivedExchanges().stream().map(e ->
e.getMessage().getBody(String.class)).toList();
+ }
+
+ @Test
+ public void testNullBodyHasNoParts() throws Exception {
+ MockEndpoint mock = split(lang().tokenize().token(",").end(), null,
null, null);
+ assertThat(mock.getReceivedExchanges()).isEmpty();
+ }
+
+ @Test
+ public void testSkipFirstOnEmptyBody() throws Exception {
+ MockEndpoint mock =
split(lang().tokenize().token("\n").skipFirst(true).end(), "", null, null);
+ assertThat(mock.getReceivedExchanges()).isEmpty();
+ }
+
+ @Test
+ public void testSkipFirstWithoutGroupFromValueBuilder() throws Exception {
+ MockEndpoint mock = split(Builder.body().tokenize(",", (String) null,
true), "header,a,b", null, null);
+ assertThat(bodies(mock)).containsExactly("a", "b");
+ }
+
+ @Test
+ public void testPairFromSource() throws Exception {
+ MockEndpoint mock =
split(lang().tokenize().token("[").endToken("]").source("header:data").end(),
+ "[body1][body2]", "data", "[hdr1][hdr2]");
+ assertThat(bodies(mock)).containsExactly("hdr1", "hdr2");
+ }
+
+ @Test
+ public void testPairSkipFirst() throws Exception {
+ MockEndpoint mock =
split(lang().tokenize().token("[").endToken("]").skipFirst(true).end(),
"[1][2][3]", null, null);
+ assertThat(bodies(mock)).containsExactly("2", "3");
+ }
+
+ @Test
+ public void testPairGroupHasNoStartTokenBetweenParts() throws Exception {
+ MockEndpoint mock =
split(lang().tokenize().token("<a>").endToken("</a>").includeTokens(true).group(2).end(),
+ "<a>1</a><a>2</a><a>3</a>", null, null);
+ assertThat(bodies(mock)).containsExactly("<a>1</a><a>2</a>",
"<a>3</a>");
+ }
+
+ @Test
+ public void testPairGroupWithoutTokensKeepsStartTokenAsBoundary() throws
Exception {
+ MockEndpoint mock =
split(lang().tokenize().token("[").endToken("]").group(2).end(),
+ "[1][2][3]", null, null);
+ assertThat(bodies(mock)).containsExactly("1[2", "3");
+ }
+
+ @Test
+ public void testPairWithSameStartAndEndTokenIsRejected() {
+ assertThatThrownBy(() ->
split(lang().tokenize().token("'").endToken("'").end(), "'a' 'b'", null, null))
+ .isInstanceOf(FailedToCreateRouteException.class)
+ .rootCause().hasMessageContaining("The start and end token
must be different");
+ }
+
+ @Test
+ public void testXmlSkipFirst() throws Exception {
+ MockEndpoint mock =
split(lang().tokenize().token("order").xml(true).skipFirst(true).end(),
+ String.format(ORDERS, "", ""), null, null);
+ assertThat(bodies(mock)).containsExactly("<order>2</order>");
+ }
+
+ @Test
+ public void testXmlGroupSkipFirst() throws Exception {
+ MockEndpoint mock =
split(lang().tokenize().token("order").xml(true).group(2).skipFirst(true).end(),
+
"<root><order>1</order><order>2</order><order>3</order></root>", null, null);
+
assertThat(bodies(mock)).containsExactly("<order>2</order><order>3</order>");
+ }
+
+ @Test
+ public void testWrapWithCommentBeforeRoot() throws Exception {
+ MockEndpoint mock =
split(lang().tokenize().token("order").xml(true).inheritNamespaceTagName("*").end(),
+ String.format(ORDERS, "<!-- licensed to you -->", ""), null,
null);
+ assertThat(bodies(mock)).containsExactly(
+ "<?xml version=\"1.0\"?><!-- licensed to you
--><root><order>1</order></root>",
+ "<?xml version=\"1.0\"?><!-- licensed to you
--><root><order>2</order></root>");
+ }
+
+ @Test
+ public void testWrapWithDoctype() throws Exception {
+ String doctype = "<!DOCTYPE root [<!ELEMENT root (order*)>]>";
+ MockEndpoint mock =
split(lang().tokenize().token("order").xml(true).inheritNamespaceTagName("*").end(),
+ String.format(ORDERS, doctype, ""), null, null);
+ assertThat(bodies(mock)).containsExactly(
+ "<?xml version=\"1.0\"?>" + doctype +
"<root><order>1</order></root>",
+ "<?xml version=\"1.0\"?>" + doctype +
"<root><order>2</order></root>");
+ }
+
+ @Test
+ public void testWrapWithDoctypeWhitespaceBeforeClose() throws Exception {
+ String doctype = "<!DOCTYPE root [<!ELEMENT root (order*)>] \n>";
+ MockEndpoint mock =
split(lang().tokenize().token("order").xml(true).inheritNamespaceTagName("*").end(),
+ String.format(ORDERS, doctype, ""), null, null);
+ assertThat(bodies(mock)).containsExactly(
+ "<?xml version=\"1.0\"?>" + doctype +
"<root><order>1</order></root>",
+ "<?xml version=\"1.0\"?>" + doctype +
"<root><order>2</order></root>");
+ }
+
+ @Test
+ public void testWrapWithMultiByteCharactersBeforeToken() throws Exception {
+ byte[] body = String.format(ORDERS, "", "
name=\"æøå\"").getBytes(StandardCharsets.UTF_8);
+ MockEndpoint mock =
split(lang().tokenize().token("order").xml(true).inheritNamespaceTagName("*").end(),
+ body, e -> e.setProperty(Exchange.CHARSET_NAME, "UTF-8"));
+ assertThat(bodies(mock)).containsExactly(
+ "<?xml version=\"1.0\"?><root
name=\"æøå\"><order>1</order></root>",
+ "<?xml version=\"1.0\"?><root
name=\"æøå\"><order>2</order></root>");
+ }
+}
diff --git
a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterTest.java
b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterTest.java
index 9963b3c87df3..c907bdc5f6d2 100644
--- a/core/camel-core/src/test/java/org/apache/camel/processor/SplitterTest.java
+++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitterTest.java
@@ -94,14 +94,19 @@ public class SplitterTest extends ContextTestSupport {
}
@Test
- public void testEmptyBody() {
+ public void testEmptyBody() throws Exception {
+ MockEndpoint resultEndpoint = getMockEndpoint("mock:result");
+ resultEndpoint.expectedMessageCount(0);
+
Exchange result = template.request("direct:seqential", new Processor()
{
public void process(Exchange exchange) {
exchange.getIn().setHeader("foo", "bar");
}
});
- assertFalse(result.hasOut(), "Should not have out");
+ // no body has no parts to split
+ assertNull(result.getException(), "Should not fail");
+ assertMockEndpointsSatisfied();
}
@Test
diff --git
a/core/camel-core/src/test/java/org/apache/camel/support/builder/RecordableInputStreamTest.java
b/core/camel-core/src/test/java/org/apache/camel/support/builder/RecordableInputStreamTest.java
index 4de2bd9d16d6..356664aa2107 100644
---
a/core/camel-core/src/test/java/org/apache/camel/support/builder/RecordableInputStreamTest.java
+++
b/core/camel-core/src/test/java/org/apache/camel/support/builder/RecordableInputStreamTest.java
@@ -102,4 +102,19 @@ public class RecordableInputStreamTest {
ris.close();
}
+ @Test
+ public void testGetTextWithMultiByteCharacters() throws Exception {
+ byte[] data = "æøåabc".getBytes(StandardCharsets.UTF_8);
+ RecordableInputStream ris = new RecordableInputStream(new
ByteArrayInputStream(data), "utf-8");
+ byte[] buf = new byte[64];
+ assertEquals(data.length, ris.read(buf, 0, buf.length));
+
+ // the position is in characters, and the rest is kept
+ assertEquals("æøåa", ris.getText(4));
+ assertEquals(2, ris.size());
+ assertEquals("bc", ris.getText(2));
+ assertEquals(0, ris.size());
+
+ ris.close();
+ }
}
diff --git
a/core/camel-support/src/main/java/org/apache/camel/support/ExchangeHelper.java
b/core/camel-support/src/main/java/org/apache/camel/support/ExchangeHelper.java
index 64f83528f874..d7ecd3df23f8 100644
---
a/core/camel-support/src/main/java/org/apache/camel/support/ExchangeHelper.java
+++
b/core/camel-support/src/main/java/org/apache/camel/support/ExchangeHelper.java
@@ -1055,7 +1055,8 @@ public final class ExchangeHelper {
} else {
// value is not a suitable type, try to convert value to a
string
String text =
exchange.getContext().getTypeConverter().convertTo(String.class, exchange,
value);
- scanner = new Scanner(text, delimiter);
+ // a null value (such as no message body) has no tokens
+ scanner = new Scanner(text != null ? text : "", delimiter);
}
}
return scanner;
diff --git
a/core/camel-support/src/main/java/org/apache/camel/support/GroupTokenIterator.java
b/core/camel-support/src/main/java/org/apache/camel/support/GroupTokenIterator.java
index 386469f3c3a3..d3e8a28ac0cf 100644
---
a/core/camel-support/src/main/java/org/apache/camel/support/GroupTokenIterator.java
+++
b/core/camel-support/src/main/java/org/apache/camel/support/GroupTokenIterator.java
@@ -140,7 +140,7 @@ public final class GroupTokenIterator implements
Iterator<Object>, Closeable {
// include token in between
if (data != null && count > 0 && token != null) {
- bos.write(token.getBytes());
+ bos.write(token.getBytes(ExchangeHelper.getCharset(exchange)));
}
if (data instanceof InputStream is) {
IOHelper.copy(is, bos);
diff --git
a/core/camel-support/src/main/java/org/apache/camel/support/builder/ExpressionBuilder.java
b/core/camel-support/src/main/java/org/apache/camel/support/builder/ExpressionBuilder.java
index 49d63c3005b3..ab024a2afd9b 100644
---
a/core/camel-support/src/main/java/org/apache/camel/support/builder/ExpressionBuilder.java
+++
b/core/camel-support/src/main/java/org/apache/camel/support/builder/ExpressionBuilder.java
@@ -1789,8 +1789,10 @@ public class ExpressionBuilder {
Object value = expression.evaluate(exchange, Object.class);
Iterator<?> it = typeConverter.tryConvertTo(Iterator.class,
exchange, value);
if (it != null) {
- // skip first
- it.next();
+ // skip first (there is nothing to skip when there is no
content)
+ if (it.hasNext()) {
+ it.next();
+ }
return it;
} else {
return value;
@@ -1838,6 +1840,11 @@ public class ExpressionBuilder {
}
public static Expression groupXmlIteratorExpression(final Expression
expression, final String group) {
+ return groupXmlIteratorExpression(expression, group, false);
+ }
+
+ public static Expression groupXmlIteratorExpression(
+ final Expression expression, final String group, final boolean
skipFirst) {
return new ExpressionAdapter() {
private Expression groupExp;
@@ -1857,7 +1864,7 @@ public class ExpressionBuilder {
} else if (parts <= 0) {
throw new RuntimeExchangeException("Group must be a
positive number, was: " + parts, exchange);
}
- return new GroupTokenIterator(exchange, it, null, parts,
false);
+ return new GroupTokenIterator(exchange, it, null, parts,
skipFirst);
}
@Override
@@ -2632,7 +2639,7 @@ public class ExpressionBuilder {
*/
public static Expression tokenizePairExpression(
Expression source, String startToken, String endToken, boolean
includeTokens) {
- return new TokenPairExpressionIterator(startToken, endToken,
includeTokens);
+ return new TokenPairExpressionIterator(source, startToken, endToken,
includeTokens);
}
/**
diff --git
a/core/camel-support/src/main/java/org/apache/camel/support/builder/RecordableInputStream.java
b/core/camel-support/src/main/java/org/apache/camel/support/builder/RecordableInputStream.java
index 4b2a7b59c819..53e7d259221c 100644
---
a/core/camel-support/src/main/java/org/apache/camel/support/builder/RecordableInputStream.java
+++
b/core/camel-support/src/main/java/org/apache/camel/support/builder/RecordableInputStream.java
@@ -40,7 +40,7 @@ class RecordableInputStream extends FilterInputStream {
@Override
public int read() throws IOException {
int c = super.read();
- if (c > 0 && recording) {
+ if (c >= 0 && recording) {
buf.write(c);
}
return c;
@@ -55,19 +55,31 @@ class RecordableInputStream extends FilterInputStream {
return n;
}
+ /**
+ * Returns the recorded text before the given position, and stops
recording.
+ *
+ * @param pos the position in characters (not bytes) as the scanner
reports it
+ */
public String getText(int pos) {
String t = null;
+ int len = 0;
recording = false;
try {
+ // decode what was recorded and cut by characters, as a character
may take more than one byte
if (charset == null) {
- t = new String(buf.getByteArray(), 0, pos);
+ t = new String(buf.getByteArray(), 0, buf.size());
+ t = t.substring(0, Math.min(pos, t.length()));
+ len = t.getBytes().length;
} else {
- t = new String(buf.getByteArray(), 0, pos, charset);
+ t = new String(buf.getByteArray(), 0, buf.size(), charset);
+ t = t.substring(0, Math.min(pos, t.length()));
+ len = t.getBytes(charset).length;
}
} catch (UnsupportedEncodingException e) {
// ignore it as this encoding exception should have been caught
earlier while scanning.
} finally {
- buf.trim(pos, 0);
+ // keep what was recorded after the text
+ buf.trim(Math.min(len, buf.size()), 0);
}
return t;
diff --git
a/core/camel-support/src/main/java/org/apache/camel/support/builder/TokenPairExpressionIterator.java
b/core/camel-support/src/main/java/org/apache/camel/support/builder/TokenPairExpressionIterator.java
index 87dbc8e6718f..dcf4dfa98e8e 100644
---
a/core/camel-support/src/main/java/org/apache/camel/support/builder/TokenPairExpressionIterator.java
+++
b/core/camel-support/src/main/java/org/apache/camel/support/builder/TokenPairExpressionIterator.java
@@ -58,6 +58,7 @@ public class TokenPairExpressionIterator extends
ExpressionAdapter {
public TokenPairExpressionIterator(Expression source, String startToken,
String endToken, boolean includeTokens) {
StringHelper.notEmpty(startToken, "startToken");
StringHelper.notEmpty(endToken, "endToken");
+ checkTokensDiffer(startToken, endToken);
this.startToken = startToken;
this.endToken = endToken;
this.includeTokens = includeTokens;
@@ -140,11 +141,20 @@ public class TokenPairExpressionIterator extends
ExpressionAdapter {
if (endExp != null) {
end = endExp.evaluate(exchange, String.class);
}
+ checkTokensDiffer(start, end);
TokenPairIterator iterator = new TokenPairIterator(start, end,
includeTokens, in, charset);
iterator.init();
return iterator;
}
+ private static void checkTokensDiffer(String startToken, String endToken) {
+ // the end token is the delimiter of the scanner, so a start token
that is the same is never found
+ if (startToken != null && startToken.equals(endToken)) {
+ throw new IllegalArgumentException(
+ "The start and end token must be different, both are: " +
startToken);
+ }
+ }
+
@Override
public String toString() {
return "tokenize[body() using tokens: " + startToken + "..." +
endToken + "]";
diff --git
a/core/camel-support/src/main/java/org/apache/camel/support/builder/TokenXMLExpressionIterator.java
b/core/camel-support/src/main/java/org/apache/camel/support/builder/TokenXMLExpressionIterator.java
index 0cd00f8b8272..009db08f3242 100644
---
a/core/camel-support/src/main/java/org/apache/camel/support/builder/TokenXMLExpressionIterator.java
+++
b/core/camel-support/src/main/java/org/apache/camel/support/builder/TokenXMLExpressionIterator.java
@@ -374,6 +374,41 @@ public class TokenXMLExpressionIterator extends
ExpressionAdapter {
return namespaces;
}
+ private static int skipMarkupDeclaration(String xmlhead, int p) {
+ String end;
+ if (xmlhead.startsWith("<!--", p)) {
+ end = "-->";
+ } else if (xmlhead.startsWith("<![CDATA[", p)) {
+ end = "]]>";
+ } else {
+ // DOCTYPE which may have an internal subset with declarations of
its own
+ int bracket = xmlhead.indexOf('[', p);
+ int gt = xmlhead.indexOf('>', p);
+ if (bracket >= 0 && (gt < 0 || bracket < gt)) {
+ return skipInternalSubset(xmlhead, bracket);
+ }
+ end = ">";
+ }
+ int ep = xmlhead.indexOf(end, p);
+ return ep < 0 ? xmlhead.length() : ep + end.length();
+ }
+
+ private static int skipInternalSubset(String xmlhead, int bracket) {
+ // the internal subset ends with ']' S? '>'
+ int p = xmlhead.indexOf(']', bracket);
+ while (p >= 0) {
+ int q = p + 1;
+ while (q < xmlhead.length() &&
Character.isWhitespace(xmlhead.charAt(q))) {
+ q++;
+ }
+ if (q < xmlhead.length() && xmlhead.charAt(q) == '>') {
+ return q + 1;
+ }
+ p = xmlhead.indexOf(']', q);
+ }
+ return xmlhead.length();
+ }
+
private static String buildXMLTail(String xmlhead) {
// assume the input text is a portion of a well-formed xml
List<String> tags = new ArrayList<>();
@@ -387,6 +422,10 @@ public class TokenXMLExpressionIterator extends
ExpressionAdapter {
if (nc == '?') {
p++;
continue;
+ } else if (nc == '!') {
+ // a comment, CDATA or DOCTYPE is not a tag to close
+ p = skipMarkupDeclaration(xmlhead, p);
+ continue;
} else if (nc == '/') {
p++;
tags.remove(tags.size() - 1);
diff --git
a/core/camel-support/src/main/java/org/apache/camel/support/builder/ValueBuilder.java
b/core/camel-support/src/main/java/org/apache/camel/support/builder/ValueBuilder.java
index b787bc3b9ed0..87e8983781b1 100644
---
a/core/camel-support/src/main/java/org/apache/camel/support/builder/ValueBuilder.java
+++
b/core/camel-support/src/main/java/org/apache/camel/support/builder/ValueBuilder.java
@@ -195,7 +195,9 @@ public class ValueBuilder implements Expression, Predicate {
// wrap in skip first (if group then it has its own skip first
logic)
newExp = ExpressionBuilder.skipFirstExpression(newExp);
}
- newExp = ExpressionBuilder.groupIteratorExpression(newExp, token,
group, skipFirst);
+ if (group != null) {
+ newExp = ExpressionBuilder.groupIteratorExpression(newExp, token,
group, skipFirst);
+ }
return onNewValueBuilder(newExp);
}
diff --git
a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_23.adoc
b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_23.adoc
index ec73dfa2e638..b805c32b9c50 100644
--- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_23.adoc
+++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_23.adoc
@@ -155,6 +155,19 @@ ratios that are all `0`, or ratios whose sum is greater
than `2147483647` now fa
`IllegalArgumentException`. Previously such a route started, but sending to it
could hang the caller, spin a CPU,
or send every message to the same endpoint.
+=== Tokenize language
+
+- A message without a body (or a `source` that has no value) now has no
tokens, the same as an empty body.
+ Prior to Camel 4.23 the tokenizer failed with a `NullPointerException`.
+- `skipFirst` now also skips the first token in `xml` mode and in pair mode
(`endToken`). Prior to Camel 4.23 the
+ option was ignored in those modes.
+- In pair mode with `group` and `includeTokens`, the grouped pairs are now
joined without a delimiter (as in
+ `xml` mode), unless `groupDelimiter` is set. Prior to Camel 4.23 the start
token was inserted between them, so a
+ group of `<a>1</a>` and `<a>2</a>` became `<a>1</a><a><a>2</a>`. Without
`includeTokens` the start token is still
+ used as the delimiter between the grouped values.
+- A pair with the same start and end token is now refused when the route is
created. Prior to Camel 4.23 it
+ silently had no tokens.
+
=== Context reload now re-applies placeholder based component options
When a context reload is triggered, for example by one of the vault components
detecting that a secret was rotated,
