This is an automated email from the ASF dual-hosted git repository.
apupier pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 863c1c173949 CAMEL-25152: camel-bindy - fixed-length marshal pads and
clips fields by code points, like unmarshal
863c1c173949 is described below
commit 863c1c173949a550068e81a11c157f7d5a23b696
Author: smjain <[email protected]>
AuthorDate: Wed Sep 30 07:51:13 2026 +0530
CAMEL-25152: camel-bindy - fixed-length marshal pads and clips fields by
code points, like unmarshal
Since CAMEL-14521 a fixed-length record is read by counting code points (or
graphemes with countGrapheme = true), but marshal still padded and clipped
each field by UTF-16 chars, and the record length check counted UTF-16
chars. A field with a character outside the BMP (or, with countGrapheme,
with a combining character) was written one padding character short per
extra char, the following fields were read shifted, clip could cut a
surrogate pair, and a correctly padded record from another system failed
the record length check.
Marshal now measures and clips the formatted value with UnicodeHelper and
the same method as unmarshal, and the record length check (and
ignoreTrailingChars) uses the same count. Text in the BMP without combining
characters is written and read as before. Upgrade guide note added.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../dataformat/bindy/BindyFixedLengthFactory.java | 26 +++-
.../bindy/fixed/BindyFixedLengthDataFormat.java | 25 ++--
.../BindyFixedLengthMarshalUnicodeTest.java | 147 +++++++++++++++++++++
.../ROOT/pages/camel-4x-upgrade-guide-4_23.adoc | 14 ++
4 files changed, 195 insertions(+), 17 deletions(-)
diff --git
a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyFixedLengthFactory.java
b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyFixedLengthFactory.java
index ac772c6c34df..0ef25f69876e 100644
---
a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyFixedLengthFactory.java
+++
b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/BindyFixedLengthFactory.java
@@ -485,8 +485,13 @@ public class BindyFixedLengthFactory extends
BindyAbstractFactory implements Bin
StringBuilder temp = new StringBuilder();
+ // count the length the same way as unmarshal does
(code points, or graphemes with
+ // countGrapheme), not in UTF-16 chars
+ UnicodeHelper unicodeResult = new
UnicodeHelper(result, unicodeMethod());
+ int resultLength = unicodeResult.length();
+
// Check if we must pad
- if (result.length() < fieldLength) {
+ if (resultLength < fieldLength) {
// No padding defined for the field
if (padCharField == 0) {
@@ -497,13 +502,13 @@ public class BindyFixedLengthFactory extends
BindyAbstractFactory implements Bin
}
if (align.contains("R")) {
- temp.append(generatePaddingChars(padChar,
fieldLength, result.length()));
+ temp.append(generatePaddingChars(padChar,
fieldLength, resultLength));
temp.append(result);
} else if (align.contains("L")) {
temp.append(result);
- temp.append(generatePaddingChars(padChar,
fieldLength, result.length()));
+ temp.append(generatePaddingChars(padChar,
fieldLength, resultLength));
} else if (align.contains("B")) {
- temp.append(generatePaddingChars(padChar,
fieldLength, result.length()));
+ temp.append(generatePaddingChars(padChar,
fieldLength, resultLength));
temp.append(result);
} else {
throw new IllegalArgumentException(
@@ -512,16 +517,16 @@ public class BindyFixedLengthFactory extends
BindyAbstractFactory implements Bin
}
result = temp.toString();
- } else if (result.length() > fieldLength) {
+ } else if (resultLength > fieldLength) {
// we are bigger than allowed
// is clipped enabled? if so clip the field
if (datafield.clip()) {
- result = result.substring(0, fieldLength);
+ result = unicodeResult.substring(0,
fieldLength);
} else {
throw new IllegalArgumentException(
"Length for the " + field.getName()
- + " must
not be larger than allowed, was: " + result.length()
+ + " must
not be larger than allowed, was: " + resultLength
+ ",
allowed: " + fieldLength);
}
}
@@ -554,6 +559,13 @@ public class BindyFixedLengthFactory extends
BindyAbstractFactory implements Bin
}
+ /**
+ * How the length of a field is counted, on unmarshal and on marshal.
+ */
+ public UnicodeHelper.Method unicodeMethod() {
+ return countGrapheme ? UnicodeHelper.Method.GRAPHEME :
UnicodeHelper.Method.CODEPOINTS;
+ }
+
private String generatePaddingChars(char pad, int lengthField, int
lengthString) {
int size = lengthField - lengthString;
diff --git
a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/fixed/BindyFixedLengthDataFormat.java
b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/fixed/BindyFixedLengthDataFormat.java
index 51404e36edcd..978c8ccabdc0 100644
---
a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/fixed/BindyFixedLengthDataFormat.java
+++
b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/fixed/BindyFixedLengthDataFormat.java
@@ -33,6 +33,7 @@ import
org.apache.camel.dataformat.bindy.BindyAbstractDataFormat;
import org.apache.camel.dataformat.bindy.BindyAbstractFactory;
import org.apache.camel.dataformat.bindy.BindyFixedLengthFactory;
import org.apache.camel.dataformat.bindy.FormatFactory;
+import org.apache.camel.dataformat.bindy.UnicodeHelper;
import org.apache.camel.dataformat.bindy.util.ConverterUtils;
import org.apache.camel.spi.annotations.Dataformat;
import org.apache.camel.support.ExchangeHelper;
@@ -297,16 +298,20 @@ public class BindyFixedLengthDataFormat extends
BindyAbstractDataFormat {
// Check if the record length corresponds to the parameter
// provided in the @FixedLengthRecord
if (factory.recordLength() > 0) {
- if (isPaddingNeededAndEnable(factory, myLine)) {
+ // count the length the same way as the fields are read (code
points, or graphemes with countGrapheme)
+ UnicodeHelper unicodeLine = new UnicodeHelper(myLine,
factory.unicodeMethod());
+ int lineLength = unicodeLine.length();
+ if (isPaddingNeededAndEnable(factory, lineLength)) {
//myLine = rightPad(myLine, factory.recordLength());
}
- if (isTrimmingNeededAndEnabled(factory, myLine)) {
- myLine = myLine.substring(0, factory.recordLength());
+ if (isTrimmingNeededAndEnabled(factory, lineLength)) {
+ myLine = unicodeLine.substring(0, factory.recordLength());
+ lineLength = factory.recordLength();
}
- if (myLine.length() < factory.recordLength()
- && !factory.isIgnoreMissingChars() || myLine.length() >
factory.recordLength()) {
+ if (lineLength < factory.recordLength()
+ && !factory.isIgnoreMissingChars() || lineLength >
factory.recordLength()) {
throw new java.lang.IllegalArgumentException(
- "Size of the record: " + myLine.length()
+ "Size of the record: " + lineLength
+ " is not equal
to the value provided in the model: "
+
factory.recordLength());
}
@@ -325,8 +330,8 @@ public class BindyFixedLengthDataFormat extends
BindyAbstractDataFormat {
return model;
}
- private boolean isTrimmingNeededAndEnabled(BindyFixedLengthFactory
factory, String myLine) {
- return factory.isIgnoreTrailingChars() && myLine.length() >
factory.recordLength();
+ private boolean isTrimmingNeededAndEnabled(BindyFixedLengthFactory
factory, int lineLength) {
+ return factory.isIgnoreTrailingChars() && lineLength >
factory.recordLength();
}
@SuppressWarnings("unused")
@@ -334,8 +339,8 @@ public class BindyFixedLengthDataFormat extends
BindyAbstractDataFormat {
return String.format("%1$-" + length + "s", myLine);
}
- private boolean isPaddingNeededAndEnable(BindyFixedLengthFactory factory,
String myLine) {
- return myLine.length() < factory.recordLength() &&
factory.isIgnoreMissingChars();
+ private boolean isPaddingNeededAndEnable(BindyFixedLengthFactory factory,
int lineLength) {
+ return lineLength < factory.recordLength() &&
factory.isIgnoreMissingChars();
}
@Override
diff --git
a/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/fixed/multibytes/BindyFixedLengthMarshalUnicodeTest.java
b/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/fixed/multibytes/BindyFixedLengthMarshalUnicodeTest.java
new file mode 100644
index 000000000000..d2ed1404a2aa
--- /dev/null
+++
b/components/camel-bindy/src/test/java/org/apache/camel/dataformat/bindy/fixed/multibytes/BindyFixedLengthMarshalUnicodeTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.dataformat.bindy.fixed.multibytes;
+
+import org.apache.camel.RoutesBuilder;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.dataformat.bindy.annotation.DataField;
+import org.apache.camel.dataformat.bindy.annotation.FixedLengthRecord;
+import org.apache.camel.dataformat.bindy.fixed.BindyFixedLengthDataFormat;
+import org.apache.camel.test.junit6.CamelTestSupport;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Unmarshal counts the field lengths in code points (or graphemes with
countGrapheme=true). Marshal must pad and clip
+ * with the same count, so that a record with characters outside the BMP (or
combining characters) is read back.
+ */
+public class BindyFixedLengthMarshalUnicodeTest extends CamelTestSupport {
+
+ // U+1F600 (one code point, two UTF-16 chars)
+ private static final String EMOJI = "\uD83D\uDE00";
+ // U+20000 CJK Extension B (one code point, two UTF-16 chars)
+ private static final String CJK_EXT_B = "\uD840\uDC00";
+ // e + combining acute accent (one grapheme, two code points)
+ private static final String E_ACUTE = "e\u0301";
+
+ @Test
+ public void testMarshalPadsByCodePoints() {
+ String text = template.requestBody("direct:marshal", record(new
CodePointRecord(), "ok" + EMOJI, "x", "y"),
+ String.class);
+ assertEquals(" ok" + EMOJI + " x y\r\n", text);
+ }
+
+ @Test
+ public void testMarshalUnmarshalCodePoints() {
+ for (String value : new String[] { "ok" + EMOJI, CJK_EXT_B +
CJK_EXT_B, "caf\u00E9" }) {
+ CodePointRecord in = record(new CodePointRecord(), value, "x",
"y");
+ String text = template.requestBody("direct:marshal", in,
String.class);
+ CodePointRecord out = template.requestBody("direct:unmarshal",
text, CodePointRecord.class);
+ assertEquals(value, out.first, text);
+ assertEquals("x", out.second, text);
+ assertEquals("y", out.third, text);
+ }
+ }
+
+ @Test
+ public void testMarshalClipsByCodePoints() {
+ ClipRecord in = new ClipRecord();
+ in.first = "abcd" + EMOJI + "f";
+ in.second = "x";
+ String text = template.requestBody("direct:marshalClip", in,
String.class);
+ assertEquals("abcd" + EMOJI + "x\r\n", text);
+ }
+
+ @Test
+ public void testMarshalUnmarshalGraphemes() {
+ GraphemeRecord in = record(new GraphemeRecord(), "caf" + E_ACUTE, "x",
"y");
+ String text = template.requestBody("direct:marshalGrapheme", in,
String.class);
+ GraphemeRecord out = template.requestBody("direct:unmarshalGrapheme",
text, GraphemeRecord.class);
+ assertEquals("caf" + E_ACUTE, out.first, text);
+ assertEquals("x", out.second, text);
+ assertEquals("y", out.third, text);
+ }
+
+ private static CodePointRecord record(CodePointRecord record, String
first, String second, String third) {
+ record.first = first;
+ record.second = second;
+ record.third = third;
+ return record;
+ }
+
+ private static GraphemeRecord record(GraphemeRecord record, String first,
String second, String third) {
+ record.first = first;
+ record.second = second;
+ record.third = third;
+ return record;
+ }
+
+ @Override
+ protected RoutesBuilder createRouteBuilder() {
+ return new RouteBuilder() {
+ @Override
+ public void configure() {
+ BindyFixedLengthDataFormat codePoints = new
BindyFixedLengthDataFormat(CodePointRecord.class);
+ from("direct:marshal").marshal(codePoints);
+ from("direct:unmarshal").unmarshal(codePoints);
+
+ from("direct:marshalClip").marshal(new
BindyFixedLengthDataFormat(ClipRecord.class));
+
+ BindyFixedLengthDataFormat graphemes = new
BindyFixedLengthDataFormat(GraphemeRecord.class);
+ from("direct:marshalGrapheme").marshal(graphemes);
+ from("direct:unmarshalGrapheme").unmarshal(graphemes);
+ }
+ };
+ }
+
+ @FixedLengthRecord(length = 12)
+ public static class CodePointRecord {
+
+ @DataField(pos = 1, length = 5, trim = true)
+ String first;
+
+ @DataField(pos = 6, length = 4, trim = true)
+ String second;
+
+ @DataField(pos = 10, length = 3, trim = true)
+ String third;
+ }
+
+ @FixedLengthRecord(countGrapheme = true)
+ public static class GraphemeRecord {
+
+ @DataField(pos = 1, length = 5, trim = true)
+ String first;
+
+ @DataField(pos = 6, length = 4, trim = true)
+ String second;
+
+ @DataField(pos = 10, length = 3, trim = true)
+ String third;
+ }
+
+ @FixedLengthRecord
+ public static class ClipRecord {
+
+ @DataField(pos = 1, length = 5, clip = true)
+ String first;
+
+ @DataField(pos = 6, length = 1)
+ String second;
+ }
+}
diff --git
a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_23.adoc
b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_23.adoc
index ea85daddbf6b..0e6cac6bcd92 100644
--- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_23.adoc
+++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_23.adoc
@@ -603,6 +603,20 @@ data format also writes the message with the charset of
the exchange when it mar
US-ASCII messages are parsed as before. A route that repaired the wrongly
decoded text itself must stop
doing so.
+=== camel-bindy - fixed-length records with characters outside the BMP
+
+Since Camel 3.1 a fixed-length record is read by counting code points (or
graphemes with
+`@FixedLengthRecord(countGrapheme = true)`), but it was written by counting
UTF-16 chars, so a field with a
+character outside the Basic Multilingual Plane (such as an emoji) was written
one padding character short and the
+following fields were read shifted. Marshal now pads and clips each field with
the same count as unmarshal, and the
+record `length` is checked with that count too. A field with such characters
(or, with `countGrapheme = true`, with
+combining characters) is padded to its length in code points (or graphemes),
so it gets one padding character more
+than before for each extra UTF-16 char (or code point). Text in the Basic
Multilingual Plane without combining
+characters is written and read as before. A record with such characters
written by an older Camel version is too
+short by that count, so it now fails the record `length` check (unless
`ignoreMissingChars = true`) instead of being
+read with shifted fields. A system that reads these records by counting UTF-16
chars must count code points (or
+graphemes) instead.
+
=== Components and Language removal
==== camel-csimple, camel-csimple-joor and csimple-maven-plugin
