This is an automated email from the ASF dual-hosted git repository.
garydgregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-csv.git
The following commit(s) were added to refs/heads/master by this push:
new 3634534d Add
CSVParserTest.testGetBytePositionMultiCharacterDelimiterWithSupplementaryChar()
from PR #605.
3634534d is described below
commit 3634534d69f557b099a06e56a8ad6ae96c5384ec
Author: Gary Gregory <[email protected]>
AuthorDate: Fri Jul 17 06:55:36 2026 -0700
Add
CSVParserTest.testGetBytePositionMultiCharacterDelimiterWithSupplementaryChar()
from PR #605.
---
.../java/org/apache/commons/csv/CSVParserTest.java | 67 ++++++++++++++--------
1 file changed, 44 insertions(+), 23 deletions(-)
diff --git a/src/test/java/org/apache/commons/csv/CSVParserTest.java
b/src/test/java/org/apache/commons/csv/CSVParserTest.java
index c18ea48c..8d88a5ff 100644
--- a/src/test/java/org/apache/commons/csv/CSVParserTest.java
+++ b/src/test/java/org/apache/commons/csv/CSVParserTest.java
@@ -691,6 +691,27 @@ class CSVParserTest {
}
}
+ @Test
+ void testGetBytePositionMultiCharacterDelimiterWithSupplementaryChar()
throws IOException {
+ // Delimiter holds a 4-byte (surrogate pair) character; the delimiter
tail is consumed through
+ // the char[] read path, where the surrogate halves must be paired
with the correct neighbor.
+ final String code = "aa[😀]bb\ncc[😀]dd\n";
+ final CSVFormat format =
CSVFormat.DEFAULT.builder().setDelimiter("[😀]").get();
+ try (CSVParser parser = CSVParser.builder()
+ .setReader(new StringReader(code))
+ .setFormat(format)
+ .setCharset(StandardCharsets.UTF_8)
+ .setTrackBytes(true)
+ .get()) {
+ final Iterator<CSVRecord> it = parser.iterator();
+ final CSVRecord first = it.next();
+ final CSVRecord second = it.next();
+ assertEquals(0, first.getBytePosition());
+ // "aa[😀]bb\n" -> 2 + 1 + 4 + 1 + 2 + 1 = 11 bytes in UTF-8
+ assertEquals(11, second.getBytePosition());
+ }
+ }
+
/**
* Tests <a
href="https://issues.apache.org/jira/browse/CSV-329";>CSV-329</a>.
*/
@@ -716,6 +737,29 @@ class CSVParserTest {
}
}
+ @Test
+ void testGetBytePositionWithBomEmittingCharset() throws IOException {
+ // The UTF-16 encoder writes a byte-order mark on every encode call,
so the per-character
+ // byte length must exclude it. Otherwise each code unit is counted as
4 bytes (2 for the
+ // BOM, 2 for the char) and getBytePosition() grows at twice the true
rate.
+ final String code = "a,b\nc,d\n";
+ try (CSVParser parser = CSVParser.builder()
+ .setReader(new StringReader(code))
+ .setFormat(CSVFormat.DEFAULT)
+ .setCharset(StandardCharsets.UTF_16)
+ .setTrackBytes(true)
+ .get()) {
+ final CSVRecord first = parser.nextRecord();
+ final CSVRecord second = parser.nextRecord();
+ assertNotNull(first);
+ assertNotNull(second);
+ assertNull(parser.nextRecord());
+ assertEquals(0, first.getBytePosition());
+ // "a,b\n" is 4 UTF-16 code units, 2 bytes each.
+ assertEquals(8, second.getBytePosition());
+ }
+ }
+
@Test
void testGetBytePositionWithCharacterOffsetAndMultiBytePrefix() throws
Exception {
final String row0 = "é,x\n";
@@ -767,29 +811,6 @@ class CSVParserTest {
}
}
- @Test
- void testGetBytePositionWithBomEmittingCharset() throws IOException {
- // The UTF-16 encoder writes a byte-order mark on every encode call,
so the per-character
- // byte length must exclude it. Otherwise each code unit is counted as
4 bytes (2 for the
- // BOM, 2 for the char) and getBytePosition() grows at twice the true
rate.
- final String code = "a,b\nc,d\n";
- try (CSVParser parser = CSVParser.builder()
- .setReader(new StringReader(code))
- .setFormat(CSVFormat.DEFAULT)
- .setCharset(StandardCharsets.UTF_16)
- .setTrackBytes(true)
- .get()) {
- final CSVRecord first = parser.nextRecord();
- final CSVRecord second = parser.nextRecord();
- assertNotNull(first);
- assertNotNull(second);
- assertNull(parser.nextRecord());
- assertEquals(0, first.getBytePosition());
- // "a,b\n" is 4 UTF-16 code units, 2 bytes each.
- assertEquals(8, second.getBytePosition());
- }
- }
-
@Test
void testGetHeaderComment_HeaderComment1() throws IOException {
try (CSVParser parser = CSVParser.parse(CSV_INPUT_HEADER_COMMENT,
FORMAT_AUTO_HEADER)) {
