(paimon) branch master updated: [core] Skip surrogate code points in truncated string max (#10335)

[lzljs3620320]

This is an automated email from the ASF dual-hosted git repository.

JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new f498ab13fb [core] Skip surrogate code points in truncated string max 
(#10335)
f498ab13fb is described below

commit f498ab13fba2e46789784af9be5e7f7c57880b7a
Author: spancer <spancer.roc....@gmail.com>
AuthorDate: Sat Oct 3 17:03:41 2026 +0800

    [core] Skip surrogate code points in truncated string max (#10335)
---
 .../statistics/TruncateSimpleColStatsCollector.java   |  4 ++++
 .../statistics/SimpleColStatsCollectorTest.java       | 19 +++++++++++++++++++
 2 files changed, 23 insertions(+)

diff --git 
a/paimon-common/src/main/java/org/apache/paimon/statistics/TruncateSimpleColStatsCollector.java
 
b/paimon-common/src/main/java/org/apache/paimon/statistics/TruncateSimpleColStatsCollector.java
index fc8d9d8cc2..b135e0d9bb 100644
--- 
a/paimon-common/src/main/java/org/apache/paimon/statistics/TruncateSimpleColStatsCollector.java
+++ 
b/paimon-common/src/main/java/org/apache/paimon/statistics/TruncateSimpleColStatsCollector.java
@@ -157,6 +157,10 @@ public class TruncateSimpleColStatsCollector extends 
AbstractSimpleColStatsColle
                 // characters = i
                 int offsetByCodePoint = 
truncatedStringBuilder.offsetByCodePoints(0, i);
                 int nextCodePoint = 
truncatedStringBuilder.codePointAt(offsetByCodePoint) + 1;
+                if (nextCodePoint >= Character.MIN_SURROGATE
+                        && nextCodePoint <= Character.MAX_SURROGATE) {
+                    nextCodePoint = Character.MAX_SURROGATE + 1;
+                }
                 // No overflow
                 if (nextCodePoint != 0 && 
Character.isValidCodePoint(nextCodePoint)) {
                     truncatedStringBuilder.setLength(offsetByCodePoint);
diff --git 
a/paimon-common/src/test/java/org/apache/paimon/statistics/SimpleColStatsCollectorTest.java
 
b/paimon-common/src/test/java/org/apache/paimon/statistics/SimpleColStatsCollectorTest.java
index a1be45b0ba..543bf09a1a 100644
--- 
a/paimon-common/src/test/java/org/apache/paimon/statistics/SimpleColStatsCollectorTest.java
+++ 
b/paimon-common/src/test/java/org/apache/paimon/statistics/SimpleColStatsCollectorTest.java
@@ -174,6 +174,25 @@ public class SimpleColStatsCollectorTest {
         
assertThat(fieldStats.max()).isEqualTo(BinaryString.fromString("\uD83E\uDD19"));
     }
 
+    @Test
+    public void testTruncateMaxSkipsSurrogateCodePoints() {
+        String prefix = "0123456789abcde";
+        BinaryString value = BinaryString.fromString(prefix + "\uD7FFzz");
+        BinaryString laterValue = BinaryString.fromString(prefix + "A");
+        BinaryString expectedMax = BinaryString.fromString(prefix + "\uE000");
+
+        TruncateSimpleColStatsCollector collector = new 
TruncateSimpleColStatsCollector(16);
+        collector.collect(value, (Serializer) BinaryStringSerializer.INSTANCE);
+        collector.collect(laterValue, (Serializer) 
BinaryStringSerializer.INSTANCE);
+        SimpleColStats stats = collector.result();
+        assertThat(stats.max()).isEqualTo(expectedMax);
+        assertThat(((BinaryString) 
stats.max()).compareTo(value)).isGreaterThan(0);
+
+        SimpleColStats converted = collector.convert(new 
SimpleColStats(laterValue, value, 0L));
+        assertThat(converted.max()).isEqualTo(expectedMax);
+        assertThat(((BinaryString) 
converted.max()).compareTo(value)).isGreaterThan(0);
+    }
+
     @Test
     public void testTruncateCopied() {
         TruncateSimpleColStatsCollector collector = new 
TruncateSimpleColStatsCollector(16);