This is an automated email from the ASF dual-hosted git repository.
sarutak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8fabde781291 [SPARK-57932][SQL] Fix regexp_instr and regexp_replace
returning wrong results for supplementary characters
8fabde781291 is described below
commit 8fabde78129109e82212c867e9d19548aef856e9
Author: Eric Yang <[email protected]>
AuthorDate: Fri Jul 10 09:05:09 2026 +0900
[SPARK-57932][SQL] Fix regexp_instr and regexp_replace returning wrong
results for supplementary characters
### What changes were proposed in this pull request?
`regexp_instr` returned matcher.start() + 1 (a UTF-16 code-unit index), and
`regexp_replace`'s position argument was used as a UTF-16 offset in
matcher.region(pos-1, ...) and its out-of-range guard. Both now use code-point
positions via codePointCount/offsetByCodePoints.
### Why are the changes needed?
Spark string positions are 1-based code points (instr, locate, position,
substring, length). For strings containing supplementary characters (code
points > U+FFFF, e.g. emojis), these two functions gave wrong results:
- regexp_instr('😀ab', 'ab') returned 3 instead of 2.
- regexp_replace('😀aXa', 'a', 'Z', 3) returned 😀ZXZ instead of 😀aXZ
(replaced a match before position); regexp_replace('😀😀', '😀', 'Z', 3) returned
😀Z instead of the input unchanged. This is the sibling of the LIKE fix in
SPARK-55453.
### Does this PR introduce _any_ user-facing change?
Yes. regexp_instr and regexp_replace now return code-point-consistent
results for strings with supplementary characters.
### How was this patch tested?
Added UT.
### Was this patch authored or co-authored using generative AI tooling?
Yes.
Closes #56998 from jiwen624/SPARK-regexp-instr-codepoint.
Authored-by: Eric Yang <[email protected]>
Signed-off-by: Kousuke Saruta <[email protected]>
---
.../catalyst/expressions/regexpExpressions.scala | 13 +++++----
.../expressions/RegexpExpressionsSuite.scala | 31 ++++++++++++++++++++++
2 files changed, 39 insertions(+), 5 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
index 3bd0352f6f07..e99c6b2b29e7 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
@@ -1124,9 +1124,10 @@ case class RegExpInStr(subject: Expression, regexp:
Expression, idx: Expression)
override def nullSafeEval(s: Any, r: Any, i: Any): Any = {
try {
- val m = getLastMatcher(s, r)
+ val source = s.toString
+ val m = getLastMatcher(source, r)
if (m.find) {
- m.toMatchResult.start() + 1
+ source.codePointCount(0, m.toMatchResult.start()) + 1
} else {
0
}
@@ -1140,6 +1141,7 @@ case class RegExpInStr(subject: Expression, regexp:
Expression, idx: Expression)
override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode = {
val matcher = ctx.freshName("matcher")
+ val source = ctx.freshName("source")
val setEvNotNull = if (nullable) {
s"${ev.isNull} = false;"
} else {
@@ -1153,7 +1155,8 @@ case class RegExpInStr(subject: Expression, regexp:
Expression, idx: Expression)
| ${RegExpUtils.initLastMatcherCode(ctx, subject, regexp, matcher,
prettyName,
collationId)}
| if ($matcher.find()) {
- | ${ev.value} = $matcher.toMatchResult().start() + 1;
+ | String $source = $subject.toString();
+ | ${ev.value} = $source.codePointCount(0,
$matcher.toMatchResult().start()) + 1;
| } else {
| ${ev.value} = 0;
| }
@@ -1232,9 +1235,9 @@ object RegExpUtils {
replacement: String,
pos: Int): UTF8String = {
val position = pos - 1
- if (position == 0 || position < source.length) {
+ if (position == 0 || position < source.codePointCount(0, source.length)) {
val matcher = pattern.matcher(source)
- matcher.region(position, source.length)
+ matcher.region(source.offsetByCodePoints(0, position), source.length)
val result = new JStringBuilder
while (matcher.find()) {
try {
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
index 19c8c8f341f0..0bf29553ea33 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
@@ -377,6 +377,26 @@ class RegexpExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
RegExpReplace(Literal("\"quote"), Literal("\"quote"),
Literal("\"quote")) :: Nil)
}
+ test("SPARK-57932: regexp_replace position is a code-point position for
supplementary chars") {
+ // scalastyle:off nonascii
+ checkEvaluation(
+ RegExpReplace(Literal("😀aXa"), Literal("a"), Literal("Z"), Literal(3)),
+ "😀aXZ")
+ checkEvaluation(
+ RegExpReplace(Literal("😀😀"), Literal("😀"),
+ Literal("Z"), Literal(3)),
+ "😀😀")
+ // Position beyond the string length: nothing is replaced.
+ checkEvaluation(
+ RegExpReplace(Literal("😀a"), Literal("a"), Literal("Z"), Literal(4)),
+ "😀a")
+ // Position at the last code-point.
+ checkEvaluation(
+ RegExpReplace(Literal("a😀b"), Literal("b"), Literal("Z"), Literal(3)),
+ "a😀Z")
+ // scalastyle:on nonascii
+ }
+
test("SPARK-22570: RegExpReplace should not create a lot of global
variables") {
val ctx = new CodegenContext
RegExpReplace(Literal("100"), Literal("(\\d+)"),
Literal("num")).genCode(ctx)
@@ -638,6 +658,17 @@ class RegexpExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
new RegExpInStr(Literal("\"quote"), Literal("\"quote")) :: Nil)
}
+ test("SPARK-57932: regexp_instr returns a code-point position for
supplementary characters") {
+ // scalastyle:off nonascii
+ checkEvaluation(RegExpInStr(Literal("😀ab"), Literal("ab"), Literal(0)), 2)
+ checkEvaluation(RegExpInStr(Literal("a😀b"), Literal("b"), Literal(0)), 3)
+ checkEvaluation(
+ RegExpInStr(Literal("😀😁xy"), Literal("xy"), Literal(0)), 3)
+ // A match made up entirely of supplementary characters.
+ checkEvaluation(RegExpInStr(Literal("😀😁😂"), Literal("😁"), Literal(0)), 2)
+ // scalastyle:on nonascii
+ }
+
test("SPARK-39758: invalid regexp pattern") {
val s = $"s".string.at(0)
val p = $"p".string.at(1)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
