From 1d64728d47fd4f6b863e0c343aed9cdb4f19d5c1 Mon Sep 17 00:00:00 2001 From: Eric Yang Date: Fri, 10 Jul 2026 09:05:09 +0900 Subject: [PATCH] [SPARK-57932][SQL] Fix regexp_instr and regexp_replace returning wrong results for supplementary characters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What changes were proposed in this pull request? `regexp_instr` returned matcher.start() + 1 (a UTF-16 code-unit index), and `regexp_replace`'s position argument was used as a UTF-16 offset in matcher.region(pos-1, ...) and its out-of-range guard. Both now use code-point positions via codePointCount/offsetByCodePoints. ### Why are the changes needed? Spark string positions are 1-based code points (instr, locate, position, substring, length). For strings containing supplementary characters (code points > U+FFFF, e.g. emojis), these two functions gave wrong results: - regexp_instr('😀ab', 'ab') returned 3 instead of 2. - regexp_replace('😀aXa', 'a', 'Z', 3) returned 😀ZXZ instead of 😀aXZ (replaced a match before position); regexp_replace('😀😀', '😀', 'Z', 3) returned 😀Z instead of the input unchanged. This is the sibling of the LIKE fix in SPARK-55453. ### Does this PR introduce _any_ user-facing change? Yes. regexp_instr and regexp_replace now return code-point-consistent results for strings with supplementary characters. ### How was this patch tested? Added UT. ### Was this patch authored or co-authored using generative AI tooling? Yes. Closes #56998 from jiwen624/SPARK-regexp-instr-codepoint. Authored-by: Eric Yang Signed-off-by: Kousuke Saruta --- .../expressions/regexpExpressions.scala | 13 +++++--- .../expressions/RegexpExpressionsSuite.scala | 31 +++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index 3bd0352f6f07c..e99c6b2b29e79 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -1124,9 +1124,10 @@ case class RegExpInStr(subject: Expression, regexp: Expression, idx: Expression) override def nullSafeEval(s: Any, r: Any, i: Any): Any = { try { - val m = getLastMatcher(s, r) + val source = s.toString + val m = getLastMatcher(source, r) if (m.find) { - m.toMatchResult.start() + 1 + source.codePointCount(0, m.toMatchResult.start()) + 1 } else { 0 } @@ -1140,6 +1141,7 @@ case class RegExpInStr(subject: Expression, regexp: Expression, idx: Expression) override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { val matcher = ctx.freshName("matcher") + val source = ctx.freshName("source") val setEvNotNull = if (nullable) { s"${ev.isNull} = false;" } else { @@ -1153,7 +1155,8 @@ case class RegExpInStr(subject: Expression, regexp: Expression, idx: Expression) | ${RegExpUtils.initLastMatcherCode(ctx, subject, regexp, matcher, prettyName, collationId)} | if ($matcher.find()) { - | ${ev.value} = $matcher.toMatchResult().start() + 1; + | String $source = $subject.toString(); + | ${ev.value} = $source.codePointCount(0, $matcher.toMatchResult().start()) + 1; | } else { | ${ev.value} = 0; | } @@ -1232,9 +1235,9 @@ object RegExpUtils { replacement: String, pos: Int): UTF8String = { val position = pos - 1 - if (position == 0 || position < source.length) { + if (position == 0 || position < source.codePointCount(0, source.length)) { val matcher = pattern.matcher(source) - matcher.region(position, source.length) + matcher.region(source.offsetByCodePoints(0, position), source.length) val result = new JStringBuilder while (matcher.find()) { try { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala index 19c8c8f341f0a..0bf29553ea33d 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala @@ -377,6 +377,26 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { RegExpReplace(Literal("\"quote"), Literal("\"quote"), Literal("\"quote")) :: Nil) } + test("SPARK-57932: regexp_replace position is a code-point position for supplementary chars") { + // scalastyle:off nonascii + checkEvaluation( + RegExpReplace(Literal("😀aXa"), Literal("a"), Literal("Z"), Literal(3)), + "😀aXZ") + checkEvaluation( + RegExpReplace(Literal("😀😀"), Literal("😀"), + Literal("Z"), Literal(3)), + "😀😀") + // Position beyond the string length: nothing is replaced. + checkEvaluation( + RegExpReplace(Literal("😀a"), Literal("a"), Literal("Z"), Literal(4)), + "😀a") + // Position at the last code-point. + checkEvaluation( + RegExpReplace(Literal("a😀b"), Literal("b"), Literal("Z"), Literal(3)), + "a😀Z") + // scalastyle:on nonascii + } + test("SPARK-22570: RegExpReplace should not create a lot of global variables") { val ctx = new CodegenContext RegExpReplace(Literal("100"), Literal("(\\d+)"), Literal("num")).genCode(ctx) @@ -638,6 +658,17 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { new RegExpInStr(Literal("\"quote"), Literal("\"quote")) :: Nil) } + test("SPARK-57932: regexp_instr returns a code-point position for supplementary characters") { + // scalastyle:off nonascii + checkEvaluation(RegExpInStr(Literal("😀ab"), Literal("ab"), Literal(0)), 2) + checkEvaluation(RegExpInStr(Literal("a😀b"), Literal("b"), Literal(0)), 3) + checkEvaluation( + RegExpInStr(Literal("😀😁xy"), Literal("xy"), Literal(0)), 3) + // A match made up entirely of supplementary characters. + checkEvaluation(RegExpInStr(Literal("😀😁😂"), Literal("😁"), Literal(0)), 2) + // scalastyle:on nonascii + } + test("SPARK-39758: invalid regexp pattern") { val s = $"s".string.at(0) val p = $"p".string.at(1)