From 2d22943645c5c6c2e8c6178e1bb0da446eeb4302 Mon Sep 17 00:00:00 2001 From: Eric Yang Date: Mon, 13 Jul 2026 12:59:05 +0900 Subject: [PATCH] [SPARK-57932][SQL] Fix regexp_instr and regexp_replace returning wrong results for supplementary characters --- .../expressions/regexpExpressions.scala | 17 +++++----- .../expressions/RegexpExpressionsSuite.scala | 31 +++++++++++++++++++ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index c6e5c480f3c2c..3b08cbe51e726 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -715,9 +715,9 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio } val source = s.toString() val position = i.asInstanceOf[Int] - 1 - if (position == 0 || position < source.length) { + if (position == 0 || position < source.codePointCount(0, source.length)) { val m = pattern.matcher(source) - m.region(position, source.length) + m.region(source.offsetByCodePoints(0, position), source.length) result.delete(0, result.length()) while (m.find) { try { @@ -770,9 +770,9 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio } String $source = $subject.toString(); int $position = $pos - 1; - if ($position == 0 || $position < $source.length()) { + if ($position == 0 || $position < $source.codePointCount(0, $source.length())) { $classNameStringBuilder $termResult = new $classNameStringBuilder(); - $matcher.region($position, $source.length()); + $matcher.region($source.offsetByCodePoints(0, $position), $source.length()); while ($matcher.find()) { try { @@ -1171,9 +1171,10 @@ case class RegExpInStr(subject: Expression, regexp: Expression, idx: Expression) override def nullSafeEval(s: Any, r: Any, i: Any): Any = { try { - val m = getLastMatcher(s, r) + val source = s.toString + val m = getLastMatcher(source, r) if (m.find) { - m.toMatchResult.start() + 1 + source.codePointCount(0, m.toMatchResult.start()) + 1 } else { 0 } @@ -1187,6 +1188,7 @@ case class RegExpInStr(subject: Expression, regexp: Expression, idx: Expression) override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { val matcher = ctx.freshName("matcher") + val source = ctx.freshName("source") val setEvNotNull = if (nullable) { s"${ev.isNull} = false;" } else { @@ -1200,7 +1202,8 @@ case class RegExpInStr(subject: Expression, regexp: Expression, idx: Expression) | ${RegExpUtils.initLastMatcherCode(ctx, subject, regexp, matcher, prettyName, collationId)} | if ($matcher.find()) { - | ${ev.value} = $matcher.toMatchResult().start() + 1; + | String $source = $subject.toString(); + | ${ev.value} = $source.codePointCount(0, $matcher.toMatchResult().start()) + 1; | } else { | ${ev.value} = 0; | } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala index 9b5a8f2fce066..270480a63c0fa 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala @@ -366,6 +366,26 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { RegExpReplace(Literal("\"quote"), Literal("\"quote"), Literal("\"quote")) :: Nil) } + test("SPARK-57932: regexp_replace position is a code-point position for supplementary chars") { + // scalastyle:off nonascii + checkEvaluation( + RegExpReplace(Literal("😀aXa"), Literal("a"), Literal("Z"), Literal(3)), + "😀aXZ") + checkEvaluation( + RegExpReplace(Literal("😀😀"), Literal("😀"), + Literal("Z"), Literal(3)), + "😀😀") + // Position beyond the string length: nothing is replaced. + checkEvaluation( + RegExpReplace(Literal("😀a"), Literal("a"), Literal("Z"), Literal(4)), + "😀a") + // Position at the last code-point. + checkEvaluation( + RegExpReplace(Literal("a😀b"), Literal("b"), Literal("Z"), Literal(3)), + "a😀Z") + // scalastyle:on nonascii + } + test("SPARK-22570: RegExpReplace should not create a lot of global variables") { val ctx = new CodegenContext RegExpReplace(Literal("100"), Literal("(\\d+)"), Literal("num")).genCode(ctx) @@ -627,6 +647,17 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { new RegExpInStr(Literal("\"quote"), Literal("\"quote")) :: Nil) } + test("SPARK-57932: regexp_instr returns a code-point position for supplementary characters") { + // scalastyle:off nonascii + checkEvaluation(RegExpInStr(Literal("😀ab"), Literal("ab"), Literal(0)), 2) + checkEvaluation(RegExpInStr(Literal("a😀b"), Literal("b"), Literal(0)), 3) + checkEvaluation( + RegExpInStr(Literal("😀😁xy"), Literal("xy"), Literal(0)), 3) + // A match made up entirely of supplementary characters. + checkEvaluation(RegExpInStr(Literal("😀😁😂"), Literal("😁"), Literal(0)), 2) + // scalastyle:on nonascii + } + test("SPARK-39758: invalid regexp pattern") { val s = $"s".string.at(0) val p = $"p".string.at(1)