diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index 3bd0352f6f07c..e99c6b2b29e79 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -1124,9 +1124,10 @@ case class RegExpInStr(subject: Expression, regexp: Expression, idx: Expression) override def nullSafeEval(s: Any, r: Any, i: Any): Any = { try { - val m = getLastMatcher(s, r) + val source = s.toString + val m = getLastMatcher(source, r) if (m.find) { - m.toMatchResult.start() + 1 + source.codePointCount(0, m.toMatchResult.start()) + 1 } else { 0 } @@ -1140,6 +1141,7 @@ case class RegExpInStr(subject: Expression, regexp: Expression, idx: Expression) override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { val matcher = ctx.freshName("matcher") + val source = ctx.freshName("source") val setEvNotNull = if (nullable) { s"${ev.isNull} = false;" } else { @@ -1153,7 +1155,8 @@ case class RegExpInStr(subject: Expression, regexp: Expression, idx: Expression) | ${RegExpUtils.initLastMatcherCode(ctx, subject, regexp, matcher, prettyName, collationId)} | if ($matcher.find()) { - | ${ev.value} = $matcher.toMatchResult().start() + 1; + | String $source = $subject.toString(); + | ${ev.value} = $source.codePointCount(0, $matcher.toMatchResult().start()) + 1; | } else { | ${ev.value} = 0; | } @@ -1232,9 +1235,9 @@ object RegExpUtils { replacement: String, pos: Int): UTF8String = { val position = pos - 1 - if (position == 0 || position < source.length) { + if (position == 0 || position < source.codePointCount(0, source.length)) { val matcher = pattern.matcher(source) - matcher.region(position, source.length) + matcher.region(source.offsetByCodePoints(0, position), source.length) val result = new JStringBuilder while (matcher.find()) { try { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala index 19c8c8f341f0a..0bf29553ea33d 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala @@ -377,6 +377,26 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { RegExpReplace(Literal("\"quote"), Literal("\"quote"), Literal("\"quote")) :: Nil) } + test("SPARK-57932: regexp_replace position is a code-point position for supplementary chars") { + // scalastyle:off nonascii + checkEvaluation( + RegExpReplace(Literal("😀aXa"), Literal("a"), Literal("Z"), Literal(3)), + "😀aXZ") + checkEvaluation( + RegExpReplace(Literal("😀😀"), Literal("😀"), + Literal("Z"), Literal(3)), + "😀😀") + // Position beyond the string length: nothing is replaced. + checkEvaluation( + RegExpReplace(Literal("😀a"), Literal("a"), Literal("Z"), Literal(4)), + "😀a") + // Position at the last code-point. + checkEvaluation( + RegExpReplace(Literal("a😀b"), Literal("b"), Literal("Z"), Literal(3)), + "a😀Z") + // scalastyle:on nonascii + } + test("SPARK-22570: RegExpReplace should not create a lot of global variables") { val ctx = new CodegenContext RegExpReplace(Literal("100"), Literal("(\\d+)"), Literal("num")).genCode(ctx) @@ -638,6 +658,17 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { new RegExpInStr(Literal("\"quote"), Literal("\"quote")) :: Nil) } + test("SPARK-57932: regexp_instr returns a code-point position for supplementary characters") { + // scalastyle:off nonascii + checkEvaluation(RegExpInStr(Literal("😀ab"), Literal("ab"), Literal(0)), 2) + checkEvaluation(RegExpInStr(Literal("a😀b"), Literal("b"), Literal(0)), 3) + checkEvaluation( + RegExpInStr(Literal("😀😁xy"), Literal("xy"), Literal(0)), 3) + // A match made up entirely of supplementary characters. + checkEvaluation(RegExpInStr(Literal("😀😁😂"), Literal("😁"), Literal(0)), 2) + // scalastyle:on nonascii + } + test("SPARK-39758: invalid regexp pattern") { val s = $"s".string.at(0) val p = $"p".string.at(1)