From 4386a800c819a3d230ba2d13bd02bcb84a0ff5a0 Mon Sep 17 00:00:00 2001 From: Ilya Gorbunov Date: Thu, 18 Nov 2021 12:57:25 +0300 Subject: [PATCH] KT-49721 Avoid matching from the same position after a zero-width match In JS, RegExp can return a match before the specified start index, if it has matched at that index, but it is in the middle of a surrogate pair. Account for that when advancing to the next position after a zero-width match so that it doesn't get to the middle of SP. --- .../tests/stdlib_external/utils.kt | 4 +++- libraries/stdlib/common/test/testUtils.kt | 4 +++- libraries/stdlib/js/src/kotlin/text/regex.kt | 15 ++++++++++++++- libraries/stdlib/js/test/core/testUtils.kt | 4 +++- libraries/stdlib/jvm/test/testUtilsJVM.kt | 4 +++- libraries/stdlib/test/text/RegexTest.kt | 18 ++++++++++++++++++ libraries/stdlib/wasm/test/testUtils.kt | 2 ++ 7 files changed, 46 insertions(+), 5 deletions(-) diff --git a/kotlin-native/backend.native/tests/stdlib_external/utils.kt b/kotlin-native/backend.native/tests/stdlib_external/utils.kt index 811106059fc..8e84b81a61f 100644 --- a/kotlin-native/backend.native/tests/stdlib_external/utils.kt +++ b/kotlin-native/backend.native/tests/stdlib_external/utils.kt @@ -30,4 +30,6 @@ public actual val isFloat32RangeEnforced: Boolean get() = true public actual val supportsSuppressedExceptions: Boolean get() = true -public actual val supportsNamedCapturingGroup: Boolean get() = false \ No newline at end of file +public actual val supportsNamedCapturingGroup: Boolean get() = false + +public actual val regexSplitUnicodeCodePointHandling: Boolean get() = true \ No newline at end of file diff --git a/libraries/stdlib/common/test/testUtils.kt b/libraries/stdlib/common/test/testUtils.kt index 1d8e4986145..22c30e289d7 100644 --- a/libraries/stdlib/common/test/testUtils.kt +++ b/libraries/stdlib/common/test/testUtils.kt @@ -18,4 +18,6 @@ public expect val isFloat32RangeEnforced: Boolean public expect val supportsSuppressedExceptions: Boolean -public expect val supportsNamedCapturingGroup: Boolean \ No newline at end of file +public expect val supportsNamedCapturingGroup: Boolean + +public expect val regexSplitUnicodeCodePointHandling: Boolean \ No newline at end of file diff --git a/libraries/stdlib/js/src/kotlin/text/regex.kt b/libraries/stdlib/js/src/kotlin/text/regex.kt index c2f63b79320..2eedcaa46df 100644 --- a/libraries/stdlib/js/src/kotlin/text/regex.kt +++ b/libraries/stdlib/js/src/kotlin/text/regex.kt @@ -359,7 +359,20 @@ private fun RegExp.findNext(input: String, from: Int, nextPattern: RegExp): Matc } override fun next(): MatchResult? = - nextPattern.findNext(input, if (range.isEmpty()) range.start + 1 else range.endInclusive + 1, nextPattern) + nextPattern.findNext(input, if (range.isEmpty()) advanceToNextCharacter(range.start) else range.endInclusive + 1, nextPattern) + + private fun advanceToNextCharacter(index: Int): Int { + if (index < input.lastIndex) { + val code1 = input.asDynamic().charCodeAt(index).unsafeCast() + if (code1 in 0xD800..0xDBFF) { + val code2 = input.asDynamic().charCodeAt(index + 1).unsafeCast() + if (code2 in 0xDC00..0xDFFF) { + return index + 2 + } + } + } + return index + 1 + } } } diff --git a/libraries/stdlib/js/test/core/testUtils.kt b/libraries/stdlib/js/test/core/testUtils.kt index 373d336dc45..ae2aa84d243 100644 --- a/libraries/stdlib/js/test/core/testUtils.kt +++ b/libraries/stdlib/js/test/core/testUtils.kt @@ -28,4 +28,6 @@ public actual val isFloat32RangeEnforced: Boolean = false actual val supportsSuppressedExceptions: Boolean get() = true // TODO: implement named group reference in replacement expression -public actual val supportsNamedCapturingGroup: Boolean get() = false \ No newline at end of file +public actual val supportsNamedCapturingGroup: Boolean get() = false + +public actual val regexSplitUnicodeCodePointHandling: Boolean get() = true \ No newline at end of file diff --git a/libraries/stdlib/jvm/test/testUtilsJVM.kt b/libraries/stdlib/jvm/test/testUtilsJVM.kt index 9b597a0f8be..5a9dbff4ce6 100644 --- a/libraries/stdlib/jvm/test/testUtilsJVM.kt +++ b/libraries/stdlib/jvm/test/testUtilsJVM.kt @@ -41,4 +41,6 @@ public actual val isFloat32RangeEnforced: Boolean = true public actual val supportsSuppressedExceptions: Boolean get() = !isJava6 -public actual val supportsNamedCapturingGroup: Boolean get() = !isJava6 \ No newline at end of file +public actual val supportsNamedCapturingGroup: Boolean get() = !isJava6 + +public actual val regexSplitUnicodeCodePointHandling: Boolean get() = false \ No newline at end of file diff --git a/libraries/stdlib/test/text/RegexTest.kt b/libraries/stdlib/test/text/RegexTest.kt index 2c055378960..80f0e29801e 100644 --- a/libraries/stdlib/test/text/RegexTest.kt +++ b/libraries/stdlib/test/text/RegexTest.kt @@ -7,6 +7,7 @@ package test.text +import test.regexSplitUnicodeCodePointHandling import test.supportsNamedCapturingGroup import kotlin.test.* @@ -354,6 +355,11 @@ class RegexTest { testSplitEquals("".split(""), "", emptyMatch) + testSplitEquals( + if (regexSplitUnicodeCodePointHandling) listOf("", "\uD83D\uDE04", "\uD801", "") else listOf("", "\uD83D", "\uDE04", "\uD801", ""), + "\uD83D\uDE04\uD801", emptyMatch + ) + val emptyMatchBeforeT = "(?=t)".toRegex() testSplitEquals(listOf("", "tes", "t"), input, emptyMatchBeforeT) @@ -397,4 +403,16 @@ class RegexTest { assertFailsWith { splits.next() } } + @Test fun findAllEmoji() { + val input = "\uD83D\uDE04\uD801x" + val regex = ".".toRegex() + + val matches = regex.findAll(input).toList() + val values = matches.map { it.value } + val ranges = matches.map { it.range } + + assertEquals(listOf("\uD83D\uDE04", "\uD801", "x"), values) + assertEquals(listOf(0..1, 2..2, 3..3), ranges) + } + } diff --git a/libraries/stdlib/wasm/test/testUtils.kt b/libraries/stdlib/wasm/test/testUtils.kt index 36e5e241014..3f58fa4ef83 100644 --- a/libraries/stdlib/wasm/test/testUtils.kt +++ b/libraries/stdlib/wasm/test/testUtils.kt @@ -31,3 +31,5 @@ actual val supportsSuppressedExceptions: Boolean get() = false // TODO: implement named group reference in replacement expression public actual val supportsNamedCapturingGroup: Boolean get() = false + +public actual val regexSplitUnicodeCodePointHandling: Boolean get() = TODO() \ No newline at end of file