From 1cb5cab28f8a336ac9d456e8791b1f0d1416abcb Mon Sep 17 00:00:00 2001 From: Abduqodiri Qurbonzoda Date: Mon, 4 Apr 2022 16:52:03 +0300 Subject: [PATCH] [K/N and WASM] Fix ESCAPE and COMMENTS modes --- .../tests/stdlib_external/utils.kt | 2 + libraries/stdlib/common/test/testUtils.kt | 2 + libraries/stdlib/jdk8/test/text/RegexTest.kt | 76 ++++++++++++++++ libraries/stdlib/js/test/core/testUtils.kt | 2 + libraries/stdlib/jvm/test/testUtilsJVM.kt | 2 + .../src/kotlin/text/regex/Lexer.kt | 31 +++---- .../test/harmony_regex/PatternTest3.kt | 86 +++++++++++++++++++ libraries/stdlib/test/text/RegexTest.kt | 32 ++++--- libraries/stdlib/wasm/test/testUtils.kt | 2 + 9 files changed, 208 insertions(+), 27 deletions(-) create mode 100644 libraries/stdlib/native-wasm/test/harmony_regex/PatternTest3.kt diff --git a/kotlin-native/backend.native/tests/stdlib_external/utils.kt b/kotlin-native/backend.native/tests/stdlib_external/utils.kt index 1eb6826d3c1..8ca0f9b13ef 100644 --- a/kotlin-native/backend.native/tests/stdlib_external/utils.kt +++ b/kotlin-native/backend.native/tests/stdlib_external/utils.kt @@ -34,6 +34,8 @@ public actual val supportsNamedCapturingGroup: Boolean get() = true public actual val supportsOctalLiteralInRegex: Boolean get() = true +public actual val supportsEscapeAnyCharInRegex: Boolean get() = true + public actual val regexSplitUnicodeCodePointHandling: Boolean get() = true public actual object BackReferenceHandling { diff --git a/libraries/stdlib/common/test/testUtils.kt b/libraries/stdlib/common/test/testUtils.kt index 856eb4769ea..5d2e02d3682 100644 --- a/libraries/stdlib/common/test/testUtils.kt +++ b/libraries/stdlib/common/test/testUtils.kt @@ -22,6 +22,8 @@ public expect val supportsNamedCapturingGroup: Boolean public expect val supportsOctalLiteralInRegex: Boolean +public expect val supportsEscapeAnyCharInRegex: Boolean + public expect val regexSplitUnicodeCodePointHandling: Boolean public enum class HandlingOption { diff --git a/libraries/stdlib/jdk8/test/text/RegexTest.kt b/libraries/stdlib/jdk8/test/text/RegexTest.kt index f8dc62defd3..349749a6a3a 100644 --- a/libraries/stdlib/jdk8/test/text/RegexTest.kt +++ b/libraries/stdlib/jdk8/test/text/RegexTest.kt @@ -45,4 +45,80 @@ class RegexTest { assertTrue("unknown_group" in e.message!!) } } + + @Test fun escapeStringInCommentsMode() { + // Q and E immediately follow backslash + " [ a - z ] \\Q [ A - Z ] \\E [ a - z ] ".toRegex(RegexOption.COMMENTS).let { regex -> + assertTrue(regex.matches("a [ A - Z ] b")) + } + // \Q is separated and the backslash quotes the following char - the space, thus escape mode is not enabled. + // \E doesn't disable any escape mode, leading to the error. + assertFailsWith { + " [ a - z ] \\ Q [ A - Z ] \\E [ a - z ] ".toRegex(RegexOption.COMMENTS) // Illegal/unsupported escape char E + } + // \E is separated, thus escape mode stands enabled till the end of pattern string. + " [ a - z ] \\Q [ A - Z ] \\ E [ a - z ] ".toRegex(RegexOption.COMMENTS).let { regex -> + assertTrue(regex.matches("a [ A - Z ] \\ E [ a - z ] ")) + } + // both \Q and \E are separated, and the backslash quotes the following char - the space. + " [ a - z ] \\ Q [ A - Z ] \\ E [ a - z ] ".toRegex(RegexOption.COMMENTS).let { regex -> + assertTrue(regex.matches("a QB Ec")) + } + } + + @Test fun specialConstructsInCommentsMode() { + // positive lookbehind + assertFailsWith { + "(? <=[a-z])\\d".toRegex(RegexOption.COMMENTS) // (?<=X) - ?< shouldn't be separated + } + "(?< =[a-z])\\d".toRegex(RegexOption.COMMENTS).let { regex -> + assertEquals("4", regex.find("...a4...B1")?.value) + } + // negative lookbehind + assertFailsWith { + "(? + assertEquals("1", regex.find("...a4...B1")?.value) + } + // positive lookahead + assertFailsWith { + "[a-z](? =\\d)".toRegex(RegexOption.COMMENTS) // (?=X) - ?= shouldn't be separated + } + // negative lookahead + assertFailsWith { + "[a-z](? !\\d)".toRegex(RegexOption.COMMENTS) // (?!X) - ?! shouldn't be separated + } + } + + @Test fun matchNamedGroupInCommentsMode() { + assertFailsWith { + "(? \\d+)".toRegex(RegexOption.COMMENTS) // (?X) - ?< shouldn't be separated + } + "( ?< first > \\d + ) - (?< se c ond >\\d+)".toRegex(RegexOption.COMMENTS).let { regex -> + val match = regex.find("123-456")!! + assertEquals("123-456", match.value) + assertEquals("123", match.groups["first"]?.value) + assertEquals("456", match.groups["second"]?.value) + } + } + + @Test fun matchBackReferenceInCommentsMode() { + "(?\\d+)-\\ k".toRegex(RegexOption.COMMENTS).let { regex -> + assertTrue(regex.matches("123- k")) // \k - \k shouldn't be separated. Otherwise, backslash quotes the following char - the space + } + "(\\d+)-\\ 1".toRegex(RegexOption.COMMENTS).let { regex -> + assertTrue(regex.matches("123- 1")) // \n - the construct shouldn't be separated. Otherwise, backslash quotes the following char - the space + } + "(?\\d+)-\\k < fi r st > ".toRegex(RegexOption.COMMENTS).let { regex -> + val match = regex.find("123-123")!! + assertEquals("123-123", match.value) + assertEquals("123", match.groups["first"]?.value) + } + "0(1(2(3(4(5(6(7(8(9(A(B(C))))))))))))\\1 1 ".toRegex(RegexOption.COMMENTS).let { regex -> + val match = regex.find("0123456789ABCBC")!! + assertEquals("BC", match.groups[11]?.value) + assertEquals("56789ABC", match.groups[5]?.value) + } + } } diff --git a/libraries/stdlib/js/test/core/testUtils.kt b/libraries/stdlib/js/test/core/testUtils.kt index e53b7fe6a79..03f000201af 100644 --- a/libraries/stdlib/js/test/core/testUtils.kt +++ b/libraries/stdlib/js/test/core/testUtils.kt @@ -31,6 +31,8 @@ public actual val supportsNamedCapturingGroup: Boolean get() = true public actual val supportsOctalLiteralInRegex: Boolean get() = false +public actual val supportsEscapeAnyCharInRegex: Boolean get() = false + public actual val regexSplitUnicodeCodePointHandling: Boolean get() = true public actual object BackReferenceHandling { diff --git a/libraries/stdlib/jvm/test/testUtilsJVM.kt b/libraries/stdlib/jvm/test/testUtilsJVM.kt index 43e1fe1c148..079d7319cdd 100644 --- a/libraries/stdlib/jvm/test/testUtilsJVM.kt +++ b/libraries/stdlib/jvm/test/testUtilsJVM.kt @@ -45,6 +45,8 @@ public actual val supportsNamedCapturingGroup: Boolean get() = !isJava6 public actual val supportsOctalLiteralInRegex: Boolean get() = true +public actual val supportsEscapeAnyCharInRegex: Boolean get() = true + public actual val regexSplitUnicodeCodePointHandling: Boolean get() = false public actual object BackReferenceHandling { diff --git a/libraries/stdlib/native-wasm/src/kotlin/text/regex/Lexer.kt b/libraries/stdlib/native-wasm/src/kotlin/text/regex/Lexer.kt index 9149d3b2b15..c56e5b8f4b3 100644 --- a/libraries/stdlib/native-wasm/src/kotlin/text/regex/Lexer.kt +++ b/libraries/stdlib/native-wasm/src/kotlin/text/regex/Lexer.kt @@ -113,6 +113,10 @@ internal class Lexer(val patternString: String, flags: Int) { this.pattern[this.pattern.size - 1] = 0.toChar() this.pattern[this.pattern.size - 2] = 0.toChar() + // Skips leading comments and whitespaces if comments flag is on. + if (flags and Pattern.COMMENTS != 0) { + skipComments() + } // Read first two tokens. movePointer() movePointer() @@ -200,10 +204,9 @@ internal class Lexer(val patternString: String, flags: Int) { */ private fun nextIndex(): Int { prevNonWhitespaceIndex = index - if (flags and Pattern.COMMENTS != 0) { + index++ + if (mode != Mode.ESCAPE && flags and Pattern.COMMENTS != 0) { skipComments() - } else { - index++ } return prevNonWhitespaceIndex } @@ -211,7 +214,6 @@ internal class Lexer(val patternString: String, flags: Int) { /** Skips comments and whitespaces */ private fun skipComments(): Int { val length = pattern.size - 2 - index++ do { while (index < length && pattern[index].isWhitespace()) { index++ @@ -289,6 +291,8 @@ internal class Lexer(val patternString: String, flags: Int) { if (lookAheadChar == 'E') { // If \E found - change the mode to the previous one and shift to the next char. mode = savedMode + index = prevNonWhitespaceIndex // index of 'E' + nextIndex() // skip 'E' and process the following chars with the saved mode lookAhead = if (index <= pattern.size - 2) nextCodePoint() else 0 } else { // If \ have no E - make a step back and return. @@ -337,7 +341,7 @@ internal class Lexer(val patternString: String, flags: Int) { } else { // Special constructs (non-capturing groups, named capturing groups, look ahead/look behind etc). nextIndex() - var char = pattern[index] + var char = pattern[prevNonWhitespaceIndex + 1] when (char) { // Look ahead or an atomic group. '!' -> { @@ -419,18 +423,13 @@ internal class Lexer(val patternString: String, flags: Int) { /** Processes an escaped (\x) character in any mode. Returns whether we need to reread the character or not */ private fun processEscapedChar() : Boolean { - lookAhead = if (index < pattern.size - 2) { - nextCodePoint() - } else { + val escapedCharIndex = prevNonWhitespaceIndex + 1 + if (escapedCharIndex >= pattern.size - 2) { throw PatternSyntaxException("Trailing \\", patternString, curTokenIndex) } - - // The current code point cannot be a surrogate pair because it is an escaped special one. - // Cast it to char or just skip it as if we pass through the else branch of the when below. - if (lookAhead.isSurrogatePair()) { - return false - } - val lookAheadChar = lookAhead.toChar() + index = escapedCharIndex + val lookAheadChar = pattern[nextIndex()] + lookAhead = lookAheadChar.toInt() when (lookAheadChar) { // Character class. @@ -455,6 +454,8 @@ internal class Lexer(val patternString: String, flags: Int) { 'Q' -> { savedMode = mode mode = Mode.ESCAPE + index = escapedCharIndex // index of 'Q' + nextIndex() // skip 'Q' and process the following chars with ESCAPE mode return true } diff --git a/libraries/stdlib/native-wasm/test/harmony_regex/PatternTest3.kt b/libraries/stdlib/native-wasm/test/harmony_regex/PatternTest3.kt new file mode 100644 index 00000000000..f0b55a24fa1 --- /dev/null +++ b/libraries/stdlib/native-wasm/test/harmony_regex/PatternTest3.kt @@ -0,0 +1,86 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package test.text.harmony_regex + +import kotlin.test.* + +class PatternTest3 { + @Test fun escapeStringInCommentsMode() { + // Q and E immediately follow backslash + " [ a - z ] \\Q [ A - Z ] \\E [ a - z ] ".toRegex(RegexOption.COMMENTS).let { regex -> + assertTrue(regex.matches("a [ A - Z ] b")) + } + // \Q is separated and the backslash quotes the following char - the space, thus escape mode is not enabled. + // \E doesn't disable any escape mode, leading to the error. + assertFailsWith { + " [ a - z ] \\ Q [ A - Z ] \\E [ a - z ] ".toRegex(RegexOption.COMMENTS) // Illegal/unsupported escape char E + } + // \E is separated, thus escape mode stands enabled till the end of pattern string. + " [ a - z ] \\Q [ A - Z ] \\ E [ a - z ] ".toRegex(RegexOption.COMMENTS).let { regex -> + assertTrue(regex.matches("a [ A - Z ] \\ E [ a - z ] ")) + } + // both \Q and \E are separated, and the backslash quotes the following char - the space. + " [ a - z ] \\ Q [ A - Z ] \\ E [ a - z ] ".toRegex(RegexOption.COMMENTS).let { regex -> + assertTrue(regex.matches("a QB Ec")) + } + } + + @Test fun specialConstructsInCommentsMode() { + // positive lookbehind + assertFailsWith { + "(? <=[a-z])\\d".toRegex(RegexOption.COMMENTS) // (?<=X) - ?< shouldn't be separated + } + "(?< =[a-z])\\d".toRegex(RegexOption.COMMENTS).let { regex -> + assertEquals("4", regex.find("...a4...B1")?.value) + } + // negative lookbehind + assertFailsWith { + "(? + assertEquals("1", regex.find("...a4...B1")?.value) + } + // positive lookahead + assertFailsWith { + "[a-z](? =\\d)".toRegex(RegexOption.COMMENTS) // (?=X) - ?= shouldn't be separated + } + // negative lookahead + assertFailsWith { + "[a-z](? !\\d)".toRegex(RegexOption.COMMENTS) // (?!X) - ?! shouldn't be separated + } + } + + @Test fun matchNamedGroupInCommentsMode() { + assertFailsWith { + "(? \\d+)".toRegex(RegexOption.COMMENTS) // (?X) - ?< shouldn't be separated + } + "( ?< first > \\d + ) - (?< se c ond >\\d+)".toRegex(RegexOption.COMMENTS).let { regex -> + val match = regex.find("123-456")!! + assertEquals("123-456", match.value) + assertEquals("123", match.groups["first"]?.value) + assertEquals("456", match.groups["second"]?.value) + } + } + + @Test fun matchBackReferenceInCommentsMode() { + "(?\\d+)-\\ k".toRegex(RegexOption.COMMENTS).let { regex -> + assertTrue(regex.matches("123- k")) // \k - \k shouldn't be separated. Otherwise, backslash quotes the following char - the space + } + "(\\d+)-\\ 1".toRegex(RegexOption.COMMENTS).let { regex -> + assertTrue(regex.matches("123- 1")) // \n - the construct shouldn't be separated. Otherwise, backslash quotes the following char - the space + } + "(?\\d+)-\\k < fi r st > ".toRegex(RegexOption.COMMENTS).let { regex -> + val match = regex.find("123-123")!! + assertEquals("123-123", match.value) + assertEquals("123", match.groups["first"]?.value) + } + "0(1(2(3(4(5(6(7(8(9(A(B(C))))))))))))\\1 1 ".toRegex(RegexOption.COMMENTS).let { regex -> + val match = regex.find("0123456789ABCBC")!! + assertEquals("BC", match.groups[11]?.value) + assertEquals("56789ABC", match.groups[5]?.value) + } + } +} \ No newline at end of file diff --git a/libraries/stdlib/test/text/RegexTest.kt b/libraries/stdlib/test/text/RegexTest.kt index 478bed971ec..3b9ec6daa38 100644 --- a/libraries/stdlib/test/text/RegexTest.kt +++ b/libraries/stdlib/test/text/RegexTest.kt @@ -10,6 +10,7 @@ package test.text import test.regexSplitUnicodeCodePointHandling import test.supportsNamedCapturingGroup import test.supportsOctalLiteralInRegex +import test.supportsEscapeAnyCharInRegex import test.BackReferenceHandling import test.HandlingOption import kotlin.test.* @@ -59,6 +60,20 @@ class RegexTest { assertEquals(null, p.find(input, input.length)) } + @Test fun matchEscapeSurrogatePair() { + if (!supportsEscapeAnyCharInRegex) return + + val regex = "\\\uD83D\uDE00".toRegex() + assertTrue(regex.matches("\uD83D\uDE00")) + } + + @Test fun matchEscapeRandomChar() { + if (!supportsEscapeAnyCharInRegex) return + + val regex = "\\-".toRegex() + assertTrue(regex.matches("-")) + } + @Test fun matchIgnoreCase() { for (input in listOf("ascii", "shrödinger")) assertTrue(input.uppercase().matches(input.lowercase().toRegex(RegexOption.IGNORE_CASE))) @@ -177,12 +192,9 @@ class RegexTest { @Test fun matchDuplicateGroupName() { if (!supportsNamedCapturingGroup) return - assertFailsWith { - "(?hi)|(?bye)".toRegex() - } - assertFailsWith { - Regex("(?\\d+)-(?\\d+)") - } + // should fail with IllegalArgumentException, but JS fails with SyntaxError + assertFails { "(?hi)|(?bye)".toRegex() } + assertFails { "(?\\d+)-(?\\d+)".toRegex() } } @Test fun matchOptionalNamedGroup() { @@ -286,7 +298,7 @@ class RegexTest { HandlingOption.IGNORE_BACK_REFERENCE_EXPRESSION -> assertEquals(matchValue, pattern.toRegex().find(input)?.value) HandlingOption.THROW -> - // TODO: should fail with IllegalArgumentException, but JS fails with SyntaxError + // should fail with IllegalArgumentException, but JS fails with SyntaxError assertFails { pattern.toRegex() } HandlingOption.MATCH_NOTHING -> assertNull(pattern.toRegex().find(input)) @@ -296,7 +308,7 @@ class RegexTest { @Test fun invalidNamedGroupDeclaration() { if (!supportsNamedCapturingGroup) return - // TODO: should fail with IllegalArgumentException, but JS fails with SyntaxError + // should fail with IllegalArgumentException, but JS fails with SyntaxError assertFails { "(?<".toRegex() @@ -318,10 +330,6 @@ class RegexTest { } } - // TODO: Test comment mode enabled and group name is separated by space, (before, in the middle, after) - // TODO: Test comment mode enabled and back reference group index is separated by space, (before, in the middle, after) - // TODO: IllegalArgumentException to java.util.regex.PatternSyntaxException where possible - @Test fun matchMultiline() { val regex = "^[a-z]*$".toRegex(setOf(RegexOption.IGNORE_CASE, RegexOption.MULTILINE)) val matchedValues = regex.findAll("test\n\nLine").map { it.value }.toList() diff --git a/libraries/stdlib/wasm/test/testUtils.kt b/libraries/stdlib/wasm/test/testUtils.kt index ef5c81dbc7f..ec6333e6c73 100644 --- a/libraries/stdlib/wasm/test/testUtils.kt +++ b/libraries/stdlib/wasm/test/testUtils.kt @@ -32,6 +32,8 @@ public actual val supportsNamedCapturingGroup: Boolean get() = true public actual val supportsOctalLiteralInRegex: Boolean get() = true +public actual val supportsEscapeAnyCharInRegex: Boolean get() = true + public actual val regexSplitUnicodeCodePointHandling: Boolean get() = true public actual object BackReferenceHandling {