azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N and WASM] Fix ESCAPE and COMMENTS modes

Browse Source
This commit is contained in:
Abduqodiri Qurbonzoda
2022-04-04 16:52:03 +03:00
committed by Space
parent 7b46738796
commit 1cb5cab28f
9 changed files with 208 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/backend.native/tests/stdlib_external/utils.kt
+2
View File
@@ -34,6 +34,8 @@ public actual val supportsNamedCapturingGroup: Boolean get() = true
public actual val supportsOctalLiteralInRegex: Boolean get() = true
public actual val supportsEscapeAnyCharInRegex: Boolean get() = true
public actual val regexSplitUnicodeCodePointHandling: Boolean get() = true
public actual object BackReferenceHandling {
libraries/stdlib/common/test/testUtils.kt
+2
View File
@@ -22,6 +22,8 @@ public expect val supportsNamedCapturingGroup: Boolean
public expect val supportsOctalLiteralInRegex: Boolean
public expect val supportsEscapeAnyCharInRegex: Boolean
public expect val regexSplitUnicodeCodePointHandling: Boolean
public enum class HandlingOption {
libraries/stdlib/jdk8/test/text/RegexTest.kt
+76
View File
@@ -45,4 +45,80 @@ class RegexTest {
assertTrue("unknown_group" in e.message!!)
}
}
@Test fun escapeStringInCommentsMode() {
// Q and E immediately follow backslash
" [ a - z ] \\Q [ A - Z ] \\E [ a - z ] ".toRegex(RegexOption.COMMENTS).let { regex ->
assertTrue(regex.matches("a [ A - Z ] b"))
}
// \Q is separated and the backslash quotes the following char - the space, thus escape mode is not enabled.
// \E doesn't disable any escape mode, leading to the error.
assertFailsWith<IllegalArgumentException> {
" [ a - z ] \\ Q [ A - Z ] \\E [ a - z ] ".toRegex(RegexOption.COMMENTS) // Illegal/unsupported escape char E
}
// \E is separated, thus escape mode stands enabled till the end of pattern string.
" [ a - z ] \\Q [ A - Z ] \\ E [ a - z ] ".toRegex(RegexOption.COMMENTS).let { regex ->
assertTrue(regex.matches("a [ A - Z ] \\ E [ a - z ] "))
}
// both \Q and \E are separated, and the backslash quotes the following char - the space.
" [ a - z ] \\ Q [ A - Z ] \\ E [ a - z ] ".toRegex(RegexOption.COMMENTS).let { regex ->
assertTrue(regex.matches("a QB Ec"))
}
}
@Test fun specialConstructsInCommentsMode() {
// positive lookbehind
assertFailsWith<IllegalArgumentException> {
"(? <=[a-z])\\d".toRegex(RegexOption.COMMENTS) // (?<=X) - ?< shouldn't be separated
}
"(?< =[a-z])\\d".toRegex(RegexOption.COMMENTS).let { regex ->
assertEquals("4", regex.find("...a4...B1")?.value)
}
// negative lookbehind
assertFailsWith<IllegalArgumentException> {
"(? <![a-z])\\d".toRegex(RegexOption.COMMENTS) // (?<=X) - ?< shouldn't be separated
}
"(?< ![a-z])\\d".toRegex(RegexOption.COMMENTS).let { regex ->
assertEquals("1", regex.find("...a4...B1")?.value)
}
// positive lookahead
assertFailsWith<IllegalArgumentException> {
"[a-z](? =\\d)".toRegex(RegexOption.COMMENTS) // (?=X) - ?= shouldn't be separated
}
// negative lookahead
assertFailsWith<IllegalArgumentException> {
"[a-z](? !\\d)".toRegex(RegexOption.COMMENTS) // (?!X) - ?! shouldn't be separated
}
}
@Test fun matchNamedGroupInCommentsMode() {
assertFailsWith<IllegalArgumentException> {
"(? <first>\\d+)".toRegex(RegexOption.COMMENTS) // (?<name>X) - ?< shouldn't be separated
}
"( ?< first > \\d + ) - (?< se c ond >\\d+)".toRegex(RegexOption.COMMENTS).let { regex ->
val match = regex.find("123-456")!!
assertEquals("123-456", match.value)
assertEquals("123", match.groups["first"]?.value)
assertEquals("456", match.groups["second"]?.value)
}
}
@Test fun matchBackReferenceInCommentsMode() {
"(?<first>\\d+)-\\ k<first>".toRegex(RegexOption.COMMENTS).let { regex ->
assertTrue(regex.matches("123- k<first>")) // \k<name> - \k shouldn't be separated. Otherwise, backslash quotes the following char - the space
}
"(\\d+)-\\ 1".toRegex(RegexOption.COMMENTS).let { regex ->
assertTrue(regex.matches("123- 1")) // \n - the construct shouldn't be separated. Otherwise, backslash quotes the following char - the space
}
"(?<first>\\d+)-\\k < fi r st > ".toRegex(RegexOption.COMMENTS).let { regex ->
val match = regex.find("123-123")!!
assertEquals("123-123", match.value)
assertEquals("123", match.groups["first"]?.value)
}
"0(1(2(3(4(5(6(7(8(9(A(B(C))))))))))))\\1 1 ".toRegex(RegexOption.COMMENTS).let { regex ->
val match = regex.find("0123456789ABCBC")!!
assertEquals("BC", match.groups[11]?.value)
assertEquals("56789ABC", match.groups[5]?.value)
}
}
}
libraries/stdlib/js/test/core/testUtils.kt
+2
View File
@@ -31,6 +31,8 @@ public actual val supportsNamedCapturingGroup: Boolean get() = true
public actual val supportsOctalLiteralInRegex: Boolean get() = false
public actual val supportsEscapeAnyCharInRegex: Boolean get() = false
public actual val regexSplitUnicodeCodePointHandling: Boolean get() = true
public actual object BackReferenceHandling {
libraries/stdlib/jvm/test/testUtilsJVM.kt
+2
View File
@@ -45,6 +45,8 @@ public actual val supportsNamedCapturingGroup: Boolean get() = !isJava6
public actual val supportsOctalLiteralInRegex: Boolean get() = true
public actual val supportsEscapeAnyCharInRegex: Boolean get() = true
public actual val regexSplitUnicodeCodePointHandling: Boolean get() = false
public actual object BackReferenceHandling {
libraries/stdlib/native-wasm/src/kotlin/text/regex/Lexer.kt
+16 -15
View File
@@ -113,6 +113,10 @@ internal class Lexer(val patternString: String, flags: Int) {
this.pattern[this.pattern.size - 1] = 0.toChar()
this.pattern[this.pattern.size - 2] = 0.toChar()
// Skips leading comments and whitespaces if comments flag is on.
if (flags and Pattern.COMMENTS != 0) {
skipComments()
}
// Read first two tokens.
movePointer()
movePointer()
@@ -200,10 +204,9 @@ internal class Lexer(val patternString: String, flags: Int) {
*/
private fun nextIndex(): Int {
prevNonWhitespaceIndex = index
if (flags and Pattern.COMMENTS != 0) {
index++
if (mode != Mode.ESCAPE && flags and Pattern.COMMENTS != 0) {
skipComments()
} else {
index++
}
return prevNonWhitespaceIndex
}
@@ -211,7 +214,6 @@ internal class Lexer(val patternString: String, flags: Int) {
/** Skips comments and whitespaces */
private fun skipComments(): Int {
val length = pattern.size - 2
index++
do {
while (index < length && pattern[index].isWhitespace()) {
index++
@@ -289,6 +291,8 @@ internal class Lexer(val patternString: String, flags: Int) {
if (lookAheadChar == 'E') {
// If \E found - change the mode to the previous one and shift to the next char.
mode = savedMode
index = prevNonWhitespaceIndex // index of 'E'
nextIndex() // skip 'E' and process the following chars with the saved mode
lookAhead = if (index <= pattern.size - 2) nextCodePoint() else 0
} else {
// If \ have no E - make a step back and return.
@@ -337,7 +341,7 @@ internal class Lexer(val patternString: String, flags: Int) {
} else {
// Special constructs (non-capturing groups, named capturing groups, look ahead/look behind etc).
nextIndex()
var char = pattern[index]
var char = pattern[prevNonWhitespaceIndex + 1]
when (char) {
// Look ahead or an atomic group.
'!' -> {
@@ -419,18 +423,13 @@ internal class Lexer(val patternString: String, flags: Int) {
/** Processes an escaped (\x) character in any mode. Returns whether we need to reread the character or not */
private fun processEscapedChar() : Boolean {
lookAhead = if (index < pattern.size - 2) {
nextCodePoint()
} else {
val escapedCharIndex = prevNonWhitespaceIndex + 1
if (escapedCharIndex >= pattern.size - 2) {
throw PatternSyntaxException("Trailing \\", patternString, curTokenIndex)
}
// The current code point cannot be a surrogate pair because it is an escaped special one.
// Cast it to char or just skip it as if we pass through the else branch of the when below.
if (lookAhead.isSurrogatePair()) {
return false
}
val lookAheadChar = lookAhead.toChar()
index = escapedCharIndex
val lookAheadChar = pattern[nextIndex()]
lookAhead = lookAheadChar.toInt()
when (lookAheadChar) {
// Character class.
@@ -455,6 +454,8 @@ internal class Lexer(val patternString: String, flags: Int) {
'Q' -> {
savedMode = mode
mode = Mode.ESCAPE
index = escapedCharIndex // index of 'Q'
nextIndex() // skip 'Q' and process the following chars with ESCAPE mode
return true
}
libraries/stdlib/native-wasm/test/harmony_regex/PatternTest3.kt
+86
View File
@@ -0,0 +1,86 @@
/*
* Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package test.text.harmony_regex
import kotlin.test.*
class PatternTest3 {
@Test fun escapeStringInCommentsMode() {
// Q and E immediately follow backslash
" [ a - z ] \\Q [ A - Z ] \\E [ a - z ] ".toRegex(RegexOption.COMMENTS).let { regex ->
assertTrue(regex.matches("a [ A - Z ] b"))
}
// \Q is separated and the backslash quotes the following char - the space, thus escape mode is not enabled.
// \E doesn't disable any escape mode, leading to the error.
assertFailsWith<IllegalArgumentException> {
" [ a - z ] \\ Q [ A - Z ] \\E [ a - z ] ".toRegex(RegexOption.COMMENTS) // Illegal/unsupported escape char E
}
// \E is separated, thus escape mode stands enabled till the end of pattern string.
" [ a - z ] \\Q [ A - Z ] \\ E [ a - z ] ".toRegex(RegexOption.COMMENTS).let { regex ->
assertTrue(regex.matches("a [ A - Z ] \\ E [ a - z ] "))
}
// both \Q and \E are separated, and the backslash quotes the following char - the space.
" [ a - z ] \\ Q [ A - Z ] \\ E [ a - z ] ".toRegex(RegexOption.COMMENTS).let { regex ->
assertTrue(regex.matches("a QB Ec"))
}
}
@Test fun specialConstructsInCommentsMode() {
// positive lookbehind
assertFailsWith<IllegalArgumentException> {
"(? <=[a-z])\\d".toRegex(RegexOption.COMMENTS) // (?<=X) - ?< shouldn't be separated
}
"(?< =[a-z])\\d".toRegex(RegexOption.COMMENTS).let { regex ->
assertEquals("4", regex.find("...a4...B1")?.value)
}
// negative lookbehind
assertFailsWith<IllegalArgumentException> {
"(? <![a-z])\\d".toRegex(RegexOption.COMMENTS) // (?<=X) - ?< shouldn't be separated
}
"(?< ![a-z])\\d".toRegex(RegexOption.COMMENTS).let { regex ->
assertEquals("1", regex.find("...a4...B1")?.value)
}
// positive lookahead
assertFailsWith<IllegalArgumentException> {
"[a-z](? =\\d)".toRegex(RegexOption.COMMENTS) // (?=X) - ?= shouldn't be separated
}
// negative lookahead
assertFailsWith<IllegalArgumentException> {
"[a-z](? !\\d)".toRegex(RegexOption.COMMENTS) // (?!X) - ?! shouldn't be separated
}
}
@Test fun matchNamedGroupInCommentsMode() {
assertFailsWith<IllegalArgumentException> {
"(? <first>\\d+)".toRegex(RegexOption.COMMENTS) // (?<name>X) - ?< shouldn't be separated
}
"( ?< first > \\d + ) - (?< se c ond >\\d+)".toRegex(RegexOption.COMMENTS).let { regex ->
val match = regex.find("123-456")!!
assertEquals("123-456", match.value)
assertEquals("123", match.groups["first"]?.value)
assertEquals("456", match.groups["second"]?.value)
}
}
@Test fun matchBackReferenceInCommentsMode() {
"(?<first>\\d+)-\\ k<first>".toRegex(RegexOption.COMMENTS).let { regex ->
assertTrue(regex.matches("123- k<first>")) // \k<name> - \k shouldn't be separated. Otherwise, backslash quotes the following char - the space
}
"(\\d+)-\\ 1".toRegex(RegexOption.COMMENTS).let { regex ->
assertTrue(regex.matches("123- 1")) // \n - the construct shouldn't be separated. Otherwise, backslash quotes the following char - the space
}
"(?<first>\\d+)-\\k < fi r st > ".toRegex(RegexOption.COMMENTS).let { regex ->
val match = regex.find("123-123")!!
assertEquals("123-123", match.value)
assertEquals("123", match.groups["first"]?.value)
}
"0(1(2(3(4(5(6(7(8(9(A(B(C))))))))))))\\1 1 ".toRegex(RegexOption.COMMENTS).let { regex ->
val match = regex.find("0123456789ABCBC")!!
assertEquals("BC", match.groups[11]?.value)
assertEquals("56789ABC", match.groups[5]?.value)
}
}
}
libraries/stdlib/test/text/RegexTest.kt
+20 -12
View File
@@ -10,6 +10,7 @@ package test.text
import test.regexSplitUnicodeCodePointHandling
import test.supportsNamedCapturingGroup
import test.supportsOctalLiteralInRegex
import test.supportsEscapeAnyCharInRegex
import test.BackReferenceHandling
import test.HandlingOption
import kotlin.test.*
@@ -59,6 +60,20 @@ class RegexTest {
assertEquals(null, p.find(input, input.length))
}
@Test fun matchEscapeSurrogatePair() {
if (!supportsEscapeAnyCharInRegex) return
val regex = "\\\uD83D\uDE00".toRegex()
assertTrue(regex.matches("\uD83D\uDE00"))
}
@Test fun matchEscapeRandomChar() {
if (!supportsEscapeAnyCharInRegex) return
val regex = "\\-".toRegex()
assertTrue(regex.matches("-"))
}
@Test fun matchIgnoreCase() {
for (input in listOf("ascii", "shrödinger"))
assertTrue(input.uppercase().matches(input.lowercase().toRegex(RegexOption.IGNORE_CASE)))
@@ -177,12 +192,9 @@ class RegexTest {
@Test fun matchDuplicateGroupName() {
if (!supportsNamedCapturingGroup) return
assertFailsWith<IllegalArgumentException> {
"(?<hi>hi)|(?<hi>bye)".toRegex()
}
assertFailsWith<IllegalArgumentException> {
Regex("(?<first>\\d+)-(?<first>\\d+)")
}
// should fail with IllegalArgumentException, but JS fails with SyntaxError
assertFails { "(?<hi>hi)|(?<hi>bye)".toRegex() }
assertFails { "(?<first>\\d+)-(?<first>\\d+)".toRegex() }
}
@Test fun matchOptionalNamedGroup() {
@@ -286,7 +298,7 @@ class RegexTest {
HandlingOption.IGNORE_BACK_REFERENCE_EXPRESSION ->
assertEquals(matchValue, pattern.toRegex().find(input)?.value)
HandlingOption.THROW ->
// TODO: should fail with IllegalArgumentException, but JS fails with SyntaxError
// should fail with IllegalArgumentException, but JS fails with SyntaxError
assertFails { pattern.toRegex() }
HandlingOption.MATCH_NOTHING ->
assertNull(pattern.toRegex().find(input))
@@ -296,7 +308,7 @@ class RegexTest {
@Test fun invalidNamedGroupDeclaration() {
if (!supportsNamedCapturingGroup) return
// TODO: should fail with IllegalArgumentException, but JS fails with SyntaxError
// should fail with IllegalArgumentException, but JS fails with SyntaxError
assertFails {
"(?<".toRegex()
@@ -318,10 +330,6 @@ class RegexTest {
}
}
// TODO: Test comment mode enabled and group name is separated by space, (before, in the middle, after)
// TODO: Test comment mode enabled and back reference group index is separated by space, (before, in the middle, after)
// TODO: IllegalArgumentException to java.util.regex.PatternSyntaxException where possible
@Test fun matchMultiline() {
val regex = "^[a-z]*$".toRegex(setOf(RegexOption.IGNORE_CASE, RegexOption.MULTILINE))
val matchedValues = regex.findAll("test\n\nLine").map { it.value }.toList()
libraries/stdlib/wasm/test/testUtils.kt
+2
View File
@@ -32,6 +32,8 @@ public actual val supportsNamedCapturingGroup: Boolean get() = true
public actual val supportsOctalLiteralInRegex: Boolean get() = true
public actual val supportsEscapeAnyCharInRegex: Boolean get() = true
public actual val regexSplitUnicodeCodePointHandling: Boolean get() = true
public actual object BackReferenceHandling {