[K/N and WASM] Fix ESCAPE and COMMENTS modes

libraries/stdlib/native-wasm/src/kotlin/text/regex/Lexer.kt

@@ -113,6 +113,10 @@ internal class Lexer(val patternString: String, flags: Int) {
         this.pattern[this.pattern.size - 1] = 0.toChar()
         this.pattern[this.pattern.size - 2] = 0.toChar()
 
+        // Skips leading comments and whitespaces if comments flag is on.
+        if (flags and Pattern.COMMENTS != 0) {
+            skipComments()
+        }
         // Read first two tokens.
         movePointer()
         movePointer()
@@ -200,10 +204,9 @@ internal class Lexer(val patternString: String, flags: Int) {
      */
     private fun nextIndex(): Int {
         prevNonWhitespaceIndex = index
-        if (flags and Pattern.COMMENTS != 0) {
+        index++
+        if (mode != Mode.ESCAPE && flags and Pattern.COMMENTS != 0) {
             skipComments()
-        } else {
-            index++
         }
         return prevNonWhitespaceIndex
     }
@@ -211,7 +214,6 @@ internal class Lexer(val patternString: String, flags: Int) {
     /** Skips comments and whitespaces */
     private fun skipComments(): Int {
         val length = pattern.size - 2
-        index++
         do {
             while (index < length && pattern[index].isWhitespace()) {
                 index++
@@ -289,6 +291,8 @@ internal class Lexer(val patternString: String, flags: Int) {
             if (lookAheadChar == 'E') {
                 // If \E found - change the mode to the previous one and shift to the next char.
                 mode = savedMode
+                index = prevNonWhitespaceIndex // index of 'E'
+                nextIndex() // skip 'E' and process the following chars with the saved mode
                 lookAhead = if (index <= pattern.size - 2) nextCodePoint() else 0
             } else {
                 // If \ have no E - make a step back and return.
@@ -337,7 +341,7 @@ internal class Lexer(val patternString: String, flags: Int) {
                 } else {
                     // Special constructs (non-capturing groups, named capturing groups, look ahead/look behind etc).
                     nextIndex()
-                    var char = pattern[index]
+                    var char = pattern[prevNonWhitespaceIndex + 1]
                     when (char) {
                         // Look ahead or an atomic group.
                         '!' -> {
@@ -419,18 +423,13 @@ internal class Lexer(val patternString: String, flags: Int) {
 
     /** Processes an escaped (\x) character in any mode. Returns whether we need to reread the character or not */
     private fun processEscapedChar() : Boolean {
-        lookAhead = if (index < pattern.size - 2) {
-            nextCodePoint()
-        } else {
+        val escapedCharIndex = prevNonWhitespaceIndex + 1
+        if (escapedCharIndex >= pattern.size - 2) {
             throw PatternSyntaxException("Trailing \\", patternString, curTokenIndex)
         }
-
-        // The current code point cannot be a surrogate pair because it is an escaped special one.
-        // Cast it to char or just skip it as if we pass through the else branch of the when below.
-        if (lookAhead.isSurrogatePair()) {
-            return false
-        }
-        val lookAheadChar = lookAhead.toChar()
+        index = escapedCharIndex
+        val lookAheadChar = pattern[nextIndex()]
+        lookAhead = lookAheadChar.toInt()
 
         when (lookAheadChar) {
             // Character class.
@@ -455,6 +454,8 @@ internal class Lexer(val patternString: String, flags: Int) {
             'Q' -> {
                 savedMode = mode
                 mode = Mode.ESCAPE
+                index = escapedCharIndex // index of 'Q'
+                nextIndex() // skip 'Q' and process the following chars with ESCAPE mode
                 return true
             }
 

libraries/stdlib/native-wasm/test/harmony_regex/PatternTest3.kt

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package test.text.harmony_regex
+
+import kotlin.test.*
+
+class PatternTest3 {
+    @Test fun escapeStringInCommentsMode() {
+        // Q and E immediately follow backslash
+        " [ a - z ] \\Q [ A - Z ] \\E [ a - z ] ".toRegex(RegexOption.COMMENTS).let { regex ->
+            assertTrue(regex.matches("a [ A - Z ] b"))
+        }
+        // \Q is separated and the backslash quotes the following char - the space, thus escape mode is not enabled.
+        // \E doesn't disable any escape mode, leading to the error.
+        assertFailsWith<IllegalArgumentException> {
+            " [ a - z ] \\ Q [ A - Z ] \\E [ a - z ] ".toRegex(RegexOption.COMMENTS) // Illegal/unsupported escape char E
+        }
+        // \E is separated, thus escape mode stands enabled till the end of pattern string.
+        " [ a - z ] \\Q [ A - Z ] \\ E [ a - z ] ".toRegex(RegexOption.COMMENTS).let { regex ->
+            assertTrue(regex.matches("a [ A - Z ] \\ E [ a - z ] "))
+        }
+        // both \Q and \E are separated, and the backslash quotes the following char - the space.
+        " [ a - z ] \\ Q [ A - Z ] \\  E [ a - z ] ".toRegex(RegexOption.COMMENTS).let { regex ->
+            assertTrue(regex.matches("a QB Ec"))
+        }
+    }
+
+    @Test fun specialConstructsInCommentsMode() {
+        // positive lookbehind
+        assertFailsWith<IllegalArgumentException> {
+            "(? <=[a-z])\\d".toRegex(RegexOption.COMMENTS) // (?<=X) - ?< shouldn't be separated
+        }
+        "(?< =[a-z])\\d".toRegex(RegexOption.COMMENTS).let { regex ->
+            assertEquals("4", regex.find("...a4...B1")?.value)
+        }
+        // negative lookbehind
+        assertFailsWith<IllegalArgumentException> {
+            "(? <![a-z])\\d".toRegex(RegexOption.COMMENTS) // (?<=X) - ?< shouldn't be separated
+        }
+        "(?< ![a-z])\\d".toRegex(RegexOption.COMMENTS).let { regex ->
+            assertEquals("1", regex.find("...a4...B1")?.value)
+        }
+        // positive lookahead
+        assertFailsWith<IllegalArgumentException> {
+            "[a-z](? =\\d)".toRegex(RegexOption.COMMENTS) // (?=X) - ?= shouldn't be separated
+        }
+        // negative lookahead
+        assertFailsWith<IllegalArgumentException> {
+            "[a-z](? !\\d)".toRegex(RegexOption.COMMENTS) // (?!X) - ?! shouldn't be separated
+        }
+    }
+
+    @Test fun matchNamedGroupInCommentsMode() {
+        assertFailsWith<IllegalArgumentException> {
+            "(? <first>\\d+)".toRegex(RegexOption.COMMENTS) // (?<name>X) - ?< shouldn't be separated
+        }
+        "( ?< first  > \\d + ) - (?< se c  ond >\\d+)".toRegex(RegexOption.COMMENTS).let { regex ->
+            val match = regex.find("123-456")!!
+            assertEquals("123-456", match.value)
+            assertEquals("123", match.groups["first"]?.value)
+            assertEquals("456", match.groups["second"]?.value)
+        }
+    }
+
+    @Test fun matchBackReferenceInCommentsMode() {
+        "(?<first>\\d+)-\\ k<first>".toRegex(RegexOption.COMMENTS).let { regex ->
+            assertTrue(regex.matches("123- k<first>")) // \k<name> - \k shouldn't be separated. Otherwise, backslash quotes the following char - the space
+        }
+        "(\\d+)-\\    1".toRegex(RegexOption.COMMENTS).let { regex ->
+            assertTrue(regex.matches("123- 1")) // \n - the construct shouldn't be separated. Otherwise, backslash quotes the following char - the space
+        }
+        "(?<first>\\d+)-\\k  < fi r  st > ".toRegex(RegexOption.COMMENTS).let { regex ->
+            val match = regex.find("123-123")!!
+            assertEquals("123-123", match.value)
+            assertEquals("123", match.groups["first"]?.value)
+        }
+        "0(1(2(3(4(5(6(7(8(9(A(B(C))))))))))))\\1  1 ".toRegex(RegexOption.COMMENTS).let { regex ->
+            val match = regex.find("0123456789ABCBC")!!
+            assertEquals("BC", match.groups[11]?.value)
+            assertEquals("56789ABC", match.groups[5]?.value)
+        }
+    }
+}