From fb31a29c3943662157a821aae41390dae36b54a7 Mon Sep 17 00:00:00 2001 From: Abduqodiri Qurbonzoda Date: Thu, 17 Nov 2022 00:04:03 +0200 Subject: [PATCH] [K/N] Fix stack overflow in regex when a quantifier is matched many times Motivation: Users often expect simple patterns, like `[a]+` or `[^a]+`, to work fast and without any problems, even with long strings. Char class from the first pattern matches only 'a' and gets wrapped into LeafQuantifierSet, and works fine with long strings indeed. Char class from the second pattern, however, matches any character except 'a', including supplementary code points. So, the number of chars it consumes is not known beforehand. There is no optimization for such char classes, and if they are matched multiple times, the stack memory gets exhausted. Modification: Introduce FixedLengthQuantifierSet node. The node represents quantifier over constructs that consume a fixed amount of characters for a given string and index. Such constructs don't need backtracking to find a different match. Thus, it is possible for the node to avoid recursion when matching multiple times. Result: Fixes KT-46211, KT-35508 and probably KT-39789. Reproducer for the latter issue is no longer available, but error stacktrace resembles those of the other issues. --- .../src/kotlin/text/regex/Pattern.kt | 147 +++++---------- .../src/kotlin/text/regex/sets/AbstractSet.kt | 9 + .../text/regex/sets/CompositeRangeSet.kt | 8 +- .../text/regex/sets/DecomposedCharSet.kt | 6 +- .../src/kotlin/text/regex/sets/DotSet.kt | 6 + .../regex/sets/FixedLengthQuantifierSet.kt | 62 +++++++ .../text/regex/sets/GroupQuantifierSet.kt | 6 +- .../regex/sets/HangulDecomposedCharSet.kt | 6 +- .../text/regex/sets/LeafQuantifierSet.kt | 7 + .../src/kotlin/text/regex/sets/LeafSet.kt | 3 + .../PossessiveFixedLengthQuantifierSet.kt | 34 ++++ .../sets/ReluctantFixedLengthQuantifierSet.kt | 48 +++++ .../text/regex/sets/SupplementaryRangeSet.kt | 6 + .../text/regex/sets/SurrogateRangeSet.kt | 2 +- .../FixedLengthQuantifierTest.kt | 174 ++++++++++++++++++ .../test/harmony_regex/MatchResultTest.kt | 32 ++-- 16 files changed, 440 insertions(+), 116 deletions(-) create mode 100644 libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/FixedLengthQuantifierSet.kt create mode 100644 libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/PossessiveFixedLengthQuantifierSet.kt create mode 100644 libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/ReluctantFixedLengthQuantifierSet.kt create mode 100644 libraries/stdlib/native-wasm/test/harmony_regex/FixedLengthQuantifierTest.kt diff --git a/libraries/stdlib/native-wasm/src/kotlin/text/regex/Pattern.kt b/libraries/stdlib/native-wasm/src/kotlin/text/regex/Pattern.kt index 309d8da39ac..3f3747664c5 100644 --- a/libraries/stdlib/native-wasm/src/kotlin/text/regex/Pattern.kt +++ b/libraries/stdlib/native-wasm/src/kotlin/text/regex/Pattern.kt @@ -349,115 +349,68 @@ internal class Pattern(val pattern: String, flags: Int = 0) { return cur } + private fun quantifierFromLexerToken(quant: Int): Quantifier { + return when (quant) { + Lexer.QUANT_COMP, Lexer.QUANT_COMP_R, Lexer.QUANT_COMP_P -> { + lexemes.nextSpecial() as Quantifier + } + else -> { + lexemes.next() + Quantifier.fromLexerToken(quant) + } + } + } + /** * Q->T(*|+|?...) also do some optimizations. */ private fun processQuantifier(last: AbstractSet, term: AbstractSet): AbstractSet { val quant = lexemes.currentChar - if (term !is LeafSet) { - return when (quant) { - Lexer.QUANT_STAR, Lexer.QUANT_PLUS -> { - val q: QuantifierSet + if (term.type == AbstractSet.TYPE_DOTSET && (quant == Lexer.QUANT_STAR || quant == Lexer.QUANT_PLUS)) { + lexemes.next() + return DotQuantifierSet(term, last, quant, AbstractLineTerminator.getInstance(flags), hasFlag(Pattern.DOTALL)) + } - lexemes.next() - if (term.type == AbstractSet.TYPE_DOTSET) { - q = DotQuantifierSet(term, last, quant, AbstractLineTerminator.getInstance(flags), hasFlag(Pattern.DOTALL)) - } else { - q = GroupQuantifierSet(Quantifier.fromLexerToken(quant), term, last, quant, groupQuantifierCount++) - } - term.next = q - q + return when (quant) { + + Lexer.QUANT_STAR, Lexer.QUANT_PLUS, Lexer.QUANT_ALT, Lexer.QUANT_COMP -> { + val quantifier = quantifierFromLexerToken(quant) + when { + term is LeafSet -> + LeafQuantifierSet(quantifier, term, last, quant) + term.consumesFixedLength -> + FixedLengthQuantifierSet(quantifier, term, last, quant) + else -> + GroupQuantifierSet(quantifier, term, last, quant, groupQuantifierCount++) } - - Lexer.QUANT_ALT -> { - lexemes.next() - val q = GroupQuantifierSet(Quantifier.fromLexerToken(quant), term, last, quant, groupQuantifierCount++) - term.next = q - q - } - - Lexer.QUANT_STAR_R, Lexer.QUANT_PLUS_R, Lexer.QUANT_ALT_R -> { - lexemes.next() - val q = ReluctantGroupQuantifierSet(Quantifier.fromLexerToken(quant), term, last, quant, groupQuantifierCount++) - term.next = q - q - } - - Lexer.QUANT_PLUS_P, Lexer.QUANT_STAR_P, Lexer.QUANT_ALT_P -> { - lexemes.next() - PossessiveGroupQuantifierSet(Quantifier.fromLexerToken(quant), term, last, quant, groupQuantifierCount++) - } - - Lexer.QUANT_COMP -> { - val q = GroupQuantifierSet(lexemes.nextSpecial() as Quantifier, term, last, Lexer.QUANT_ALT, groupQuantifierCount++) - term.next = q - q - } - - Lexer.QUANT_COMP_R -> { - val q = ReluctantGroupQuantifierSet(lexemes.nextSpecial() as Quantifier, term, last, Lexer.QUANT_ALT, groupQuantifierCount++) - term.next = q - q - } - - Lexer.QUANT_COMP_P -> { - return PossessiveGroupQuantifierSet(lexemes.nextSpecial() as Quantifier, term, last, Lexer.QUANT_ALT, groupQuantifierCount++) - } - - else -> term } - } else { - val leaf: LeafSet = term - return when (quant) { - Lexer.QUANT_STAR, Lexer.QUANT_PLUS -> { - lexemes.next() - val q = LeafQuantifierSet(Quantifier.fromLexerToken(quant), leaf, last, quant) - leaf.next = q - q - } - Lexer.QUANT_STAR_R, Lexer.QUANT_PLUS_R -> { - lexemes.next() - val q = ReluctantLeafQuantifierSet(Quantifier.fromLexerToken(quant), leaf, last, quant) - leaf.next = q - q + Lexer.QUANT_STAR_R, Lexer.QUANT_PLUS_R, Lexer.QUANT_ALT_R, Lexer.QUANT_COMP_R -> { + val quantifier = quantifierFromLexerToken(quant) + when { + term is LeafSet -> + ReluctantLeafQuantifierSet(quantifier, term, last, quant) + term.consumesFixedLength -> + ReluctantFixedLengthQuantifierSet(quantifier, term, last, quant) + else -> + ReluctantGroupQuantifierSet(quantifier, term, last, quant, groupQuantifierCount++) } - - Lexer.QUANT_PLUS_P, Lexer.QUANT_STAR_P -> { - lexemes.next() - val q = PossessiveLeafQuantifierSet(Quantifier.fromLexerToken(quant), leaf, last, quant) - leaf.next = q - q - } - - Lexer.QUANT_ALT -> { - lexemes.next() - LeafQuantifierSet(Quantifier.altQuantifier, leaf, last, Lexer.QUANT_ALT) - } - - Lexer.QUANT_ALT_R -> { - lexemes.next() - ReluctantLeafQuantifierSet(Quantifier.altQuantifier, leaf, last, Lexer.QUANT_ALT_R) - } - - Lexer.QUANT_ALT_P -> { - lexemes.next() - PossessiveLeafQuantifierSet(Quantifier.altQuantifier, leaf, last, Lexer.QUANT_ALT_P) - } - - Lexer.QUANT_COMP -> { - LeafQuantifierSet(lexemes.nextSpecial() as Quantifier, leaf, last, Lexer.QUANT_COMP) - } - Lexer.QUANT_COMP_R -> { - ReluctantLeafQuantifierSet(lexemes.nextSpecial() as Quantifier, leaf, last, Lexer.QUANT_COMP_R) - } - Lexer.QUANT_COMP_P -> { - ReluctantLeafQuantifierSet(lexemes.nextSpecial() as Quantifier, leaf, last, Lexer.QUANT_COMP_P) - } - - else -> term } + + Lexer.QUANT_PLUS_P, Lexer.QUANT_STAR_P, Lexer.QUANT_ALT_P, Lexer.QUANT_COMP_P -> { + val quantifier = quantifierFromLexerToken(quant) + when { + term is LeafSet -> + PossessiveLeafQuantifierSet(quantifier, term, last, quant) + term.consumesFixedLength -> + PossessiveFixedLengthQuantifierSet(quantifier, term, last, quant) + else -> + PossessiveGroupQuantifierSet(quantifier, term, last, quant, groupQuantifierCount++) + } + } + + else -> term } } diff --git a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/AbstractSet.kt b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/AbstractSet.kt index 87cb05545df..afbb82781e2 100644 --- a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/AbstractSet.kt +++ b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/AbstractSet.kt @@ -108,6 +108,15 @@ internal abstract class AbstractSet(val type: Int = 0) { = (rightLimit downTo leftLimit).firstOrNull { index -> matches(index, testString, matchResult) >= 0 } ?: -1 + /** + * Returns `true` if this node consumes a constant number of characters and doesn't need backtracking to find a different match. + * Otherwise, returns `false`. + * + * This information is used to avoid recursion when matching a quantifier node with this inner node. + */ + open val consumesFixedLength: Boolean + get() = false + /** * Returns true, if this node has consumed any characters during * positive match attempt, for example node representing character always diff --git a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/CompositeRangeSet.kt b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/CompositeRangeSet.kt index c6b4b57a168..fb3e8ffbeb6 100644 --- a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/CompositeRangeSet.kt +++ b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/CompositeRangeSet.kt @@ -93,7 +93,7 @@ package kotlin.text.regex * from the parent range. This class represents the parent range split in such a manner. */ internal class CompositeRangeSet(/* range without surrogates */ val withoutSurrogates: AbstractSet, - /* range containing surrogates only */ val surrogates: AbstractSet) : SimpleSet() { + /* range containing surrogates only */ val surrogates: SurrogateRangeSet) : SimpleSet() { override var next: AbstractSet = dummyNext get() = field @@ -103,6 +103,12 @@ internal class CompositeRangeSet(/* range without surrogates */ val withoutSurro withoutSurrogates.next = next } + // This node consumes a single character starting at `startIndex`. + // A single surrogate char is consumed only if it is unpaired, see [SurrogateRangeSet]. + // Thus, for a given [testString] and [startIndex] a fixed amount of chars are consumed. + override val consumesFixedLength: Boolean + get() = true + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { var result = withoutSurrogates.matches(startIndex, testString, matchResult) if (result < 0) { diff --git a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/DecomposedCharSet.kt b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/DecomposedCharSet.kt index 349729dd17a..b37fc5bf27b 100644 --- a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/DecomposedCharSet.kt +++ b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/DecomposedCharSet.kt @@ -27,7 +27,8 @@ open internal class DecomposedCharSet( /** Decomposition of the Unicode codepoint */ private val decomposedChar: IntArray, /** Length of useful part of decomposedChar decomposedCharLength <= decomposedChar.length */ - private val decomposedCharLength: Int) : SimpleSet() { + private val decomposedCharLength: Int +) : SimpleSet() { /** Contains information about number of chars that were read for a codepoint last time */ private var readCharsForCodePoint = 1 @@ -42,6 +43,9 @@ open internal class DecomposedCharSet( return@lazy strBuff.toString() } + override val consumesFixedLength: Boolean + get() = true + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { var strIndex = startIndex val rightBound = testString.length diff --git a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/DotSet.kt b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/DotSet.kt index ec5174194e2..0b0ae6a03dd 100644 --- a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/DotSet.kt +++ b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/DotSet.kt @@ -28,6 +28,12 @@ package kotlin.text.regex internal class DotSet(val lt: AbstractLineTerminator, val matchLineTerminator: Boolean) : SimpleSet(AbstractSet.TYPE_DOTSET) { + // This node consumes any character. If the character is supplementary, this node consumes both surrogate chars representing it. + // Otherwise, consumes a single char. + // Thus, for a given [testString] and [startIndex] a fixed amount of chars are consumed. + override val consumesFixedLength: Boolean + get() = true + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { val rightBound = testString.length if (startIndex >= rightBound) { diff --git a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/FixedLengthQuantifierSet.kt b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/FixedLengthQuantifierSet.kt new file mode 100644 index 00000000000..0119797e996 --- /dev/null +++ b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/FixedLengthQuantifierSet.kt @@ -0,0 +1,62 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package kotlin.text.regex + +/** + * Greedy quantifier over constructions that consume a fixed number of characters. + */ +open internal class FixedLengthQuantifierSet( + val quantifier: Quantifier, + innerSet: AbstractSet, + next: AbstractSet, + type: Int +) : QuantifierSet(innerSet, next, type) { + + init { + require(innerSet.consumesFixedLength) + innerSet.next = FSet.possessiveFSet + } + + val min: Int get() = quantifier.min + val max: Int get() = quantifier.max + + override val consumesFixedLength: Boolean + get() = (min == max) + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var index = startIndex + val matches = mutableListOf() + + // Process occurrences between 0 and max. + while (max == Quantifier.INF || matches.size < max) { + val nextIndex = innerSet.matches(index, testString, matchResult) + if (nextIndex < 0) { + if (matches.size < min) { + return -1 + } else { + break + } + } + matches.add(index) + index = nextIndex + } + + // Roll back if the next node doesn't match the remaining string. + while (matches.size > min) { + val nextIndex = next.matches(index, testString, matchResult) + if (nextIndex >= 0) { + return nextIndex + } + index = matches.removeLast() + } + + return next.matches(index, testString, matchResult) + } + + override fun toString(): String { + return "${this::class}(innerSet = $innerSet, next = $next)" + } +} \ No newline at end of file diff --git a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/GroupQuantifierSet.kt b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/GroupQuantifierSet.kt index 95fecf1149c..61c49a67074 100644 --- a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/GroupQuantifierSet.kt +++ b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/GroupQuantifierSet.kt @@ -26,7 +26,6 @@ package kotlin.text.regex * Default quantifier over groups, in fact this type of quantifier is * generally used for constructions we cant identify number of characters they * consume. - */ open internal class GroupQuantifierSet( val quantifier: Quantifier, @@ -36,6 +35,11 @@ open internal class GroupQuantifierSet( val groupQuantifierIndex: Int // It's used to remember a number of the innerSet occurrences during the recursive search. ) : QuantifierSet(innerSet, next, type) { + init { + require(!innerSet.consumesFixedLength) + innerSet.next = this + } + val max: Int get() = quantifier.max val min: Int get() = quantifier.min diff --git a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/HangulDecomposedCharSet.kt b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/HangulDecomposedCharSet.kt index d805d62ae79..784affc9f13 100644 --- a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/HangulDecomposedCharSet.kt +++ b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/HangulDecomposedCharSet.kt @@ -38,7 +38,8 @@ internal class HangulDecomposedCharSet( * Length of useful part of decomposedChar * decomposedCharLength <= decomposedChar.length */ - private val decomposedCharLength: Int) : SimpleSet() { + private val decomposedCharLength: Int +) : SimpleSet() { /** * String representing syllable @@ -50,6 +51,9 @@ internal class HangulDecomposedCharSet( override val name: String get() = "decomposed Hangul syllable: $decomposedCharUTF16" + override val consumesFixedLength: Boolean + get() = true + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { var index = startIndex diff --git a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/LeafQuantifierSet.kt b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/LeafQuantifierSet.kt index 2aa103bc235..223900056db 100644 --- a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/LeafQuantifierSet.kt +++ b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/LeafQuantifierSet.kt @@ -37,10 +37,17 @@ open internal class LeafQuantifierSet(var quantifier: Quantifier, type: Int ) : QuantifierSet(innerSet, next, type) { + init { + innerSet.next = FSet.possessiveFSet + } + val leaf: LeafSet get() = super.innerSet as LeafSet val min: Int get() = quantifier.min val max: Int get() = quantifier.max + override val consumesFixedLength: Boolean + get() = (min == max) + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { var index = startIndex var occurrences = 0 diff --git a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/LeafSet.kt b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/LeafSet.kt index 26837e22ead..3f0badc8b4d 100644 --- a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/LeafSet.kt +++ b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/LeafSet.kt @@ -32,6 +32,9 @@ internal abstract class LeafSet : SimpleSet(AbstractSet.TYPE_LEAF) { /** Returns "shift", the number of accepted chars. Commonly internal function, but called by quantifiers. */ abstract fun accepts(startIndex: Int, testString: CharSequence): Int + override val consumesFixedLength: Boolean + get() = true + /** * Checks if we can enter this state and pass the control to the next one. * Return positive value if match succeeds, negative otherwise. diff --git a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/PossessiveFixedLengthQuantifierSet.kt b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/PossessiveFixedLengthQuantifierSet.kt new file mode 100644 index 00000000000..c0af8bbdba5 --- /dev/null +++ b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/PossessiveFixedLengthQuantifierSet.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package kotlin.text.regex + +/** + * Possessive version of the fixed length quantifier set. + */ +internal class PossessiveFixedLengthQuantifierSet( + quantifier: Quantifier, + innerSet: AbstractSet, + next: AbstractSet, + type: Int +) : FixedLengthQuantifierSet(quantifier, innerSet, next, type) { + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var index = startIndex + + // Process occurrences between 0 and max. + var occurrences = 0 + while (max == Quantifier.INF || occurrences < max) { + val nextIndex = innerSet.matches(index, testString, matchResult) + if (nextIndex < 0) { + break + } + index = nextIndex + occurrences += 1 + } + + return if (occurrences < min) -1 else next.matches(index, testString, matchResult) + } +} \ No newline at end of file diff --git a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/ReluctantFixedLengthQuantifierSet.kt b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/ReluctantFixedLengthQuantifierSet.kt new file mode 100644 index 00000000000..63929f648e6 --- /dev/null +++ b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/ReluctantFixedLengthQuantifierSet.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package kotlin.text.regex + +/** + * Reluctant version of the fixed length quantifier set. + */ +internal class ReluctantFixedLengthQuantifierSet( + quantifier: Quantifier, + innerSet: AbstractSet, + next: AbstractSet, + type: Int +) : FixedLengthQuantifierSet(quantifier, innerSet, next, type) { + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var index = startIndex + + // Process first min occurrences. + repeat(min) { + val nextIndex = innerSet.matches(index, testString, matchResult) + if (nextIndex < 0) { + return -1 + } + index = nextIndex + } + + // Process occurrences between min and max. + var occurrences = min + while (max == Quantifier.INF || occurrences < max) { + var nextIndex = next.matches(index, testString, matchResult) + if (nextIndex < 0) { + nextIndex = innerSet.matches(index, testString, matchResult) + if (nextIndex < 0) { + return -1 + } + index = nextIndex + occurrences += 1 + } else { + return nextIndex + } + } + + return next.matches(index, testString, matchResult) + } +} \ No newline at end of file diff --git a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/SupplementaryRangeSet.kt b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/SupplementaryRangeSet.kt index 6edc13f887e..40830ff4092 100644 --- a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/SupplementaryRangeSet.kt +++ b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/SupplementaryRangeSet.kt @@ -96,6 +96,12 @@ open internal class SupplementaryRangeSet(charClass: AbstractCharClass, val igno val chars = charClass.instance + // This node can consume a single char or a supplementary code point consisting of two surrogate chars. + // But this node can't match an unpaired surrogate char. + // Thus, for a given [testString] and [startIndex] a fixed amount of chars are consumed. + override val consumesFixedLength: Boolean + get() = true + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { val rightBound = testString.length if (startIndex >= rightBound) { diff --git a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/SurrogateRangeSet.kt b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/SurrogateRangeSet.kt index d3ba3796d7e..cdb7c92683b 100644 --- a/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/SurrogateRangeSet.kt +++ b/libraries/stdlib/native-wasm/src/kotlin/text/regex/sets/SurrogateRangeSet.kt @@ -87,7 +87,7 @@ package kotlin.text.regex -/* +/** * This class is a range that contains only surrogate characters. */ internal class SurrogateRangeSet(surrChars: AbstractCharClass) : RangeSet(surrChars) { diff --git a/libraries/stdlib/native-wasm/test/harmony_regex/FixedLengthQuantifierTest.kt b/libraries/stdlib/native-wasm/test/harmony_regex/FixedLengthQuantifierTest.kt new file mode 100644 index 00000000000..2c966bf2599 --- /dev/null +++ b/libraries/stdlib/native-wasm/test/harmony_regex/FixedLengthQuantifierTest.kt @@ -0,0 +1,174 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package test.text.harmony_regex + +import kotlin.text.* +import kotlin.test.* + +class FixedLengthQuantifierTest { + companion object { + private val quantifierMatchCount = 100_000 + private val compositeMin = 50_000 + private val compositeMax = 150_000 + + private val input = "a".repeat(quantifierMatchCount) + private val inputDescription = "\"a\".repeat($quantifierMatchCount)" + } + + private fun testMatches(regex: Regex, input: String, inputDescription: String, expected: Boolean = true) { + val message = "$regex should ${if (expected) "" else "not " }match $inputDescription" + assertEquals(expected, regex.matches(input), message) + } + + @Test + fun fixedLengthQualifierGreedy() { + val plusRegex = Regex("[^\\s]+") + testMatches(plusRegex, input, inputDescription) + + val starRegex = Regex("[^\\s]*") + testMatches(starRegex, input, inputDescription) + + Regex("[^\\s\\d]{$compositeMin,$compositeMax}").let { compositeRegex -> + testMatches(compositeRegex, input, inputDescription) + } + Regex("[^\\s\\d]{$compositeMin,$quantifierMatchCount}").let { compositeRegex -> + testMatches(compositeRegex, input, inputDescription) + } + Regex("[^\\s\\d]{$compositeMin,${quantifierMatchCount - 1}}").let { compositeRegex -> + testMatches(compositeRegex, input, inputDescription, expected = false) + } + Regex("[^\\s\\d]{$quantifierMatchCount,$compositeMax}").let { compositeRegex -> + testMatches(compositeRegex, input, inputDescription) + } + Regex("[^\\s\\d]{${quantifierMatchCount + 1},$compositeMax}").let { compositeRegex -> + testMatches(compositeRegex, input, inputDescription, expected = false) + } + } + + @Test + fun fixedLengthQualifierReluctant() { + val plusRegex = Regex(".+?") + testMatches(plusRegex, input, inputDescription) + + val starRegex = Regex(".*?") + testMatches(starRegex, input, inputDescription) + + Regex(".{$compositeMin,$compositeMax}?").let { compositeRegex -> + testMatches(compositeRegex, input, inputDescription) + } + Regex(".{$compositeMin,$quantifierMatchCount}?").let { compositeRegex -> + testMatches(compositeRegex, input, inputDescription) + } + Regex(".{$compositeMin,${quantifierMatchCount - 1}}?").let { compositeRegex -> + testMatches(compositeRegex, input, inputDescription, expected = false) + } + Regex(".{$quantifierMatchCount,$compositeMax}?").let { compositeRegex -> + testMatches(compositeRegex, input, inputDescription) + } + Regex(".{${quantifierMatchCount + 1},$compositeMax}?").let { compositeRegex -> + testMatches(compositeRegex, input, inputDescription, expected = false) + } + } + + @Test + fun fixedLengthQualifierPossesive() { + val plusRegex = Regex("\\p{Ll}++") + testMatches(plusRegex, input, inputDescription) + + val starRegex = Regex("\\p{Ll}*+") + testMatches(starRegex, input, inputDescription) + + Regex("\\p{Ll}{$compositeMin,$compositeMax}+").let { compositeRegex -> + testMatches(compositeRegex, input, inputDescription) + } + Regex("\\p{Ll}{$compositeMin,$quantifierMatchCount}+").let { compositeRegex -> + testMatches(compositeRegex, input, inputDescription) + } + Regex("\\p{Ll}{$compositeMin,${quantifierMatchCount - 1}}+").let { compositeRegex -> + testMatches(compositeRegex, input, inputDescription, expected = false) + } + Regex("\\p{Ll}{$quantifierMatchCount,$compositeMax}+").let { compositeRegex -> + testMatches(compositeRegex, input, inputDescription) + } + Regex("\\p{Ll}{${quantifierMatchCount + 1},$compositeMax}+").let { compositeRegex -> + testMatches(compositeRegex, input, inputDescription, expected = false) + } + } + + @Test + fun leafQuantifierGreedy() { + val plusRegex = Regex("a+") + testMatches(plusRegex, input, inputDescription) + + val starRegex = Regex("a*") + testMatches(starRegex, input, inputDescription) + + val compositeRegex = Regex("a{$compositeMin,$compositeMax}") + testMatches(compositeRegex, input, inputDescription) + } + + @Test + fun kt46211_space() { + val regex = "(https?|ftp)://[^\\s/$.?#].[^\\s]*".toRegex(RegexOption.IGNORE_CASE) + val link = "http://" + input + testMatches(regex, link, "\"http://\" + $inputDescription") + } + + @Test + fun kt46211() { + val regex = Regex("[a]+") + val output = regex.replace(input, "") + assertEquals("", output) + } + + @Test + fun kt53352() { + val test = input + "b c" + val regex = """(.*?b.*?c)""".toRegex() + val res = regex.find(test)!! + assertEquals(test, res.groupValues[1]) + } + + @Test + fun kt35508() { + val doesNotWork = """=== EREIGNISLISTE ====== +""" + "\u001b" + """Kn +BEGINN 28.06 13:25 +EREIGNISSE 62 +50 5 28.06 1325 +3402 28.06 1325 +3412 28.06 1325 +63 3 28.06 1325 +63 0 28.06 1325 +EE06 28.06 1325 +EE06 28.06 1322 +EE07 28.06 1322 +63 3 28.06 1322 +EE06 28.06 1322 +EE07 28.06 1322 +63 3 28.06 1322 +63 3 28.06 1322 +63 3 28.06 1323 +63 3 28.06 1500 +50 4 28.06 1500 +50 5 30.06 1226 +3402 30.06 1226 +3412 30.06 1226 +50 4 30.06 1227 +50 5 30.06 1228 +3402 30.06 1228""" + + val regex = Regex("(\\x1b\\w[\\s\\S]{1,2})([\\s\\S]+?(?=\\x1b\\w[\\s\\S]{1,2}|\$))") + + fun regexTest(content: String): List { + return regex.findAll(content).map { + it.groupValues[1] + }.toList() + } + + assertEquals("\u001BKn\n", regexTest(doesNotWork).single()) + } +} \ No newline at end of file diff --git a/libraries/stdlib/native-wasm/test/harmony_regex/MatchResultTest.kt b/libraries/stdlib/native-wasm/test/harmony_regex/MatchResultTest.kt index 46b070a8c97..4623e292b43 100644 --- a/libraries/stdlib/native-wasm/test/harmony_regex/MatchResultTest.kt +++ b/libraries/stdlib/native-wasm/test/harmony_regex/MatchResultTest.kt @@ -452,23 +452,27 @@ class MatchResultTest { * Regression test for https://github.com/JetBrains/kotlin-native/issues/2297 */ @Test fun test2297() { - assertTrue(Regex("^(:[0-5]?[0-9])+$").matches(":20:30")) - assertTrue(Regex("(.{1,}){2}").matches("aa")) + fun testMatches(pattern: String, input: String) { + assertTrue(Regex(pattern).matches(input), "\"$pattern\" should match \"$input\"") + } - assertTrue(Regex("(.+b)+").matches("0b0b")) - assertTrue(Regex("(.+?b)+").matches("0b0b")) - assertTrue(Regex("(.?b)+").matches("0b0b")) - assertTrue(Regex("(.??b)+").matches("0b0b")) - assertTrue(Regex("(.*b)+").matches("0b0b")) - assertTrue(Regex("(.*?b)+").matches("0b0b")) - assertTrue(Regex("(.{1,2}b)+").matches("0b00b")) - assertTrue(Regex("(.{1,2}?b)+").matches("0b00b")) + testMatches("^(:[0-5]?[0-9])+$", input = ":20:30") + testMatches("(.{1,}){2}", input = "aa") - assertTrue(Regex("([0]?[0]?)+").matches("0000")) - assertTrue(Regex("([0]?[0]?b)+").matches("00b00b")) - assertTrue(Regex("((b{2}){3})+").matches("bbbbbbbbbbbb")) + testMatches("(.+b)+", input = "0b0b") + testMatches("(.+?b)+", input = "0b0b") + testMatches("(.?b)+", input = "0b0b") + testMatches("(.??b)+", input = "0b0b") + testMatches("(.*b)+", input = "0b0b") + testMatches("(.*?b)+", input = "0b0b") + testMatches("(.{1,2}b)+", input = "0b00b") + testMatches("(.{1,2}?b)+", input = "0b00b") - assertTrue(Regex("[^a]").matches("b")) + testMatches("([0]?[0]?)+", input = "0000") + testMatches("([0]?[0]?b)+", input = "00b00b") + testMatches("((b{2}){3})+", input = "bbbbbbbbbbbb") + + testMatches("[^a]", input = "b") } @Test fun kt28158() {