From 22aead3d18dbd392729eb94b29b02567eb75059a Mon Sep 17 00:00:00 2001 From: Ilya Matveev Date: Fri, 16 Jun 2017 18:47:40 +0700 Subject: [PATCH] regex: Perform char decomposition on C++ side --- runtime/src/main/cpp/Regex.cpp | 60 ++++++++++-- runtime/src/main/kotlin/kotlin/text/Regex.kt | 14 +-- .../src/main/kotlin/kotlin/text/Strings.kt | 2 +- .../main/kotlin/kotlin/text/regex/Lexer.kt | 98 +++++-------------- .../kotlin/text/regex/MatchResultImpl.kt | 6 +- .../text/regex/sets/DecomposedCharSet.kt | 42 +++----- .../text/regex/sets/LeafQuantifierSet.kt | 2 +- .../regex/sets/PossessiveLeafQuantifierSet.kt | 2 +- .../regex/sets/ReluctantLeafQuantifierSet.kt | 2 +- 9 files changed, 103 insertions(+), 125 deletions(-) diff --git a/runtime/src/main/cpp/Regex.cpp b/runtime/src/main/cpp/Regex.cpp index b6e65d15bee..4ae4a72c794 100644 --- a/runtime/src/main/cpp/Regex.cpp +++ b/runtime/src/main/cpp/Regex.cpp @@ -1,3 +1,4 @@ +#include #include "Types.h" #include "KString.h" #include "Natives.h" @@ -560,21 +561,64 @@ KBoolean Kotlin_text_regex_hasSingleCodepointDecompositionInternal(KInt ch) { return singleDecompositions[index] == ch; } -OBJ_GETTER(Kotlin_text_getDecompositionInternal, KInt ch) { - // TODO: Move all uses of decomposition table into C++ code. - int index = binarySearchRange(decompositionKeys, ARRAY_SIZE(decompositionKeys), ch); - if (decompositionKeys[index] != ch) { +const Decomposition* getDecomposition(KInt codePoint) { + int index = binarySearchRange(decompositionKeys, ARRAY_SIZE(decompositionKeys), codePoint); + if (decompositionKeys[index] != codePoint) { return nullptr; } - const Decomposition& decomposition = decompositionValues[index]; - ArrayHeader* result = AllocArrayInstance(theIntArrayTypeInfo, decomposition.length, OBJ_RESULT)->array(); + return &decompositionValues[index]; +} + +OBJ_GETTER(Kotlin_text_regex_getDecompositionInternal, KInt ch) { + const Decomposition* decomposition = getDecomposition(ch); + if (decomposition == nullptr) { + return nullptr; + } + ArrayHeader* result = AllocArrayInstance(theIntArrayTypeInfo, decomposition->length, OBJ_RESULT)->array(); KInt* resultRaw = IntArrayAddressOfElementAt(result, 0); - for (int i = 0; i < decomposition.length; i++) { - *resultRaw++ = decomposition.array[i]; + for (int i = 0; i < decomposition->length; i++) { + *resultRaw++ = decomposition->array[i]; } RETURN_OBJ(result->obj()); } +KInt Kotlin_text_regex_decomposeString(ArrayHeader* inputCodePoints, KInt inputLength, ArrayHeader* outputCodePoints) { + RuntimeAssert(inputCodePoints->type_info() == theIntArrayTypeInfo, "Must use an Int array"); + RuntimeAssert(outputCodePoints->type_info() == theIntArrayTypeInfo, "Must use an Int array"); + RuntimeAssert(inputLength >= 0, "Input length must be >= 0"); + if (inputLength == 0) { + return 0; + } + + int outputLength = 0; + const KInt* inputArray = IntArrayAddressOfElementAt(inputCodePoints, 0); + KInt* outputArray = IntArrayAddressOfElementAt(outputCodePoints, 0); + for (int i = 0; i < inputLength; i++) { + const Decomposition* decomposition = getDecomposition(inputArray[i]); + if (decomposition == nullptr) { + outputArray[outputLength++] = inputArray[i]; + } else { + memcpy(outputArray + outputLength, decomposition->array, decomposition->length * sizeof(KInt)); + outputLength+=decomposition->length; + } + } + return outputLength; +} + +KInt Kotlin_text_regex_decomposeCodePoint(KInt codePoint, ArrayHeader* outputCodePoints, KInt fromIndex) { + RuntimeAssert(outputCodePoints->type_info() == theIntArrayTypeInfo, "Must be an Int array"); + RuntimeAssert(fromIndex >= 0 && fromIndex < outputCodePoints->count_, "Start index must be >= 0 and < array size"); + KInt* rawResult = IntArrayAddressOfElementAt(outputCodePoints, fromIndex); + const Decomposition* decomposition = getDecomposition(codePoint); + if (decomposition == nullptr) { + *rawResult = codePoint; + return 1; + } else { + memcpy(rawResult, decomposition->array, decomposition->length * sizeof(KInt)); + return decomposition->length; + } +} + } // extern "C" diff --git a/runtime/src/main/kotlin/kotlin/text/Regex.kt b/runtime/src/main/kotlin/kotlin/text/Regex.kt index 1a1a713be28..b963fc656d2 100644 --- a/runtime/src/main/kotlin/kotlin/text/Regex.kt +++ b/runtime/src/main/kotlin/kotlin/text/Regex.kt @@ -113,7 +113,6 @@ class Regex internal constructor(internal val nativePattern: Pattern) { companion object { /** Returns a literal regex for the specified [literal] string. */ - // TODO: Uncomment for native fun fromLiteral(literal: String): Regex = Regex(literal, RegexOption.LITERAL) /** Returns a literal pattern for the specified [literal] string. */ @@ -159,7 +158,6 @@ class Regex internal constructor(internal val nativePattern: Pattern) { infix fun matches(input: CharSequence): Boolean = doMatch(input, Mode.MATCH) != null /** Indicates whether the regular expression can find at least one match in the specified [input]. */ - // TODO: Looks like we don't need Mode anymore. fun containsMatchIn(input: CharSequence): Boolean = find(input) != null /** @@ -172,7 +170,6 @@ class Regex internal constructor(internal val nativePattern: Pattern) { if (startIndex < 0 || startIndex > input.length) { throw IndexOutOfBoundsException() // TODO: Add a message. } - // TODO: reuse the match result? val matchResult = MatchResultImpl(input, this) matchResult.mode = Mode.FIND matchResult.startIndex = startIndex @@ -181,14 +178,13 @@ class Regex internal constructor(internal val nativePattern: Pattern) { matchResult.finalizeMatch() return matchResult } else { - /*matchResult.hitEnd = true - matchResult.startIndex = -1*/ return null } } /** - * Returns a sequence of all occurrences of a regular expression within the [input] string, beginning at the specified [startIndex]. + * Returns a sequence of all occurrences of a regular expression within the [input] string, + * beginning at the specified [startIndex]. */ fun findAll(input: CharSequence, startIndex: Int = 0): Sequence = generateSequence({ find(input, startIndex) }, MatchResult::next) @@ -233,9 +229,10 @@ class Regex internal constructor(internal val nativePattern: Pattern) { } /** - * Replaces all occurrences of this regular expression in the specified [input] string with specified [replacement] expression. + * Replaces all occurrences of this regular expression in the specified [input] string with + * specified [replacement] expression. * - * @param replacement A replacement expression that can include substitutions. See [Matcher.appendReplacement] for details. + * @param replacement A replacement expression that can include substitutions. */ fun replace(input: CharSequence, replacement: String): String = replace(input) { match -> processReplacement(match, replacement) } @@ -280,7 +277,6 @@ class Regex internal constructor(internal val nativePattern: Pattern) { * @param limit Non-negative value specifying the maximum number of substrings the string can be split to. * Zero by default means no limit is set. */ - // TODO: replace all argument checks with require function. fun split(input: CharSequence, limit: Int = 0): List { require(limit >= 0, { "Limit must be non-negative, but was $limit." } ) if (input.isEmpty()) { diff --git a/runtime/src/main/kotlin/kotlin/text/Strings.kt b/runtime/src/main/kotlin/kotlin/text/Strings.kt index 516e01c00ca..da96a7585c0 100644 --- a/runtime/src/main/kotlin/kotlin/text/Strings.kt +++ b/runtime/src/main/kotlin/kotlin/text/Strings.kt @@ -671,7 +671,7 @@ public inline fun CharSequence.replaceFirst(regex: Regex, replacement: String): * Returns `true` if this char sequence matches the given regular expression. */ @kotlin.internal.InlineOnly -public inline fun CharSequence.matches(regex: Regex): Boolean = regex.matches(this) +public infix inline fun CharSequence.matches(regex: Regex): Boolean = regex.matches(this) /** * Implementation of [regionMatches] for CharSequences. diff --git a/runtime/src/main/kotlin/kotlin/text/regex/Lexer.kt b/runtime/src/main/kotlin/kotlin/text/regex/Lexer.kt index e9dcc7b8508..25c383a3838 100644 --- a/runtime/src/main/kotlin/kotlin/text/regex/Lexer.kt +++ b/runtime/src/main/kotlin/kotlin/text/regex/Lexer.kt @@ -28,9 +28,15 @@ external private fun getCanonicalClassInternal(ch: Int): Int external private fun hasSingleCodepointDecompositionInternal(ch: Int): Boolean /** Returns a decomposition for a given codepoint. */ -@SymbolName("Kotlin_text_getDecompositionInternal") +@SymbolName("Kotlin_text_regex_getDecompositionInternal") external private fun getDecompositionInternal(ch: Int): IntArray? +/** + * Decomposes the given string represented as an array of codepoints. Saves the decomposition into [outputCodepoints] array. + * Returns the length of the decomposition. + */ +@SymbolName("Kotlin_text_regex_decomposeString") +external private fun decomposeString(inputCodePoints: IntArray, inputLength: Int, outputCodePoints: IntArray): Int // ============================================================================================================= /** @@ -792,13 +798,8 @@ internal class Lexer(val patternString: String, flags: Int) { // ============================================================================================================= /** - * Normalize given expression. - - * @param input - expression to normalize - * * - * @return normalized expression. + * Normalize given string. */ - // TODO: Implement on the C++ side. fun normalize(input: String): String { val inputChars = input.toCharArray() val inputLength = inputChars.size @@ -824,45 +825,22 @@ internal class Lexer(val patternString: String, flags: Int) { //result of canonical decomposition of input in UTF-16 encoding val result = StringBuilder() - run { - var i = 0 - while (i < inputLength) { - ch = input.codePointAt(i) - inputCodePoints[inputCodePointsIndex++] = ch - i += if (Char.isSupplementaryCodePoint(ch)) 2 else 1 - } + var i = 0 + while (i < inputLength) { + ch = input.codePointAt(i) + inputCodePoints[inputCodePointsIndex++] = ch + i += if (Char.isSupplementaryCodePoint(ch)) 2 else 1 } - /* - * Canonical decomposition based on mappings in decompTable - */ - for (i in 0..inputCodePointsIndex - 1) { - ch = inputCodePoints[i] + // Canonical decomposition based on mappings in decomposition table. + resCodePointsIndex = decomposeString(inputCodePoints, inputCodePointsIndex, resCodePoints) - decomp = Lexer.getDecomposition(ch) - if (decomp == null) { - resCodePoints[resCodePointsIndex++] = ch - } else { - val curSymbDecompLength = decomp.size - - for (j in 0..curSymbDecompLength - 1) { - resCodePoints[resCodePointsIndex++] = decomp[j] - } - } - } - - /* - * Canonical ordering. - * See http://www.unicode.org/reports/tr15/#Decomposition for - * details - */ + // Canonical ordering. + // See http://www.unicode.org/reports/tr15/#Decomposition for details resCodePoints = Lexer.getCanonicalOrder(resCodePoints, resCodePointsIndex) - /* - * Decomposition for Hangul syllables. - * See http://www.unicode.org/reports/tr15/#Hangul for - * details - */ + // Decomposition for Hangul syllables. + // See http://www.unicode.org/reports/tr15/#Hangul for details decompHangul = IntArray(resCodePoints.size) for (i in 0..resCodePointsIndex - 1) { @@ -872,11 +850,7 @@ internal class Lexer(val patternString: String, flags: Int) { if (decomp == null) { decompHangul[decompHangulIndex++] = curSymb } else { - - /* - * Note that Hangul decompositions have length that is - * equal 2 or 3. - */ + // Note that Hangul decompositions have length that is equal 2 or 3. decompHangul[decompHangulIndex++] = decomp[0] decompHangul[decompHangulIndex++] = decomp[1] if (decomp.size == 3) { @@ -885,9 +859,7 @@ internal class Lexer(val patternString: String, flags: Int) { } } - /* - * Translating into UTF-16 encoding - */ + // Translating into UTF-16 encoding for (i in 0..decompHangulIndex - 1) { result.append(Char.toChars(decompHangul[i])) } @@ -896,17 +868,8 @@ internal class Lexer(val patternString: String, flags: Int) { } /** - * Rearrange codepoints according - * to canonical order. - - * @param inputInts - array that contains Unicode codepoints - * * - * @param length - index of last Unicode codepoint plus 1 - * * - * * - * @return array that contains rearranged codepoints. + * Rearrange codepoints in [inputInts] according to canonical order. Return an array with rearranged codepoints. */ - // TODO: C++ side! fun getCanonicalOrder(inputInts: IntArray, length: Int): IntArray { val inputLength = if (length < inputInts.size) length @@ -914,12 +877,9 @@ internal class Lexer(val patternString: String, flags: Int) { inputInts.size /* - * Simple bubble-sort algorithm. - * Note that many codepoints have 0 - * canonical class, so this algorithm works - * almost lineary in overwhelming majority - * of cases. This is due to specific of Unicode - * combining classes and codepoints. + * Simple bubble-sort algorithm. Note that many codepoints have 0 canonical class, so this algorithm works + * almost lineary in overwhelming majority of cases. This is due to specific of Unicode combining + * classes and codepoints. */ for (i in 1..inputLength - 1) { var j = i - 1 @@ -951,14 +911,8 @@ internal class Lexer(val patternString: String, flags: Int) { /** * Gets decomposition for given Hangul syllable. * This is an implementation of Hangul decomposition algorithm - * according to http://www.unicode.org/versions/Unicode4.0.0/ch03.pdf - * "3.12 Conjoining Jamo Behavior". - - * @param ch - given Hangul syllable - * * - * @return canonical decomposition of currentChar. + * according to http://www.unicode.org/versions/Unicode4.0.0/ch03.pdf "3.12 Conjoining Jamo Behavior". */ - // TODO: C++ fun getHangulDecomposition(ch: Int): IntArray? { val SIndex = ch - SBase diff --git a/runtime/src/main/kotlin/kotlin/text/regex/MatchResultImpl.kt b/runtime/src/main/kotlin/kotlin/text/regex/MatchResultImpl.kt index e06e638d160..ca6e69b2d69 100644 --- a/runtime/src/main/kotlin/kotlin/text/regex/MatchResultImpl.kt +++ b/runtime/src/main/kotlin/kotlin/text/regex/MatchResultImpl.kt @@ -116,13 +116,13 @@ constructor (internal val input: CharSequence, override fun next(): MatchResult? { var nextStart = range.endInclusive + 1 - if (nextStart == input.length) { - return null - } // If the current match is empty - shift by 1. if (nextStart == range.start) { nextStart++ } + if (nextStart > input.length) { + return null + } return regex.find(input, nextStart) } // ================================================================================================================= diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/DecomposedCharSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/DecomposedCharSet.kt index 9a9b2149b39..15c907ba5b2 100644 --- a/runtime/src/main/kotlin/kotlin/text/regex/sets/DecomposedCharSet.kt +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/DecomposedCharSet.kt @@ -17,6 +17,13 @@ package kotlin.text.regex +/** + * Decomposes the given codepoint. Saves the decomposition into [outputCodepoints] array starting with [fromIndex]. + * Returns the length of the decomposition. + */ +@SymbolName("Kotlin_text_regex_decomposeCodePoint") +external private fun decomposeCodePoint(codePoint: Int, outputCodePoints: IntArray, fromIndex: Int): Int + /** Represents canonical decomposition of Unicode character. Is used when CANON_EQ flag of Pattern class is specified. */ open internal class DecomposedCharSet( /** Decomposition of the Unicode codepoint */ @@ -48,19 +55,11 @@ open internal class DecomposedCharSet( // We read testString and decompose it gradually to compare with this decomposedChar at position strIndex var curChar = codePointAt(strIndex, testString, rightBound) strIndex += readCharsForCodePoint - var decomposedCurrentCodePoint: IntArray? = Lexer.getDecomposition(curChar) var readCodePoints = 0 var i = 0 // All decompositions have length that is less or equal Lexer.MAX_DECOMPOSITION_LENGTH - var decomposedCodePoint: IntArray - if (decomposedCurrentCodePoint == null) { - decomposedCodePoint = IntArray(Lexer.MAX_DECOMPOSITION_LENGTH) - decomposedCodePoint[readCodePoints++] = curChar - } else { - i = decomposedCurrentCodePoint.size - decomposedCodePoint = decomposedCurrentCodePoint.copyOf(Lexer.MAX_DECOMPOSITION_LENGTH) - readCodePoints += i - } + var decomposedCodePoint: IntArray = IntArray(Lexer.MAX_DECOMPOSITION_LENGTH) + readCodePoints += decomposeCodePoint(curChar, decomposedCodePoint, readCodePoints) if (strIndex < rightBound) { curChar = codePointAt(strIndex, testString, rightBound) @@ -71,26 +70,11 @@ open internal class DecomposedCharSet( if (!Lexer.hasDecompositionNonNullCanClass(curChar)) { decomposedCodePoint[readCodePoints++] = curChar } else { - /* - * A few codepoints have decompositions and non null - * canonical classes, we have to take them into - * consideration, but general rule is: - * if canonical class != 0 then no decomposition + * A few codepoints have decompositions and non null canonical classes, we have to take them into + * consideration, but general rule is: if canonical class != 0 then no decomposition */ - decomposedCurrentCodePoint = Lexer.getDecomposition(curChar) - - /* - * Length of such decomposition is 1 or 2. See UnicodeData file - * http://www.unicode.org/Public/4.0-Update/UnicodeData-4.0.0.txt - */ - // hasDecompositionNonNullCanClass(curChar) == true, so decomposedCurrentCodePoint != null. - if (decomposedCurrentCodePoint!!.size == 2) { - decomposedCodePoint[readCodePoints++] = decomposedCurrentCodePoint[0] - decomposedCodePoint[readCodePoints++] = decomposedCurrentCodePoint[1] - } else { - decomposedCodePoint[readCodePoints++] = decomposedCurrentCodePoint[0] - } + readCodePoints += decomposeCodePoint(curChar, decomposedCodePoint, readCodePoints) } strIndex += readCharsForCodePoint @@ -133,7 +117,7 @@ open internal class DecomposedCharSet( } override val name: String - get() = "decomposed char: $decomposedChar" + get() = "decomposed char: $decomposedChar" /** Reads Unicode codepoint from [testString] starting from [strIndex] until [rightBound]. */ fun codePointAt(strIndex: Int, testString: CharSequence, rightBound: Int): Int { diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/LeafQuantifierSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/LeafQuantifierSet.kt index bc99d807300..e9a2872337d 100644 --- a/runtime/src/main/kotlin/kotlin/text/regex/sets/LeafQuantifierSet.kt +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/LeafQuantifierSet.kt @@ -57,7 +57,7 @@ open internal class LeafQuantifierSet(var quantifier: Quantifier, } // Process occurrences between min and max. - while ((max < 0 || occurrences < max) && index + leaf.charCount <= testString.length) { + while ((max == Quantifier.INF || occurrences < max) && index + leaf.charCount <= testString.length) { val shift = leaf.accepts(index, testString) if (shift < 1) { break diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/PossessiveLeafQuantifierSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/PossessiveLeafQuantifierSet.kt index 5b3cfd37a34..dd80615e637 100644 --- a/runtime/src/main/kotlin/kotlin/text/regex/sets/PossessiveLeafQuantifierSet.kt +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/PossessiveLeafQuantifierSet.kt @@ -49,7 +49,7 @@ internal class PossessiveLeafQuantifierSet( occurrences++ } - while ((max >= 0 || occurrences < max) && index + leaf.charCount <= testString.length) { + while ((max == Quantifier.INF || occurrences < max) && index + leaf.charCount <= testString.length) { val shift = leaf.accepts(index, testString) if (shift < 1) { break diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/ReluctantLeafQuantifierSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/ReluctantLeafQuantifierSet.kt index 027f9f1278f..fa48a3c5804 100644 --- a/runtime/src/main/kotlin/kotlin/text/regex/sets/ReluctantLeafQuantifierSet.kt +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/ReluctantLeafQuantifierSet.kt @@ -62,7 +62,7 @@ internal class ReluctantLeafQuantifierSet( occurrences++ } - } while (shift >= 1 && occurrences <= max) + } while (shift >= 1 && (max == Quantifier.INF || occurrences <= max)) return -1 }