regex: Perform char decomposition on C++ side

runtime/src/main/cpp/Regex.cpp

@@ -1,3 +1,4 @@
+#include <cstring>
 #include "Types.h"
 #include "KString.h"
 #include "Natives.h"
@@ -560,21 +561,64 @@ KBoolean Kotlin_text_regex_hasSingleCodepointDecompositionInternal(KInt ch) {
   return singleDecompositions[index] == ch;
 }
 
-OBJ_GETTER(Kotlin_text_getDecompositionInternal, KInt ch) {
+const Decomposition* getDecomposition(KInt codePoint) {
-  // TODO: Move all uses of decomposition table into C++ code.
+  int index = binarySearchRange(decompositionKeys, ARRAY_SIZE(decompositionKeys), codePoint);
-  int index = binarySearchRange(decompositionKeys, ARRAY_SIZE(decompositionKeys), ch);
+  if (decompositionKeys[index] != codePoint) {
-  if (decompositionKeys[index] != ch) {
     return nullptr;
   }
-  const Decomposition& decomposition = decompositionValues[index];
+  return &decompositionValues[index];
-  ArrayHeader* result = AllocArrayInstance(theIntArrayTypeInfo, decomposition.length, OBJ_RESULT)->array();
+}
+
+OBJ_GETTER(Kotlin_text_regex_getDecompositionInternal, KInt ch) {
+  const Decomposition* decomposition = getDecomposition(ch);
+  if (decomposition == nullptr) {
+    return nullptr;
+  }
+  ArrayHeader* result = AllocArrayInstance(theIntArrayTypeInfo, decomposition->length, OBJ_RESULT)->array();
   KInt* resultRaw = IntArrayAddressOfElementAt(result, 0);
-  for (int i = 0; i < decomposition.length; i++) {
+  for (int i = 0; i < decomposition->length; i++) {
-    *resultRaw++ = decomposition.array[i];
+    *resultRaw++ = decomposition->array[i];
   }
   RETURN_OBJ(result->obj());
 }
 
+KInt Kotlin_text_regex_decomposeString(ArrayHeader* inputCodePoints, KInt inputLength, ArrayHeader* outputCodePoints) {
+  RuntimeAssert(inputCodePoints->type_info() == theIntArrayTypeInfo, "Must use an Int array");
+  RuntimeAssert(outputCodePoints->type_info() == theIntArrayTypeInfo, "Must use an Int array");
+  RuntimeAssert(inputLength >= 0, "Input length must be >= 0");
+  if (inputLength == 0) {
+    return 0;
+  }
+
+  int outputLength = 0;
+  const KInt* inputArray = IntArrayAddressOfElementAt(inputCodePoints, 0);
+  KInt* outputArray = IntArrayAddressOfElementAt(outputCodePoints, 0);
+  for (int i = 0; i < inputLength; i++) {
+    const Decomposition* decomposition = getDecomposition(inputArray[i]);
+    if (decomposition == nullptr) {
+      outputArray[outputLength++] = inputArray[i];
+    } else {
+      memcpy(outputArray + outputLength, decomposition->array, decomposition->length * sizeof(KInt));
+      outputLength+=decomposition->length;
+    }
+  }
+  return outputLength;
+}
+
+KInt Kotlin_text_regex_decomposeCodePoint(KInt codePoint, ArrayHeader* outputCodePoints, KInt fromIndex) {
+  RuntimeAssert(outputCodePoints->type_info() == theIntArrayTypeInfo, "Must be an Int array");
+  RuntimeAssert(fromIndex >= 0 && fromIndex < outputCodePoints->count_, "Start index must be >= 0 and < array size");
+  KInt* rawResult = IntArrayAddressOfElementAt(outputCodePoints, fromIndex);
+  const Decomposition* decomposition = getDecomposition(codePoint);
+  if (decomposition == nullptr) {
+    *rawResult = codePoint;
+    return 1;
+  } else {
+    memcpy(rawResult, decomposition->array, decomposition->length * sizeof(KInt));
+    return decomposition->length;
+  }
+}
+
 } // extern "C"
 
 

runtime/src/main/kotlin/kotlin/text/Regex.kt

@@ -113,7 +113,6 @@ class Regex internal constructor(internal val nativePattern: Pattern) {
 
     companion object {
         /** Returns a literal regex for the specified [literal] string. */
-        // TODO: Uncomment for native
         fun fromLiteral(literal: String): Regex = Regex(literal, RegexOption.LITERAL)
 
         /** Returns a literal pattern for the specified [literal] string. */
@@ -159,7 +158,6 @@ class Regex internal constructor(internal val nativePattern: Pattern) {
     infix fun matches(input: CharSequence): Boolean = doMatch(input, Mode.MATCH) != null
 
     /** Indicates whether the regular expression can find at least one match in the specified [input]. */
-    // TODO: Looks like we don't need Mode anymore.
     fun containsMatchIn(input: CharSequence): Boolean = find(input) != null
 
     /**
@@ -172,7 +170,6 @@ class Regex internal constructor(internal val nativePattern: Pattern) {
         if (startIndex < 0 || startIndex > input.length) {
             throw IndexOutOfBoundsException() // TODO: Add a message.
         }
-        // TODO: reuse the match result?
         val matchResult = MatchResultImpl(input, this)
         matchResult.mode = Mode.FIND
         matchResult.startIndex = startIndex
@@ -181,14 +178,13 @@ class Regex internal constructor(internal val nativePattern: Pattern) {
             matchResult.finalizeMatch()
             return matchResult
         } else {
-            /*matchResult.hitEnd = true
-            matchResult.startIndex = -1*/
             return null
         }
     }
 
     /**
-     * Returns a sequence of all occurrences of a regular expression within the [input] string, beginning at the specified [startIndex].
+     * Returns a sequence of all occurrences of a regular expression within the [input] string,
+     * beginning at the specified [startIndex].
      */
     fun findAll(input: CharSequence, startIndex: Int = 0): Sequence<MatchResult>
             = generateSequence({ find(input, startIndex) }, MatchResult::next)
@@ -233,9 +229,10 @@ class Regex internal constructor(internal val nativePattern: Pattern) {
     }
 
     /**
-     * Replaces all occurrences of this regular expression in the specified [input] string with specified [replacement] expression.
+     * Replaces all occurrences of this regular expression in the specified [input] string with
+     * specified [replacement] expression.
      *
-     * @param replacement A replacement expression that can include substitutions. See [Matcher.appendReplacement] for details.
+     * @param replacement A replacement expression that can include substitutions.
      */
     fun replace(input: CharSequence, replacement: String): String
             = replace(input) { match -> processReplacement(match, replacement) }
@@ -280,7 +277,6 @@ class Regex internal constructor(internal val nativePattern: Pattern) {
      * @param limit Non-negative value specifying the maximum number of substrings the string can be split to.
      *              Zero by default means no limit is set.
      */
-    // TODO: replace all argument checks with require function.
     fun split(input: CharSequence, limit: Int = 0): List<String> {
         require(limit >= 0, { "Limit must be non-negative, but was $limit." } )
         if (input.isEmpty()) {

runtime/src/main/kotlin/kotlin/text/Strings.kt

@@ -671,7 +671,7 @@ public inline fun CharSequence.replaceFirst(regex: Regex, replacement: String):
  * Returns `true` if this char sequence matches the given regular expression.
  */
 @kotlin.internal.InlineOnly
-public inline fun CharSequence.matches(regex: Regex): Boolean = regex.matches(this)
+public infix inline fun CharSequence.matches(regex: Regex): Boolean = regex.matches(this)
 
 /**
  * Implementation of [regionMatches] for CharSequences.

runtime/src/main/kotlin/kotlin/text/regex/Lexer.kt

@@ -28,9 +28,15 @@ external private fun getCanonicalClassInternal(ch: Int): Int
 external private fun hasSingleCodepointDecompositionInternal(ch: Int): Boolean
 
 /** Returns a decomposition for a given codepoint. */
-@SymbolName("Kotlin_text_getDecompositionInternal")
+@SymbolName("Kotlin_text_regex_getDecompositionInternal")
 external private fun getDecompositionInternal(ch: Int): IntArray?
 
+/**
+ * Decomposes the given string represented as an array of codepoints. Saves the decomposition into [outputCodepoints] array.
+ * Returns the length of the decomposition.
+ */
+@SymbolName("Kotlin_text_regex_decomposeString")
+external private fun decomposeString(inputCodePoints: IntArray, inputLength: Int, outputCodePoints: IntArray): Int
 // =============================================================================================================
 
 /**
@@ -792,13 +798,8 @@ internal class Lexer(val patternString: String, flags: Int) {
         // =============================================================================================================
 
         /**
-         * Normalize given expression.
+         * Normalize given string.
-
-         * @param input - expression to normalize
-         * *
-         * @return normalized expression.
          */
-        // TODO: Implement on the C++ side.
         fun normalize(input: String): String {
             val inputChars = input.toCharArray()
             val inputLength = inputChars.size
@@ -824,45 +825,22 @@ internal class Lexer(val patternString: String, flags: Int) {
             //result of canonical decomposition of input in UTF-16 encoding
             val result = StringBuilder()
 
-            run {
+            var i = 0
-                var i = 0
+            while (i < inputLength) {
-                while (i < inputLength) {
+                ch = input.codePointAt(i)
-                    ch = input.codePointAt(i)
+                inputCodePoints[inputCodePointsIndex++] = ch
-                    inputCodePoints[inputCodePointsIndex++] = ch
+                i += if (Char.isSupplementaryCodePoint(ch)) 2 else 1
-                    i += if (Char.isSupplementaryCodePoint(ch)) 2 else 1
-                }
             }
 
-            /*
+            // Canonical decomposition based on mappings in decomposition table.
-             * Canonical decomposition based on mappings in decompTable
+            resCodePointsIndex = decomposeString(inputCodePoints, inputCodePointsIndex, resCodePoints)
-             */
-            for (i in 0..inputCodePointsIndex - 1) {
-                ch = inputCodePoints[i]
 
-                decomp = Lexer.getDecomposition(ch)
+            // Canonical ordering.
-                if (decomp == null) {
+            // See http://www.unicode.org/reports/tr15/#Decomposition for details
-                    resCodePoints[resCodePointsIndex++] = ch
-                } else {
-                    val curSymbDecompLength = decomp.size
-
-                    for (j in 0..curSymbDecompLength - 1) {
-                        resCodePoints[resCodePointsIndex++] = decomp[j]
-                    }
-                }
-            }
-
-            /*
-             * Canonical ordering.
-             * See http://www.unicode.org/reports/tr15/#Decomposition for
-             * details
-             */
             resCodePoints = Lexer.getCanonicalOrder(resCodePoints, resCodePointsIndex)
 
-            /*
+            // Decomposition for Hangul syllables.
-             * Decomposition for Hangul syllables.
+            // See http://www.unicode.org/reports/tr15/#Hangul for details
-             * See http://www.unicode.org/reports/tr15/#Hangul for
-             * details
-             */
             decompHangul = IntArray(resCodePoints.size)
 
             for (i in 0..resCodePointsIndex - 1) {
@@ -872,11 +850,7 @@ internal class Lexer(val patternString: String, flags: Int) {
                 if (decomp == null) {
                     decompHangul[decompHangulIndex++] = curSymb
                 } else {
-
+                    // Note that Hangul decompositions have length that is equal 2 or 3.
-                    /*
-                     * Note that Hangul decompositions have length that is
-                     * equal 2 or 3.
-                     */
                     decompHangul[decompHangulIndex++] = decomp[0]
                     decompHangul[decompHangulIndex++] = decomp[1]
                     if (decomp.size == 3) {
@@ -885,9 +859,7 @@ internal class Lexer(val patternString: String, flags: Int) {
                 }
             }
 
-            /*
+            // Translating into UTF-16 encoding
-             * Translating into UTF-16 encoding
-             */
             for (i in 0..decompHangulIndex - 1) {
                 result.append(Char.toChars(decompHangul[i]))
             }
@@ -896,17 +868,8 @@ internal class Lexer(val patternString: String, flags: Int) {
         }
 
         /**
-         * Rearrange codepoints according
+         * Rearrange codepoints in [inputInts] according to canonical order. Return an array with rearranged codepoints.
-         * to canonical order.
-
-         * @param inputInts - array that contains Unicode codepoints
-         * *
-         * @param length - index of last Unicode codepoint plus 1
-         * *
-         * *
-         * @return array that contains rearranged codepoints.
          */
-        // TODO: C++ side!
         fun getCanonicalOrder(inputInts: IntArray, length: Int): IntArray {
             val inputLength = if (length < inputInts.size)
                 length
@@ -914,12 +877,9 @@ internal class Lexer(val patternString: String, flags: Int) {
                 inputInts.size
 
             /*
-             * Simple bubble-sort algorithm.
+             * Simple bubble-sort algorithm. Note that many codepoints have 0 canonical class, so this algorithm works
-             * Note that many codepoints have 0
+             * almost lineary in overwhelming majority of cases. This is due to specific of Unicode combining
-             * canonical class, so this algorithm works
+             * classes and codepoints.
-             * almost lineary in overwhelming majority
-             * of cases. This is due to specific of Unicode
-             * combining classes and codepoints.
              */
             for (i in 1..inputLength - 1) {
                 var j = i - 1
@@ -951,14 +911,8 @@ internal class Lexer(val patternString: String, flags: Int) {
         /**
          * Gets decomposition for given Hangul syllable.
          * This is an implementation of Hangul decomposition algorithm
-         * according to http://www.unicode.org/versions/Unicode4.0.0/ch03.pdf
+         * according to http://www.unicode.org/versions/Unicode4.0.0/ch03.pdf "3.12 Conjoining Jamo Behavior".
-         * "3.12 Conjoining Jamo Behavior".
-
-         * @param ch - given Hangul syllable
-         * *
-         * @return canonical decomposition of currentChar.
          */
-        // TODO: C++
         fun getHangulDecomposition(ch: Int): IntArray? {
             val SIndex = ch - SBase
 

runtime/src/main/kotlin/kotlin/text/regex/MatchResultImpl.kt

@@ -116,13 +116,13 @@ constructor (internal val input: CharSequence,
 
     override fun next(): MatchResult? {
         var nextStart = range.endInclusive + 1
-        if (nextStart == input.length) {
-            return null
-        }
         // If the current match is empty - shift by 1.
         if (nextStart == range.start) {
             nextStart++
         }
+        if (nextStart > input.length) {
+            return null
+        }
         return regex.find(input, nextStart)
     }
     // =================================================================================================================

runtime/src/main/kotlin/kotlin/text/regex/sets/DecomposedCharSet.kt

@@ -17,6 +17,13 @@
 
 package kotlin.text.regex
 
+/**
+ * Decomposes the given codepoint. Saves the decomposition into [outputCodepoints] array starting with [fromIndex].
+ * Returns the length of the decomposition.
+ */
+@SymbolName("Kotlin_text_regex_decomposeCodePoint")
+external private fun decomposeCodePoint(codePoint: Int, outputCodePoints: IntArray, fromIndex: Int): Int
+
 /** Represents canonical decomposition of Unicode character. Is used when CANON_EQ flag of Pattern class is specified. */
 open internal class DecomposedCharSet(
         /** Decomposition of the Unicode codepoint */
@@ -48,19 +55,11 @@ open internal class DecomposedCharSet(
         // We read testString and decompose it gradually to compare with this decomposedChar at position strIndex
         var curChar = codePointAt(strIndex, testString, rightBound)
         strIndex += readCharsForCodePoint
-        var decomposedCurrentCodePoint: IntArray? = Lexer.getDecomposition(curChar)
         var readCodePoints = 0
         var i = 0
         // All decompositions have length that is less or equal Lexer.MAX_DECOMPOSITION_LENGTH
-        var decomposedCodePoint: IntArray
+        var decomposedCodePoint: IntArray = IntArray(Lexer.MAX_DECOMPOSITION_LENGTH)
-        if (decomposedCurrentCodePoint == null) {
+        readCodePoints += decomposeCodePoint(curChar, decomposedCodePoint, readCodePoints)
-            decomposedCodePoint = IntArray(Lexer.MAX_DECOMPOSITION_LENGTH)
-            decomposedCodePoint[readCodePoints++] = curChar
-        } else {
-            i = decomposedCurrentCodePoint.size
-            decomposedCodePoint = decomposedCurrentCodePoint.copyOf(Lexer.MAX_DECOMPOSITION_LENGTH)
-            readCodePoints += i
-        }
 
         if (strIndex < rightBound) {
             curChar = codePointAt(strIndex, testString, rightBound)
@@ -71,26 +70,11 @@ open internal class DecomposedCharSet(
                 if (!Lexer.hasDecompositionNonNullCanClass(curChar)) {
                     decomposedCodePoint[readCodePoints++] = curChar
                 } else {
-
                     /*
-                     * A few codepoints have decompositions and non null
+                     * A few codepoints have decompositions and non null canonical classes, we have to take them into
-                     * canonical classes, we have to take them into
+                     * consideration, but general rule is: if canonical class != 0 then no decomposition
-                     * consideration, but general rule is:
-                     * if canonical class != 0 then no decomposition
                      */
-                    decomposedCurrentCodePoint = Lexer.getDecomposition(curChar)
+                    readCodePoints += decomposeCodePoint(curChar, decomposedCodePoint, readCodePoints)
-
-                    /*
-                     * Length of such decomposition is 1 or 2. See UnicodeData file
-                     * http://www.unicode.org/Public/4.0-Update/UnicodeData-4.0.0.txt
-                     */
-                    // hasDecompositionNonNullCanClass(curChar) == true, so decomposedCurrentCodePoint != null.
-                    if (decomposedCurrentCodePoint!!.size == 2) {
-                        decomposedCodePoint[readCodePoints++] = decomposedCurrentCodePoint[0]
-                        decomposedCodePoint[readCodePoints++] = decomposedCurrentCodePoint[1]
-                    } else {
-                        decomposedCodePoint[readCodePoints++] = decomposedCurrentCodePoint[0]
-                    }
                 }
 
                 strIndex += readCharsForCodePoint
@@ -133,7 +117,7 @@ open internal class DecomposedCharSet(
     }
 
     override val name: String
-            get() = "decomposed char: $decomposedChar"
+        get() = "decomposed char: $decomposedChar"
 
     /** Reads Unicode codepoint from [testString] starting from [strIndex] until [rightBound]. */
     fun codePointAt(strIndex: Int, testString: CharSequence, rightBound: Int): Int {

runtime/src/main/kotlin/kotlin/text/regex/sets/LeafQuantifierSet.kt

@@ -57,7 +57,7 @@ open internal class LeafQuantifierSet(var quantifier: Quantifier,
         }
 
         // Process occurrences between min and max.
-        while  ((max < 0 || occurrences < max) && index + leaf.charCount <= testString.length) {
+        while  ((max == Quantifier.INF || occurrences < max) && index + leaf.charCount <= testString.length) {
             val shift = leaf.accepts(index, testString)
             if (shift < 1) {
                 break

runtime/src/main/kotlin/kotlin/text/regex/sets/PossessiveLeafQuantifierSet.kt

@@ -49,7 +49,7 @@ internal class PossessiveLeafQuantifierSet(
             occurrences++
         }
 
-        while ((max >= 0 || occurrences < max) && index + leaf.charCount <= testString.length) {
+        while ((max == Quantifier.INF || occurrences < max) && index + leaf.charCount <= testString.length) {
             val shift = leaf.accepts(index, testString)
             if (shift < 1) {
                 break

runtime/src/main/kotlin/kotlin/text/regex/sets/ReluctantLeafQuantifierSet.kt

@@ -62,7 +62,7 @@ internal class ReluctantLeafQuantifierSet(
                 occurrences++
             }
 
-        } while (shift >= 1 && occurrences <= max)
+        } while (shift >= 1 && (max == Quantifier.INF || occurrences <= max))
 
         return -1
     }