azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

regex: Perform char decomposition on C++ side

Browse Source
This commit is contained in:
Ilya Matveev
2017-06-16 18:47:40 +07:00
committed by ilmat192
parent 1e7d132a88
commit 22aead3d18
9 changed files with 103 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/src/main/cpp/Regex.cpp
+52 -8
View File
@@ -1,3 +1,4 @@
#include <cstring>
#include "Types.h"
#include "KString.h"
#include "Natives.h"
@@ -560,21 +561,64 @@ KBoolean Kotlin_text_regex_hasSingleCodepointDecompositionInternal(KInt ch) {
return singleDecompositions[index] == ch;
}
OBJ_GETTER(Kotlin_text_getDecompositionInternal, KInt ch) {
// TODO: Move all uses of decomposition table into C++ code.
int index = binarySearchRange(decompositionKeys, ARRAY_SIZE(decompositionKeys), ch);
if (decompositionKeys[index] != ch) {
const Decomposition* getDecomposition(KInt codePoint) {
int index = binarySearchRange(decompositionKeys, ARRAY_SIZE(decompositionKeys), codePoint);
if (decompositionKeys[index] != codePoint) {
return nullptr;
}
const Decomposition& decomposition = decompositionValues[index];
ArrayHeader* result = AllocArrayInstance(theIntArrayTypeInfo, decomposition.length, OBJ_RESULT)->array();
return &decompositionValues[index];
}
OBJ_GETTER(Kotlin_text_regex_getDecompositionInternal, KInt ch) {
const Decomposition* decomposition = getDecomposition(ch);
if (decomposition == nullptr) {
return nullptr;
}
ArrayHeader* result = AllocArrayInstance(theIntArrayTypeInfo, decomposition->length, OBJ_RESULT)->array();
KInt* resultRaw = IntArrayAddressOfElementAt(result, 0);
for (int i = 0; i < decomposition.length; i++) {
*resultRaw++ = decomposition.array[i];
for (int i = 0; i < decomposition->length; i++) {
*resultRaw++ = decomposition->array[i];
}
RETURN_OBJ(result->obj());
}
KInt Kotlin_text_regex_decomposeString(ArrayHeader* inputCodePoints, KInt inputLength, ArrayHeader* outputCodePoints) {
RuntimeAssert(inputCodePoints->type_info() == theIntArrayTypeInfo, "Must use an Int array");
RuntimeAssert(outputCodePoints->type_info() == theIntArrayTypeInfo, "Must use an Int array");
RuntimeAssert(inputLength >= 0, "Input length must be >= 0");
if (inputLength == 0) {
return 0;
}
int outputLength = 0;
const KInt* inputArray = IntArrayAddressOfElementAt(inputCodePoints, 0);
KInt* outputArray = IntArrayAddressOfElementAt(outputCodePoints, 0);
for (int i = 0; i < inputLength; i++) {
const Decomposition* decomposition = getDecomposition(inputArray[i]);
if (decomposition == nullptr) {
outputArray[outputLength++] = inputArray[i];
} else {
memcpy(outputArray + outputLength, decomposition->array, decomposition->length * sizeof(KInt));
outputLength+=decomposition->length;
}
}
return outputLength;
}
KInt Kotlin_text_regex_decomposeCodePoint(KInt codePoint, ArrayHeader* outputCodePoints, KInt fromIndex) {
RuntimeAssert(outputCodePoints->type_info() == theIntArrayTypeInfo, "Must be an Int array");
RuntimeAssert(fromIndex >= 0 && fromIndex < outputCodePoints->count_, "Start index must be >= 0 and < array size");
KInt* rawResult = IntArrayAddressOfElementAt(outputCodePoints, fromIndex);
const Decomposition* decomposition = getDecomposition(codePoint);
if (decomposition == nullptr) {
*rawResult = codePoint;
return 1;
} else {
memcpy(rawResult, decomposition->array, decomposition->length * sizeof(KInt));
return decomposition->length;
}
}
} // extern "C"
runtime/src/main/kotlin/kotlin/text/Regex.kt
+5 -9
View File
@@ -113,7 +113,6 @@ class Regex internal constructor(internal val nativePattern: Pattern) {
companion object {
/** Returns a literal regex for the specified [literal] string. */
// TODO: Uncomment for native
fun fromLiteral(literal: String): Regex = Regex(literal, RegexOption.LITERAL)
/** Returns a literal pattern for the specified [literal] string. */
@@ -159,7 +158,6 @@ class Regex internal constructor(internal val nativePattern: Pattern) {
infix fun matches(input: CharSequence): Boolean = doMatch(input, Mode.MATCH) != null
/** Indicates whether the regular expression can find at least one match in the specified [input]. */
// TODO: Looks like we don't need Mode anymore.
fun containsMatchIn(input: CharSequence): Boolean = find(input) != null
/**
@@ -172,7 +170,6 @@ class Regex internal constructor(internal val nativePattern: Pattern) {
if (startIndex < 0 || startIndex > input.length) {
throw IndexOutOfBoundsException() // TODO: Add a message.
}
// TODO: reuse the match result?
val matchResult = MatchResultImpl(input, this)
matchResult.mode = Mode.FIND
matchResult.startIndex = startIndex
@@ -181,14 +178,13 @@ class Regex internal constructor(internal val nativePattern: Pattern) {
matchResult.finalizeMatch()
return matchResult
} else {
/*matchResult.hitEnd = true
matchResult.startIndex = -1*/
return null
}
}
/**
* Returns a sequence of all occurrences of a regular expression within the [input] string, beginning at the specified [startIndex].
* Returns a sequence of all occurrences of a regular expression within the [input] string,
* beginning at the specified [startIndex].
*/
fun findAll(input: CharSequence, startIndex: Int = 0): Sequence<MatchResult>
= generateSequence({ find(input, startIndex) }, MatchResult::next)
@@ -233,9 +229,10 @@ class Regex internal constructor(internal val nativePattern: Pattern) {
}
/**
* Replaces all occurrences of this regular expression in the specified [input] string with specified [replacement] expression.
* Replaces all occurrences of this regular expression in the specified [input] string with
* specified [replacement] expression.
*
* @param replacement A replacement expression that can include substitutions. See [Matcher.appendReplacement] for details.
* @param replacement A replacement expression that can include substitutions.
*/
fun replace(input: CharSequence, replacement: String): String
= replace(input) { match -> processReplacement(match, replacement) }
@@ -280,7 +277,6 @@ class Regex internal constructor(internal val nativePattern: Pattern) {
* @param limit Non-negative value specifying the maximum number of substrings the string can be split to.
* Zero by default means no limit is set.
*/
// TODO: replace all argument checks with require function.
fun split(input: CharSequence, limit: Int = 0): List<String> {
require(limit >= 0, { "Limit must be non-negative, but was $limit." } )
if (input.isEmpty()) {
runtime/src/main/kotlin/kotlin/text/Strings.kt
+1 -1
View File
@@ -671,7 +671,7 @@ public inline fun CharSequence.replaceFirst(regex: Regex, replacement: String):
* Returns `true` if this char sequence matches the given regular expression.
*/
@kotlin.internal.InlineOnly
public inline fun CharSequence.matches(regex: Regex): Boolean = regex.matches(this)
public infix inline fun CharSequence.matches(regex: Regex): Boolean = regex.matches(this)
/**
* Implementation of [regionMatches] for CharSequences.
runtime/src/main/kotlin/kotlin/text/regex/Lexer.kt
+26 -72
View File
@@ -28,9 +28,15 @@ external private fun getCanonicalClassInternal(ch: Int): Int
external private fun hasSingleCodepointDecompositionInternal(ch: Int): Boolean
/** Returns a decomposition for a given codepoint. */
@SymbolName("Kotlin_text_getDecompositionInternal")
@SymbolName("Kotlin_text_regex_getDecompositionInternal")
external private fun getDecompositionInternal(ch: Int): IntArray?
/**
* Decomposes the given string represented as an array of codepoints. Saves the decomposition into [outputCodepoints] array.
* Returns the length of the decomposition.
*/
@SymbolName("Kotlin_text_regex_decomposeString")
external private fun decomposeString(inputCodePoints: IntArray, inputLength: Int, outputCodePoints: IntArray): Int
// =============================================================================================================
/**
@@ -792,13 +798,8 @@ internal class Lexer(val patternString: String, flags: Int) {
// =============================================================================================================
/**
* Normalize given expression.
* @param input - expression to normalize
* *
* @return normalized expression.
* Normalize given string.
*/
// TODO: Implement on the C++ side.
fun normalize(input: String): String {
val inputChars = input.toCharArray()
val inputLength = inputChars.size
@@ -824,45 +825,22 @@ internal class Lexer(val patternString: String, flags: Int) {
//result of canonical decomposition of input in UTF-16 encoding
val result = StringBuilder()
run {
var i = 0
while (i < inputLength) {
ch = input.codePointAt(i)
inputCodePoints[inputCodePointsIndex++] = ch
i += if (Char.isSupplementaryCodePoint(ch)) 2 else 1
}
var i = 0
while (i < inputLength) {
ch = input.codePointAt(i)
inputCodePoints[inputCodePointsIndex++] = ch
i += if (Char.isSupplementaryCodePoint(ch)) 2 else 1
}
/*
* Canonical decomposition based on mappings in decompTable
*/
for (i in 0..inputCodePointsIndex - 1) {
ch = inputCodePoints[i]
// Canonical decomposition based on mappings in decomposition table.
resCodePointsIndex = decomposeString(inputCodePoints, inputCodePointsIndex, resCodePoints)
decomp = Lexer.getDecomposition(ch)
if (decomp == null) {
resCodePoints[resCodePointsIndex++] = ch
} else {
val curSymbDecompLength = decomp.size
for (j in 0..curSymbDecompLength - 1) {
resCodePoints[resCodePointsIndex++] = decomp[j]
}
}
}
/*
* Canonical ordering.
* See http://www.unicode.org/reports/tr15/#Decomposition for
* details
*/
// Canonical ordering.
// See http://www.unicode.org/reports/tr15/#Decomposition for details
resCodePoints = Lexer.getCanonicalOrder(resCodePoints, resCodePointsIndex)
/*
* Decomposition for Hangul syllables.
* See http://www.unicode.org/reports/tr15/#Hangul for
* details
*/
// Decomposition for Hangul syllables.
// See http://www.unicode.org/reports/tr15/#Hangul for details
decompHangul = IntArray(resCodePoints.size)
for (i in 0..resCodePointsIndex - 1) {
@@ -872,11 +850,7 @@ internal class Lexer(val patternString: String, flags: Int) {
if (decomp == null) {
decompHangul[decompHangulIndex++] = curSymb
} else {
/*
* Note that Hangul decompositions have length that is
* equal 2 or 3.
*/
// Note that Hangul decompositions have length that is equal 2 or 3.
decompHangul[decompHangulIndex++] = decomp[0]
decompHangul[decompHangulIndex++] = decomp[1]
if (decomp.size == 3) {
@@ -885,9 +859,7 @@ internal class Lexer(val patternString: String, flags: Int) {
}
}
/*
* Translating into UTF-16 encoding
*/
// Translating into UTF-16 encoding
for (i in 0..decompHangulIndex - 1) {
result.append(Char.toChars(decompHangul[i]))
}
@@ -896,17 +868,8 @@ internal class Lexer(val patternString: String, flags: Int) {
}
/**
* Rearrange codepoints according
* to canonical order.
* @param inputInts - array that contains Unicode codepoints
* *
* @param length - index of last Unicode codepoint plus 1
* *
* *
* @return array that contains rearranged codepoints.
* Rearrange codepoints in [inputInts] according to canonical order. Return an array with rearranged codepoints.
*/
// TODO: C++ side!
fun getCanonicalOrder(inputInts: IntArray, length: Int): IntArray {
val inputLength = if (length < inputInts.size)
length
@@ -914,12 +877,9 @@ internal class Lexer(val patternString: String, flags: Int) {
inputInts.size
/*
* Simple bubble-sort algorithm.
* Note that many codepoints have 0
* canonical class, so this algorithm works
* almost lineary in overwhelming majority
* of cases. This is due to specific of Unicode
* combining classes and codepoints.
* Simple bubble-sort algorithm. Note that many codepoints have 0 canonical class, so this algorithm works
* almost lineary in overwhelming majority of cases. This is due to specific of Unicode combining
* classes and codepoints.
*/
for (i in 1..inputLength - 1) {
var j = i - 1
@@ -951,14 +911,8 @@ internal class Lexer(val patternString: String, flags: Int) {
/**
* Gets decomposition for given Hangul syllable.
* This is an implementation of Hangul decomposition algorithm
* according to http://www.unicode.org/versions/Unicode4.0.0/ch03.pdf
* "3.12 Conjoining Jamo Behavior".
* @param ch - given Hangul syllable
* *
* @return canonical decomposition of currentChar.
* according to http://www.unicode.org/versions/Unicode4.0.0/ch03.pdf "3.12 Conjoining Jamo Behavior".
*/
// TODO: C++
fun getHangulDecomposition(ch: Int): IntArray? {
val SIndex = ch - SBase
runtime/src/main/kotlin/kotlin/text/regex/MatchResultImpl.kt
+3 -3
View File
@@ -116,13 +116,13 @@ constructor (internal val input: CharSequence,
override fun next(): MatchResult? {
var nextStart = range.endInclusive + 1
if (nextStart == input.length) {
return null
}
// If the current match is empty - shift by 1.
if (nextStart == range.start) {
nextStart++
}
if (nextStart > input.length) {
return null
}
return regex.find(input, nextStart)
}
// =================================================================================================================
runtime/src/main/kotlin/kotlin/text/regex/sets/DecomposedCharSet.kt
+13 -29
View File
@@ -17,6 +17,13 @@
package kotlin.text.regex
/**
* Decomposes the given codepoint. Saves the decomposition into [outputCodepoints] array starting with [fromIndex].
* Returns the length of the decomposition.
*/
@SymbolName("Kotlin_text_regex_decomposeCodePoint")
external private fun decomposeCodePoint(codePoint: Int, outputCodePoints: IntArray, fromIndex: Int): Int
/** Represents canonical decomposition of Unicode character. Is used when CANON_EQ flag of Pattern class is specified. */
open internal class DecomposedCharSet(
/** Decomposition of the Unicode codepoint */
@@ -48,19 +55,11 @@ open internal class DecomposedCharSet(
// We read testString and decompose it gradually to compare with this decomposedChar at position strIndex
var curChar = codePointAt(strIndex, testString, rightBound)
strIndex += readCharsForCodePoint
var decomposedCurrentCodePoint: IntArray? = Lexer.getDecomposition(curChar)
var readCodePoints = 0
var i = 0
// All decompositions have length that is less or equal Lexer.MAX_DECOMPOSITION_LENGTH
var decomposedCodePoint: IntArray
if (decomposedCurrentCodePoint == null) {
decomposedCodePoint = IntArray(Lexer.MAX_DECOMPOSITION_LENGTH)
decomposedCodePoint[readCodePoints++] = curChar
} else {
i = decomposedCurrentCodePoint.size
decomposedCodePoint = decomposedCurrentCodePoint.copyOf(Lexer.MAX_DECOMPOSITION_LENGTH)
readCodePoints += i
}
var decomposedCodePoint: IntArray = IntArray(Lexer.MAX_DECOMPOSITION_LENGTH)
readCodePoints += decomposeCodePoint(curChar, decomposedCodePoint, readCodePoints)
if (strIndex < rightBound) {
curChar = codePointAt(strIndex, testString, rightBound)
@@ -71,26 +70,11 @@ open internal class DecomposedCharSet(
if (!Lexer.hasDecompositionNonNullCanClass(curChar)) {
decomposedCodePoint[readCodePoints++] = curChar
} else {
/*
* A few codepoints have decompositions and non null
* canonical classes, we have to take them into
* consideration, but general rule is:
* if canonical class != 0 then no decomposition
* A few codepoints have decompositions and non null canonical classes, we have to take them into
* consideration, but general rule is: if canonical class != 0 then no decomposition
*/
decomposedCurrentCodePoint = Lexer.getDecomposition(curChar)
/*
* Length of such decomposition is 1 or 2. See UnicodeData file
* http://www.unicode.org/Public/4.0-Update/UnicodeData-4.0.0.txt
*/
// hasDecompositionNonNullCanClass(curChar) == true, so decomposedCurrentCodePoint != null.
if (decomposedCurrentCodePoint!!.size == 2) {
decomposedCodePoint[readCodePoints++] = decomposedCurrentCodePoint[0]
decomposedCodePoint[readCodePoints++] = decomposedCurrentCodePoint[1]
} else {
decomposedCodePoint[readCodePoints++] = decomposedCurrentCodePoint[0]
}
readCodePoints += decomposeCodePoint(curChar, decomposedCodePoint, readCodePoints)
}
strIndex += readCharsForCodePoint
@@ -133,7 +117,7 @@ open internal class DecomposedCharSet(
}
override val name: String
get() = "decomposed char: $decomposedChar"
get() = "decomposed char: $decomposedChar"
/** Reads Unicode codepoint from [testString] starting from [strIndex] until [rightBound]. */
fun codePointAt(strIndex: Int, testString: CharSequence, rightBound: Int): Int {
runtime/src/main/kotlin/kotlin/text/regex/sets/LeafQuantifierSet.kt
+1 -1
View File
@@ -57,7 +57,7 @@ open internal class LeafQuantifierSet(var quantifier: Quantifier,
}
// Process occurrences between min and max.
while ((max < 0 || occurrences < max) && index + leaf.charCount <= testString.length) {
while ((max == Quantifier.INF || occurrences < max) && index + leaf.charCount <= testString.length) {
val shift = leaf.accepts(index, testString)
if (shift < 1) {
break
runtime/src/main/kotlin/kotlin/text/regex/sets/PossessiveLeafQuantifierSet.kt
+1 -1
View File
@@ -49,7 +49,7 @@ internal class PossessiveLeafQuantifierSet(
occurrences++
}
while ((max >= 0 || occurrences < max) && index + leaf.charCount <= testString.length) {
while ((max == Quantifier.INF || occurrences < max) && index + leaf.charCount <= testString.length) {
val shift = leaf.accepts(index, testString)
if (shift < 1) {
break
runtime/src/main/kotlin/kotlin/text/regex/sets/ReluctantLeafQuantifierSet.kt
+1 -1
View File
@@ -62,7 +62,7 @@ internal class ReluctantLeafQuantifierSet(
occurrences++
}
} while (shift >= 1 && occurrences <= max)
} while (shift >= 1 && (max == Quantifier.INF || occurrences <= max))
return -1
}