From e64a5f42f06a09bc63cb5c4b157b8f3b59f8636c Mon Sep 17 00:00:00 2001 From: Ilya Matveev Date: Thu, 8 Jun 2017 14:04:23 +0700 Subject: [PATCH] stdlib: Improve character API. The patch adds methods for surrogate characters processing and Unicode category support needed for regular expression library. --- backend.native/tests/build.gradle | 5 + .../tests/runtime/basic/tostring2.kt | 2 +- backend.native/tests/runtime/text/chars0.kt | 113 ++++++++++++ .../tests/runtime/text/string_builder0.kt | 2 - runtime/src/main/cpp/KString.cpp | 5 +- .../src/main/kotlin/konan/internal/Strings.kt | 6 + runtime/src/main/kotlin/kotlin/Char.kt | 15 ++ .../kotlin/kotlin/collections/Collections.kt | 11 ++ runtime/src/main/kotlin/kotlin/text/Char.kt | 56 ++++++ .../main/kotlin/kotlin/text/CharCategory.kt | 171 ++++++++++++++++++ .../main/kotlin/kotlin/text/StringBuilder.kt | 9 +- 11 files changed, 385 insertions(+), 10 deletions(-) create mode 100644 backend.native/tests/runtime/text/chars0.kt create mode 100644 runtime/src/main/kotlin/kotlin/text/CharCategory.kt diff --git a/backend.native/tests/build.gradle b/backend.native/tests/build.gradle index e841dae69f9..5f503a8da77 100644 --- a/backend.native/tests/build.gradle +++ b/backend.native/tests/build.gradle @@ -1268,6 +1268,11 @@ task to_string0(type: RunKonanTest) { source = "runtime/text/to_string0.kt" } +task chars0(type: RunKonanTest) { + source = "runtime/text/chars0.kt" + expectedExitStatus = 0 +} + task catch1(type: RunKonanTest) { goldValue = "Before\nCaught Throwable\nDone\n" source = "runtime/exceptions/catch1.kt" diff --git a/backend.native/tests/runtime/basic/tostring2.kt b/backend.native/tests/runtime/basic/tostring2.kt index aa17c92b735..98b48764cf7 100644 --- a/backend.native/tests/runtime/basic/tostring2.kt +++ b/backend.native/tests/runtime/basic/tostring2.kt @@ -1,6 +1,6 @@ fun main(args : Array) { val hello = "Hello" - val array = toCharArray(hello) + val array = hello.toCharArray() for (ch in array) { print(ch) print(" ") diff --git a/backend.native/tests/runtime/text/chars0.kt b/backend.native/tests/runtime/text/chars0.kt new file mode 100644 index 00000000000..9465caa209b --- /dev/null +++ b/backend.native/tests/runtime/text/chars0.kt @@ -0,0 +1,113 @@ +fun assertTrue(v: Boolean) = if (!v) throw AssertionError() else Unit +fun assertFalse(v: Boolean) = if (v) throw AssertionError() else Unit +fun assertEquals(a: Int, b: Int) { if (a != b) throw AssertionError() } + +fun testIsSupplementaryCodePoint() { + assertFalse(Char.isSupplementaryCodePoint(-1)) + for (c in 0..0xFFFF) { + assertFalse(Char.isSupplementaryCodePoint(c.toInt())) + } + for (c in 0xFFFF + 1..0x10FFFF) { + assertTrue(Char.isSupplementaryCodePoint(c)) + } + assertFalse(Char.isSupplementaryCodePoint(0x10FFFF + 1)) +} + +fun testIsSurrogatePair() { + assertFalse(Char.isSurrogatePair('\u0000', '\u0000')) + assertFalse(Char.isSurrogatePair('\u0000', '\uDC00')) + assertTrue( Char.isSurrogatePair('\uD800', '\uDC00')) + assertTrue( Char.isSurrogatePair('\uD800', '\uDFFF')) + assertTrue( Char.isSurrogatePair('\uDBFF', '\uDFFF')) + assertFalse(Char.isSurrogatePair('\uDBFF', '\uF000')) +} + +fun testToChars() { + assertTrue(charArrayOf('\uD800', '\uDC00').contentEquals(Char.toChars(0x010000))) + assertTrue(charArrayOf('\uD800', '\uDC01').contentEquals(Char.toChars(0x010001))) + assertTrue(charArrayOf('\uD801', '\uDC01').contentEquals(Char.toChars(0x010401))) + assertTrue(charArrayOf('\uDBFF', '\uDFFF').contentEquals(Char.toChars(0x10FFFF))) + + try { + Char.toChars(Int.MAX_VALUE) + throw AssertionError() + } catch (e: IllegalArgumentException) {} +} + +fun testToCodePoint() { + assertEquals(0x010000, Char.toCodePoint('\uD800', '\uDC00')) + assertEquals(0x010001, Char.toCodePoint('\uD800', '\uDC01')) + assertEquals(0x010401, Char.toCodePoint('\uD801', '\uDC01')) + assertEquals(0x10FFFF, Char.toCodePoint('\uDBFF', '\uDFFF')) +} + +// TODO: Uncomment when such operations are supported for supplementary codepoints and the API is public. +fun testCase() { + /* + assertEquals('A'.toInt(), Char.toUpperCase('a'.toInt())) + assertEquals('A'.toInt(), Char.toUpperCase('A'.toInt())) + assertEquals('1'.toInt(), Char.toUpperCase('1'.toInt())) + + assertEquals('a'.toInt(), Char.toLowerCase('A'.toInt())) + assertEquals('a'.toInt(), Char.toLowerCase('a'.toInt())) + assertEquals('1'.toInt(), Char.toLowerCase('1'.toInt())) + + assertEquals(0x010400, Char.toUpperCase(0x010428)) + assertEquals(0x010400, Char.toUpperCase(0x010400)) + assertEquals(0x10FFFF, Char.toUpperCase(0x10FFFF)) + assertEquals(0x110000, Char.toUpperCase(0x110000)) + + assertEquals(0x010428, Char.toLowerCase(0x010400)) + assertEquals(0x010428, Char.toLowerCase(0x010428)) + assertEquals(0x10FFFF, Char.toLowerCase(0x10FFFF)) + assertEquals(0x110000, Char.toLowerCase(0x110000)) + */ +} + +fun testCategory() { + assertEquals('\n'.category.value, CharCategory.CONTROL.value) + assertEquals('1'.category.value, CharCategory.DECIMAL_DIGIT_NUMBER.value) + assertEquals(' '.category.value, CharCategory.SPACE_SEPARATOR.value) + assertEquals('a'.category.value, CharCategory.LOWERCASE_LETTER.value) + assertEquals('A'.category.value, CharCategory.UPPERCASE_LETTER.value) + assertEquals('<'.category.value, CharCategory.MATH_SYMBOL.value) + assertEquals(';'.category.value, CharCategory.OTHER_PUNCTUATION.value) + assertEquals('_'.category.value, CharCategory.CONNECTOR_PUNCTUATION.value) + assertEquals('$'.category.value, CharCategory.CURRENCY_SYMBOL.value) + assertEquals('\u2029'.category.value, CharCategory.PARAGRAPH_SEPARATOR.value) + + assertTrue('\n' in CharCategory.CONTROL) + assertTrue('1' in CharCategory.DECIMAL_DIGIT_NUMBER) + assertTrue(' ' in CharCategory.SPACE_SEPARATOR) + assertTrue('a' in CharCategory.LOWERCASE_LETTER) + assertTrue('A' in CharCategory.UPPERCASE_LETTER) + assertTrue('<' in CharCategory.MATH_SYMBOL) + assertTrue(';' in CharCategory.OTHER_PUNCTUATION) + assertTrue('_' in CharCategory.CONNECTOR_PUNCTUATION) + assertTrue('$' in CharCategory.CURRENCY_SYMBOL) + assertTrue('\u2029' in CharCategory.PARAGRAPH_SEPARATOR) + + try { + CharCategory.valueOf(-1) + throw AssertionError() + } catch (e: IllegalArgumentException) {} + + try { + CharCategory.valueOf(31) + throw AssertionError() + } catch (e: IllegalArgumentException) {} + + try { + CharCategory.valueOf(17) + throw AssertionError() + } catch (e: IllegalArgumentException) {} +} + +fun main(args: Array) { + testIsSurrogatePair() + testToChars() + testToCodePoint() + testIsSupplementaryCodePoint() + testCase() + testCategory() +} \ No newline at end of file diff --git a/backend.native/tests/runtime/text/string_builder0.kt b/backend.native/tests/runtime/text/string_builder0.kt index dfb009bf62a..b4481a72e0e 100644 --- a/backend.native/tests/runtime/text/string_builder0.kt +++ b/backend.native/tests/runtime/text/string_builder0.kt @@ -29,8 +29,6 @@ fun assertException(body: () -> Unit) { } catch (e: IndexOutOfBoundsException) {} } -fun String.toCharArray(): CharArray = CharArray(length) { i -> this[i] } - // Insert =================================================================================================== fun testInsertString(initial: String, index: Int, toInsert: String, expected: String) { assertEquals(StringBuilder(initial).insert(index, toInsert), expected) diff --git a/runtime/src/main/cpp/KString.cpp b/runtime/src/main/cpp/KString.cpp index 3ca8a4c8469..30d3883f353 100644 --- a/runtime/src/main/cpp/KString.cpp +++ b/runtime/src/main/cpp/KString.cpp @@ -163,7 +163,6 @@ enum CharacterClass { OTHER_SYMBOL = 28, /** * Unicode category constant Pi. - * */ INITIAL_QUOTE_PUNCTUATION = 29, /** @@ -973,6 +972,10 @@ KChar Kotlin_Char_toUpperCase(KChar ch) { return towupper_Konan(ch); } +KInt Kotlin_Char_getType(KChar ch) { + return getType(ch); +} + constexpr KInt digits[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, -1, diff --git a/runtime/src/main/kotlin/konan/internal/Strings.kt b/runtime/src/main/kotlin/konan/internal/Strings.kt index 353c7df184f..af066726ae1 100644 --- a/runtime/src/main/kotlin/konan/internal/Strings.kt +++ b/runtime/src/main/kotlin/konan/internal/Strings.kt @@ -172,6 +172,12 @@ external public fun String.toUpperCase(): String @Suppress("NOTHING_TO_INLINE") external public inline fun String.toLowerCase(): String +/** + * Returns an array containing all characters of the specified string. + */ +@SymbolName("Kotlin_String_toCharArray") +external public fun String.toCharArray() : CharArray + /** * Returns a copy of this string having its first letter uppercased, or the original string, * if it's empty or already starts with an upper case letter. diff --git a/runtime/src/main/kotlin/kotlin/Char.kt b/runtime/src/main/kotlin/kotlin/Char.kt index aa43a20f740..ac558fb9890 100644 --- a/runtime/src/main/kotlin/kotlin/Char.kt +++ b/runtime/src/main/kotlin/kotlin/Char.kt @@ -106,6 +106,21 @@ public final class Char : Comparable { */ public const val MAX_SURROGATE: Char = MAX_LOW_SURROGATE + /** + * The minimum value of a supplementary code point, `\u0x10000`. Kotlin/Native specific. + */ + public const val MIN_SUPPLEMENTARY_CODE_POINT: Int = 0x10000 + + /** + * The minimum value of a Unicode code point. Kotlin/Native specific. + */ + public const val MIN_CODE_POINT = 0x000000 + + /** + * The maximum value of a Unicode code point. Kotlin/Native specific. + */ + public const val MAX_CODE_POINT = 0X10FFFF + /** * The minimum radix available for conversion to and from strings. */ diff --git a/runtime/src/main/kotlin/kotlin/collections/Collections.kt b/runtime/src/main/kotlin/kotlin/collections/Collections.kt index f1e9b91d4c4..710c4685db1 100644 --- a/runtime/src/main/kotlin/kotlin/collections/Collections.kt +++ b/runtime/src/main/kotlin/kotlin/collections/Collections.kt @@ -1275,6 +1275,17 @@ public fun > MutableList.sortDescending(): Unit { sortWith(reverseOrder()) } +/** + * Replaces each element in the list with a result of a transformation specified. + */ +public fun MutableList.replaceAll(transformation: (T) -> T) { + val it = listIterator() + while (it.hasNext()) { + val element = it.next() + it.set(transformation(element)) + } +} + /** * Returns a list of all elements sorted according to their natural sort order. */ diff --git a/runtime/src/main/kotlin/kotlin/text/Char.kt b/runtime/src/main/kotlin/kotlin/text/Char.kt index 1e08066c001..daa8618cf48 100644 --- a/runtime/src/main/kotlin/kotlin/text/Char.kt +++ b/runtime/src/main/kotlin/kotlin/text/Char.kt @@ -16,6 +16,8 @@ package kotlin.text +import kotlin.IllegalArgumentException + /** * Returns `true` if this character (Unicode code point) is defined in Unicode. */ @@ -102,6 +104,15 @@ internal fun digitOf(char: Char, radix: Int): Int = digitOfChecked(char, checkRa @SymbolName("Kotlin_Char_digitOfChecked") external internal fun digitOfChecked(char: Char, radix: Int): Int +/** + * Returns a value indicating a character's general category. + */ +public val Char.category: CharCategory get() = CharCategory.valueOf(getType()) + +/** Retrun a Unicode category of the character as an Int. */ +@SymbolName("Kotlin_Char_getType") +external internal fun Char.getType(): Int + /** * Checks whether the given [radix] is valid radix for string to number and number to string conversion. */ @@ -112,3 +123,48 @@ internal fun checkRadix(radix: Int): Int { } return radix } + +// Char.Compaion methods. Konan specific. + +// TODO: Make public when supplementary codepoints are supported. +/** Converts a unicode code point to lower case. */ +internal fun Char.Companion.toLowerCase(codePoint: Int): Int = + if (codePoint < MIN_SUPPLEMENTARY_CODE_POINT) { + codePoint.toChar().toLowerCase().toInt() + } else { + codePoint // TODO: Implement this transformation for supplementary codepoints. + } + +/** Converts a unicode code point to upper case. */ +internal fun Char.Companion.toUpperCase(codePoint: Int): Int = + if (codePoint < MIN_SUPPLEMENTARY_CODE_POINT) { + codePoint.toChar().toUpperCase().toInt() + } else { + codePoint // TODO: Implement this transformation for supplementary codepoints. + } + +/** Converts a surrogate pair to a unicode code point. Doesn't validate that the characters are a valid surrogate pair. */ +public fun Char.Companion.toCodePoint(high: Char, low: Char): Int = + (((high - MIN_HIGH_SURROGATE) shl 10) or (low - MIN_LOW_SURROGATE)) + 0x10000 + +/** Checks if the codepoint specified is a supplementary codepoint or not. */ +public fun Char.Companion.isSupplementaryCodePoint(codepoint: Int): Boolean = + codepoint in MIN_SUPPLEMENTARY_CODE_POINT..MAX_CODE_POINT + +public fun Char.Companion.isSurrogatePair(high: Char, low: Char): Boolean = high.isHighSurrogate() && low.isLowSurrogate() + +/** + * Converts the codepoint specified to a char array. If the codepoint is not supplementary, the method will + * return an array with one element otherwise it will return an array A with a high surrogate in A[0] and + * a low surrogate in A[1]. + */ +public fun Char.Companion.toChars(codePoint: Int): CharArray = + when { + codePoint in 0 until MIN_SUPPLEMENTARY_CODE_POINT -> charArrayOf(codePoint.toChar()) + codePoint in MIN_SUPPLEMENTARY_CODE_POINT..MAX_CODE_POINT -> { + val low = ((codePoint - 0x10000) and 0x3FF) + MIN_LOW_SURROGATE.toInt() + val high = (((codePoint - 0x10000) ushr 10) and 0x3FF) + MIN_HIGH_SURROGATE.toInt() + charArrayOf(high.toChar(), low.toChar()) + } + else -> throw IllegalArgumentException() + } diff --git a/runtime/src/main/kotlin/kotlin/text/CharCategory.kt b/runtime/src/main/kotlin/kotlin/text/CharCategory.kt new file mode 100644 index 00000000000..fae60f25826 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/CharCategory.kt @@ -0,0 +1,171 @@ +package kotlin.text + +// The values duplicate constants defined in KString.cpp. +/** + * Represents the character general category in the Unicode specification. + */ +public enum class CharCategory(public val value: Int, public val code: String) { + /** + * General category "Cn" in the Unicode specification. + */ + UNASSIGNED(0, "Cn"), + + /** + * General category "Lu" in the Unicode specification. + */ + UPPERCASE_LETTER(1, "Lu"), + + /** + * General category "Ll" in the Unicode specification. + */ + LOWERCASE_LETTER(2, "Ll"), + + /** + * General category "Lt" in the Unicode specification. + */ + TITLECASE_LETTER(3, "Lt"), + + /** + * General category "Lm" in the Unicode specification. + */ + MODIFIER_LETTER(4, "Lm"), + + /** + * General category "Lo" in the Unicode specification. + */ + OTHER_LETTER(5, "Lo"), + + /** + * General category "Mn" in the Unicode specification. + */ + NON_SPACING_MARK(6, "Mn"), + + /** + * General category "Me" in the Unicode specification. + */ + ENCLOSING_MARK(7, "Me"), + + /** + * General category "Mc" in the Unicode specification. + */ + COMBINING_SPACING_MARK(8, "Mc"), + + /** + * General category "Nd" in the Unicode specification. + */ + DECIMAL_DIGIT_NUMBER(9, "Nd"), + + /** + * General category "Nl" in the Unicode specification. + */ + LETTER_NUMBER(10, "Nl"), + + /** + * General category "No" in the Unicode specification. + */ + OTHER_NUMBER(11, "No"), + + /** + * General category "Zs" in the Unicode specification. + */ + SPACE_SEPARATOR(12, "Zs"), + + /** + * General category "Zl" in the Unicode specification. + */ + LINE_SEPARATOR(13, "Zl"), + + /** + * General category "Zp" in the Unicode specification. + */ + PARAGRAPH_SEPARATOR(14, "Zp"), + + /** + * General category "Cc" in the Unicode specification. + */ + CONTROL(15, "Cc"), + + /** + * General category "Cf" in the Unicode specification. + */ + FORMAT(16, "Cf"), + + /** + * General category "Co" in the Unicode specification. + */ + PRIVATE_USE(18, "Co"), + + /** + * General category "Cs" in the Unicode specification. + */ + SURROGATE(19, "Cs"), + + /** + * General category "Pd" in the Unicode specification. + */ + DASH_PUNCTUATION(20, "Pd"), + + /** + * General category "Ps" in the Unicode specification. + */ + START_PUNCTUATION(21, "Ps"), + + /** + * General category "Pe" in the Unicode specification. + */ + END_PUNCTUATION(22, "Pe"), + + /** + * General category "Pc" in the Unicode specification. + */ + CONNECTOR_PUNCTUATION(23, "Pc"), + + /** + * General category "Po" in the Unicode specification. + */ + OTHER_PUNCTUATION(24, "Po"), + + /** + * General category "Sm" in the Unicode specification. + */ + MATH_SYMBOL(25, "Sm"), + + /** + * General category "Sc" in the Unicode specification. + */ + CURRENCY_SYMBOL(26, "Sc"), + + /** + * General category "Sk" in the Unicode specification. + */ + MODIFIER_SYMBOL(27, "Sk"), + + /** + * General category "So" in the Unicode specification. + */ + OTHER_SYMBOL(28, "So"), + + /** + * General category "Pi" in the Unicode specification. + */ + INITIAL_QUOTE_PUNCTUATION(29, "Pi"), + + /** + * General category "Pf" in the Unicode specification. + */ + FINAL_QUOTE_PUNCTUATION(30, "Pf"); + + /** + * Returns `true` if [char] character belongs to this category. + */ + public operator fun contains(char: Char): Boolean = char.getType() == this.value + + public companion object { + public fun valueOf(category: Int): CharCategory = + when { + category >=0 && category <= 16 -> values()[category] + category >= 18 && category <= 30 -> values()[category - 1] + else -> throw IllegalArgumentException("Category #$category is not defined.") + } + } +} \ No newline at end of file diff --git a/runtime/src/main/kotlin/kotlin/text/StringBuilder.kt b/runtime/src/main/kotlin/kotlin/text/StringBuilder.kt index d5e9818a25c..0f934a5d8ec 100644 --- a/runtime/src/main/kotlin/kotlin/text/StringBuilder.kt +++ b/runtime/src/main/kotlin/kotlin/text/StringBuilder.kt @@ -26,9 +26,6 @@ external fun toUtf8Array(string: String, start: Int, size: Int) : ByteArray @SymbolName("Kotlin_String_fromCharArray") external fun fromCharArray(array: CharArray, start: Int, size: Int) : String -@SymbolName("Kotlin_String_toCharArray") -external fun toCharArray(string: String) : CharArray - /** * Builds new string by populating newly created [StringBuilder] using provided [builderAction] * and then converting it to [String]. @@ -59,7 +56,7 @@ class StringBuilder private constructor ( constructor(capacity: Int) : this(CharArray(capacity)) - constructor(string: String) : this(toCharArray(string)) { + constructor(string: String) : this(string.toCharArray()) { length = array.size } @@ -217,7 +214,7 @@ class StringBuilder private constructor ( return this } - fun insert(index: Int, string: String): StringBuilder = insert(index, toCharArray(string)) + fun insert(index: Int, string: String): StringBuilder = insert(index, string.toCharArray()) fun insert(index: Int, value: Boolean) = insert(index, value.toString()) fun insert(index: Int, value: Byte) = insert(index, value.toString()) @@ -260,7 +257,7 @@ class StringBuilder private constructor ( return this } - fun append(it: String): StringBuilder = append(toCharArray(it)) + fun append(it: String): StringBuilder = append(it.toCharArray()) fun append(it: Boolean) = append(it.toString()) fun append(it: Byte) = append(it.toString())