azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

stdlib: Improve character API.

Browse Source 
The patch adds methods for surrogate characters processing and Unicode
category support needed for regular expression library.
This commit is contained in:
Ilya Matveev
2017-06-08 14:04:23 +07:00
committed by ilmat192
parent 96e807697a
commit e64a5f42f0
11 changed files with 385 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend.native/tests/build.gradle
+5
View File
@@ -1268,6 +1268,11 @@ task to_string0(type: RunKonanTest) {
source = "runtime/text/to_string0.kt"
}
task chars0(type: RunKonanTest) {
source = "runtime/text/chars0.kt"
expectedExitStatus = 0
}
task catch1(type: RunKonanTest) {
goldValue = "Before\nCaught Throwable\nDone\n"
source = "runtime/exceptions/catch1.kt"
backend.native/tests/runtime/basic/tostring2.kt
+1 -1
View File
@@ -1,6 +1,6 @@
fun main(args : Array<String>) {
val hello = "Hello"
val array = toCharArray(hello)
val array = hello.toCharArray()
for (ch in array) {
print(ch)
print(" ")
backend.native/tests/runtime/text/chars0.kt
+113
View File
@@ -0,0 +1,113 @@
fun assertTrue(v: Boolean) = if (!v) throw AssertionError() else Unit
fun assertFalse(v: Boolean) = if (v) throw AssertionError() else Unit
fun assertEquals(a: Int, b: Int) { if (a != b) throw AssertionError() }
fun testIsSupplementaryCodePoint() {
assertFalse(Char.isSupplementaryCodePoint(-1))
for (c in 0..0xFFFF) {
assertFalse(Char.isSupplementaryCodePoint(c.toInt()))
}
for (c in 0xFFFF + 1..0x10FFFF) {
assertTrue(Char.isSupplementaryCodePoint(c))
}
assertFalse(Char.isSupplementaryCodePoint(0x10FFFF + 1))
}
fun testIsSurrogatePair() {
assertFalse(Char.isSurrogatePair('\u0000', '\u0000'))
assertFalse(Char.isSurrogatePair('\u0000', '\uDC00'))
assertTrue( Char.isSurrogatePair('\uD800', '\uDC00'))
assertTrue( Char.isSurrogatePair('\uD800', '\uDFFF'))
assertTrue( Char.isSurrogatePair('\uDBFF', '\uDFFF'))
assertFalse(Char.isSurrogatePair('\uDBFF', '\uF000'))
}
fun testToChars() {
assertTrue(charArrayOf('\uD800', '\uDC00').contentEquals(Char.toChars(0x010000)))
assertTrue(charArrayOf('\uD800', '\uDC01').contentEquals(Char.toChars(0x010001)))
assertTrue(charArrayOf('\uD801', '\uDC01').contentEquals(Char.toChars(0x010401)))
assertTrue(charArrayOf('\uDBFF', '\uDFFF').contentEquals(Char.toChars(0x10FFFF)))
try {
Char.toChars(Int.MAX_VALUE)
throw AssertionError()
} catch (e: IllegalArgumentException) {}
}
fun testToCodePoint() {
assertEquals(0x010000, Char.toCodePoint('\uD800', '\uDC00'))
assertEquals(0x010001, Char.toCodePoint('\uD800', '\uDC01'))
assertEquals(0x010401, Char.toCodePoint('\uD801', '\uDC01'))
assertEquals(0x10FFFF, Char.toCodePoint('\uDBFF', '\uDFFF'))
}
// TODO: Uncomment when such operations are supported for supplementary codepoints and the API is public.
fun testCase() {
/*
assertEquals('A'.toInt(), Char.toUpperCase('a'.toInt()))
assertEquals('A'.toInt(), Char.toUpperCase('A'.toInt()))
assertEquals('1'.toInt(), Char.toUpperCase('1'.toInt()))
assertEquals('a'.toInt(), Char.toLowerCase('A'.toInt()))
assertEquals('a'.toInt(), Char.toLowerCase('a'.toInt()))
assertEquals('1'.toInt(), Char.toLowerCase('1'.toInt()))
assertEquals(0x010400, Char.toUpperCase(0x010428))
assertEquals(0x010400, Char.toUpperCase(0x010400))
assertEquals(0x10FFFF, Char.toUpperCase(0x10FFFF))
assertEquals(0x110000, Char.toUpperCase(0x110000))
assertEquals(0x010428, Char.toLowerCase(0x010400))
assertEquals(0x010428, Char.toLowerCase(0x010428))
assertEquals(0x10FFFF, Char.toLowerCase(0x10FFFF))
assertEquals(0x110000, Char.toLowerCase(0x110000))
*/
}
fun testCategory() {
assertEquals('\n'.category.value, CharCategory.CONTROL.value)
assertEquals('1'.category.value, CharCategory.DECIMAL_DIGIT_NUMBER.value)
assertEquals(' '.category.value, CharCategory.SPACE_SEPARATOR.value)
assertEquals('a'.category.value, CharCategory.LOWERCASE_LETTER.value)
assertEquals('A'.category.value, CharCategory.UPPERCASE_LETTER.value)
assertEquals('<'.category.value, CharCategory.MATH_SYMBOL.value)
assertEquals(';'.category.value, CharCategory.OTHER_PUNCTUATION.value)
assertEquals('_'.category.value, CharCategory.CONNECTOR_PUNCTUATION.value)
assertEquals('$'.category.value, CharCategory.CURRENCY_SYMBOL.value)
assertEquals('\u2029'.category.value, CharCategory.PARAGRAPH_SEPARATOR.value)
assertTrue('\n' in CharCategory.CONTROL)
assertTrue('1' in CharCategory.DECIMAL_DIGIT_NUMBER)
assertTrue(' ' in CharCategory.SPACE_SEPARATOR)
assertTrue('a' in CharCategory.LOWERCASE_LETTER)
assertTrue('A' in CharCategory.UPPERCASE_LETTER)
assertTrue('<' in CharCategory.MATH_SYMBOL)
assertTrue(';' in CharCategory.OTHER_PUNCTUATION)
assertTrue('_' in CharCategory.CONNECTOR_PUNCTUATION)
assertTrue('$' in CharCategory.CURRENCY_SYMBOL)
assertTrue('\u2029' in CharCategory.PARAGRAPH_SEPARATOR)
try {
CharCategory.valueOf(-1)
throw AssertionError()
} catch (e: IllegalArgumentException) {}
try {
CharCategory.valueOf(31)
throw AssertionError()
} catch (e: IllegalArgumentException) {}
try {
CharCategory.valueOf(17)
throw AssertionError()
} catch (e: IllegalArgumentException) {}
}
fun main(args: Array<String>) {
testIsSurrogatePair()
testToChars()
testToCodePoint()
testIsSupplementaryCodePoint()
testCase()
testCategory()
}
backend.native/tests/runtime/text/string_builder0.kt
-2
View File
@@ -29,8 +29,6 @@ fun assertException(body: () -> Unit) {
} catch (e: IndexOutOfBoundsException) {}
}
fun String.toCharArray(): CharArray = CharArray(length) { i -> this[i] }
// Insert ===================================================================================================
fun testInsertString(initial: String, index: Int, toInsert: String, expected: String) {
assertEquals(StringBuilder(initial).insert(index, toInsert), expected)
runtime/src/main/cpp/KString.cpp
+4 -1
View File
@@ -163,7 +163,6 @@ enum CharacterClass {
OTHER_SYMBOL = 28,
/**
* Unicode category constant Pi.
*
*/
INITIAL_QUOTE_PUNCTUATION = 29,
/**
@@ -973,6 +972,10 @@ KChar Kotlin_Char_toUpperCase(KChar ch) {
return towupper_Konan(ch);
}
KInt Kotlin_Char_getType(KChar ch) {
return getType(ch);
}
constexpr KInt digits[] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
-1, -1, -1, -1, -1, -1, -1,
runtime/src/main/kotlin/konan/internal/Strings.kt
+6
View File
@@ -172,6 +172,12 @@ external public fun String.toUpperCase(): String
@Suppress("NOTHING_TO_INLINE")
external public inline fun String.toLowerCase(): String
/**
* Returns an array containing all characters of the specified string.
*/
@SymbolName("Kotlin_String_toCharArray")
external public fun String.toCharArray() : CharArray
/**
* Returns a copy of this string having its first letter uppercased, or the original string,
* if it's empty or already starts with an upper case letter.
runtime/src/main/kotlin/kotlin/Char.kt
+15
View File
@@ -106,6 +106,21 @@ public final class Char : Comparable<Char> {
*/
public const val MAX_SURROGATE: Char = MAX_LOW_SURROGATE
/**
* The minimum value of a supplementary code point, `\u0x10000`. Kotlin/Native specific.
*/
public const val MIN_SUPPLEMENTARY_CODE_POINT: Int = 0x10000
/**
* The minimum value of a Unicode code point. Kotlin/Native specific.
*/
public const val MIN_CODE_POINT = 0x000000
/**
* The maximum value of a Unicode code point. Kotlin/Native specific.
*/
public const val MAX_CODE_POINT = 0X10FFFF
/**
* The minimum radix available for conversion to and from strings.
*/
runtime/src/main/kotlin/kotlin/collections/Collections.kt
+11
View File
@@ -1275,6 +1275,17 @@ public fun <T : Comparable<T>> MutableList<T>.sortDescending(): Unit {
sortWith(reverseOrder())
}
/**
* Replaces each element in the list with a result of a transformation specified.
*/
public fun <T> MutableList<T>.replaceAll(transformation: (T) -> T) {
val it = listIterator()
while (it.hasNext()) {
val element = it.next()
it.set(transformation(element))
}
}
/**
* Returns a list of all elements sorted according to their natural sort order.
*/
runtime/src/main/kotlin/kotlin/text/Char.kt
+56
View File
@@ -16,6 +16,8 @@
package kotlin.text
import kotlin.IllegalArgumentException
/**
* Returns `true` if this character (Unicode code point) is defined in Unicode.
*/
@@ -102,6 +104,15 @@ internal fun digitOf(char: Char, radix: Int): Int = digitOfChecked(char, checkRa
@SymbolName("Kotlin_Char_digitOfChecked")
external internal fun digitOfChecked(char: Char, radix: Int): Int
/**
* Returns a value indicating a character's general category.
*/
public val Char.category: CharCategory get() = CharCategory.valueOf(getType())
/** Retrun a Unicode category of the character as an Int. */
@SymbolName("Kotlin_Char_getType")
external internal fun Char.getType(): Int
/**
* Checks whether the given [radix] is valid radix for string to number and number to string conversion.
*/
@@ -112,3 +123,48 @@ internal fun checkRadix(radix: Int): Int {
}
return radix
}
// Char.Compaion methods. Konan specific.
// TODO: Make public when supplementary codepoints are supported.
/** Converts a unicode code point to lower case. */
internal fun Char.Companion.toLowerCase(codePoint: Int): Int =
if (codePoint < MIN_SUPPLEMENTARY_CODE_POINT) {
codePoint.toChar().toLowerCase().toInt()
} else {
codePoint // TODO: Implement this transformation for supplementary codepoints.
}
/** Converts a unicode code point to upper case. */
internal fun Char.Companion.toUpperCase(codePoint: Int): Int =
if (codePoint < MIN_SUPPLEMENTARY_CODE_POINT) {
codePoint.toChar().toUpperCase().toInt()
} else {
codePoint // TODO: Implement this transformation for supplementary codepoints.
}
/** Converts a surrogate pair to a unicode code point. Doesn't validate that the characters are a valid surrogate pair. */
public fun Char.Companion.toCodePoint(high: Char, low: Char): Int =
(((high - MIN_HIGH_SURROGATE) shl 10) or (low - MIN_LOW_SURROGATE)) + 0x10000
/** Checks if the codepoint specified is a supplementary codepoint or not. */
public fun Char.Companion.isSupplementaryCodePoint(codepoint: Int): Boolean =
codepoint in MIN_SUPPLEMENTARY_CODE_POINT..MAX_CODE_POINT
public fun Char.Companion.isSurrogatePair(high: Char, low: Char): Boolean = high.isHighSurrogate() && low.isLowSurrogate()
/**
* Converts the codepoint specified to a char array. If the codepoint is not supplementary, the method will
* return an array with one element otherwise it will return an array A with a high surrogate in A[0] and
* a low surrogate in A[1].
*/
public fun Char.Companion.toChars(codePoint: Int): CharArray =
when {
codePoint in 0 until MIN_SUPPLEMENTARY_CODE_POINT -> charArrayOf(codePoint.toChar())
codePoint in MIN_SUPPLEMENTARY_CODE_POINT..MAX_CODE_POINT -> {
val low = ((codePoint - 0x10000) and 0x3FF) + MIN_LOW_SURROGATE.toInt()
val high = (((codePoint - 0x10000) ushr 10) and 0x3FF) + MIN_HIGH_SURROGATE.toInt()
charArrayOf(high.toChar(), low.toChar())
}
else -> throw IllegalArgumentException()
}
runtime/src/main/kotlin/kotlin/text/CharCategory.kt
+171
View File
@@ -0,0 +1,171 @@
package kotlin.text
// The values duplicate constants defined in KString.cpp.
/**
* Represents the character general category in the Unicode specification.
*/
public enum class CharCategory(public val value: Int, public val code: String) {
/**
* General category "Cn" in the Unicode specification.
*/
UNASSIGNED(0, "Cn"),
/**
* General category "Lu" in the Unicode specification.
*/
UPPERCASE_LETTER(1, "Lu"),
/**
* General category "Ll" in the Unicode specification.
*/
LOWERCASE_LETTER(2, "Ll"),
/**
* General category "Lt" in the Unicode specification.
*/
TITLECASE_LETTER(3, "Lt"),
/**
* General category "Lm" in the Unicode specification.
*/
MODIFIER_LETTER(4, "Lm"),
/**
* General category "Lo" in the Unicode specification.
*/
OTHER_LETTER(5, "Lo"),
/**
* General category "Mn" in the Unicode specification.
*/
NON_SPACING_MARK(6, "Mn"),
/**
* General category "Me" in the Unicode specification.
*/
ENCLOSING_MARK(7, "Me"),
/**
* General category "Mc" in the Unicode specification.
*/
COMBINING_SPACING_MARK(8, "Mc"),
/**
* General category "Nd" in the Unicode specification.
*/
DECIMAL_DIGIT_NUMBER(9, "Nd"),
/**
* General category "Nl" in the Unicode specification.
*/
LETTER_NUMBER(10, "Nl"),
/**
* General category "No" in the Unicode specification.
*/
OTHER_NUMBER(11, "No"),
/**
* General category "Zs" in the Unicode specification.
*/
SPACE_SEPARATOR(12, "Zs"),
/**
* General category "Zl" in the Unicode specification.
*/
LINE_SEPARATOR(13, "Zl"),
/**
* General category "Zp" in the Unicode specification.
*/
PARAGRAPH_SEPARATOR(14, "Zp"),
/**
* General category "Cc" in the Unicode specification.
*/
CONTROL(15, "Cc"),
/**
* General category "Cf" in the Unicode specification.
*/
FORMAT(16, "Cf"),
/**
* General category "Co" in the Unicode specification.
*/
PRIVATE_USE(18, "Co"),
/**
* General category "Cs" in the Unicode specification.
*/
SURROGATE(19, "Cs"),
/**
* General category "Pd" in the Unicode specification.
*/
DASH_PUNCTUATION(20, "Pd"),
/**
* General category "Ps" in the Unicode specification.
*/
START_PUNCTUATION(21, "Ps"),
/**
* General category "Pe" in the Unicode specification.
*/
END_PUNCTUATION(22, "Pe"),
/**
* General category "Pc" in the Unicode specification.
*/
CONNECTOR_PUNCTUATION(23, "Pc"),
/**
* General category "Po" in the Unicode specification.
*/
OTHER_PUNCTUATION(24, "Po"),
/**
* General category "Sm" in the Unicode specification.
*/
MATH_SYMBOL(25, "Sm"),
/**
* General category "Sc" in the Unicode specification.
*/
CURRENCY_SYMBOL(26, "Sc"),
/**
* General category "Sk" in the Unicode specification.
*/
MODIFIER_SYMBOL(27, "Sk"),
/**
* General category "So" in the Unicode specification.
*/
OTHER_SYMBOL(28, "So"),
/**
* General category "Pi" in the Unicode specification.
*/
INITIAL_QUOTE_PUNCTUATION(29, "Pi"),
/**
* General category "Pf" in the Unicode specification.
*/
FINAL_QUOTE_PUNCTUATION(30, "Pf");
/**
* Returns `true` if [char] character belongs to this category.
*/
public operator fun contains(char: Char): Boolean = char.getType() == this.value
public companion object {
public fun valueOf(category: Int): CharCategory =
when {
category >=0 && category <= 16 -> values()[category]
category >= 18 && category <= 30 -> values()[category - 1]
else -> throw IllegalArgumentException("Category #$category is not defined.")
}
}
}
runtime/src/main/kotlin/kotlin/text/StringBuilder.kt
+3 -6
View File
@@ -26,9 +26,6 @@ external fun toUtf8Array(string: String, start: Int, size: Int) : ByteArray
@SymbolName("Kotlin_String_fromCharArray")
external fun fromCharArray(array: CharArray, start: Int, size: Int) : String
@SymbolName("Kotlin_String_toCharArray")
external fun toCharArray(string: String) : CharArray
/**
* Builds new string by populating newly created [StringBuilder] using provided [builderAction]
* and then converting it to [String].
@@ -59,7 +56,7 @@ class StringBuilder private constructor (
constructor(capacity: Int) : this(CharArray(capacity))
constructor(string: String) : this(toCharArray(string)) {
constructor(string: String) : this(string.toCharArray()) {
length = array.size
}
@@ -217,7 +214,7 @@ class StringBuilder private constructor (
return this
}
fun insert(index: Int, string: String): StringBuilder = insert(index, toCharArray(string))
fun insert(index: Int, string: String): StringBuilder = insert(index, string.toCharArray())
fun insert(index: Int, value: Boolean) = insert(index, value.toString())
fun insert(index: Int, value: Byte) = insert(index, value.toString())
@@ -260,7 +257,7 @@ class StringBuilder private constructor (
return this
}
fun append(it: String): StringBuilder = append(toCharArray(it))
fun append(it: String): StringBuilder = append(it.toCharArray())
fun append(it: Boolean) = append(it.toString())
fun append(it: Byte) = append(it.toString())