diff --git a/kotlin-native/runtime/src/main/kotlin/generated/_LetterChars.kt b/kotlin-native/runtime/src/main/kotlin/generated/_LetterChars.kt index 5b54e6ed224..751f80ba42e 100644 --- a/kotlin-native/runtime/src/main/kotlin/generated/_LetterChars.kt +++ b/kotlin-native/runtime/src/main/kotlin/generated/_LetterChars.kt @@ -68,17 +68,17 @@ internal fun Char.isLetterImpl(): Boolean { } /** - * Returns `true` if this character is a lower case letter. + * Returns `true` if this character is a lower case letter, or it has contributory property Other_Lowercase. */ internal fun Char.isLowerCaseImpl(): Boolean { - return getLetterType() == 1 + return getLetterType() == 1 || code.isOtherLowercase() } /** - * Returns `true` if this character is an upper case letter. + * Returns `true` if this character is an upper case letter, or it has contributory property Other_Uppercase. */ internal fun Char.isUpperCaseImpl(): Boolean { - return getLetterType() == 2 + return getLetterType() == 2 || code.isOtherUppercase() } /** diff --git a/kotlin-native/runtime/src/main/kotlin/generated/_OtherLowercaseChars.kt b/kotlin-native/runtime/src/main/kotlin/generated/_OtherLowercaseChars.kt new file mode 100644 index 00000000000..d4beacb7154 --- /dev/null +++ b/kotlin-native/runtime/src/main/kotlin/generated/_OtherLowercaseChars.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package kotlin.text + +// +// NOTE: THIS FILE IS AUTO-GENERATED by the GenerateUnicodeData.kt +// See: https://github.com/JetBrains/kotlin/tree/master/libraries/stdlib +// + +@SharedImmutable +private val otherLowerStart = intArrayOf( + 0x00aa, 0x00ba, 0x02b0, 0x02c0, 0x02e0, 0x0345, 0x037a, 0x1d2c, 0x1d78, 0x1d9b, 0x2071, 0x207f, 0x2090, 0x2170, 0x24d0, 0x2c7c, 0xa69c, 0xa770, 0xa7f8, 0xab5c, +) +@SharedImmutable +private val otherLowerLength = intArrayOf( + 1, 1, 9, 2, 5, 1, 1, 63, 1, 37, 1, 1, 13, 16, 26, 2, 2, 1, 2, 4, +) + +internal fun Int.isOtherLowercase(): Boolean { + val index = binarySearchRange(otherLowerStart, this) + return index >= 0 && this < otherLowerStart[index] + otherLowerLength[index] +} diff --git a/kotlin-native/runtime/src/main/kotlin/generated/_OtherUppercaseChars.kt b/kotlin-native/runtime/src/main/kotlin/generated/_OtherUppercaseChars.kt new file mode 100644 index 00000000000..b5a4b08083b --- /dev/null +++ b/kotlin-native/runtime/src/main/kotlin/generated/_OtherUppercaseChars.kt @@ -0,0 +1,19 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package kotlin.text + +// +// NOTE: THIS FILE IS AUTO-GENERATED by the GenerateUnicodeData.kt +// See: https://github.com/JetBrains/kotlin/tree/master/libraries/stdlib +// + +internal fun Int.isOtherUppercase(): Boolean { + return this in 0x2160..0x216f + || this in 0x24b6..0x24cf + || this in 0x1f130..0x1f149 + || this in 0x1f150..0x1f169 + || this in 0x1f170..0x1f189 +} diff --git a/kotlin-native/runtime/src/main/kotlin/generated/_StringLowercase.kt b/kotlin-native/runtime/src/main/kotlin/generated/_StringLowercase.kt index a6b19050e37..2427fdde2e9 100644 --- a/kotlin-native/runtime/src/main/kotlin/generated/_StringLowercase.kt +++ b/kotlin-native/runtime/src/main/kotlin/generated/_StringLowercase.kt @@ -12,17 +12,13 @@ package kotlin.text @SharedImmutable private val casedStart = intArrayOf( - 0x00aa, 0x00ba, 0x02b0, 0x02c0, 0x02e0, 0x0345, 0x037a, 0x1d2c, 0x1d78, 0x1d9b, 0x2071, 0x207f, 0x2090, 0x2160, 0x2170, 0x24b6, 0x24d0, 0x2c7c, 0xa69c, 0xa770, - 0xa7f8, 0xab5c, 0x10400, 0x104b0, 0x104d8, 0x10c80, 0x10cc0, 0x118a0, 0x16e40, 0x1d400, 0x1d456, 0x1d49e, 0x1d4a2, 0x1d4a5, 0x1d4a9, 0x1d4ae, 0x1d4bb, 0x1d4bd, 0x1d4c5, 0x1d507, - 0x1d50d, 0x1d516, 0x1d51e, 0x1d53b, 0x1d540, 0x1d546, 0x1d54a, 0x1d552, 0x1d6a8, 0x1d6c2, 0x1d6dc, 0x1d6fc, 0x1d716, 0x1d736, 0x1d750, 0x1d770, 0x1d78a, 0x1d7aa, 0x1d7c4, 0x1e900, - 0x1f130, 0x1f150, 0x1f170, + 0x10400, 0x104b0, 0x104d8, 0x10c80, 0x10cc0, 0x118a0, 0x16e40, 0x1d400, 0x1d456, 0x1d49e, 0x1d4a2, 0x1d4a5, 0x1d4a9, 0x1d4ae, 0x1d4bb, 0x1d4bd, 0x1d4c5, 0x1d507, 0x1d50d, 0x1d516, + 0x1d51e, 0x1d53b, 0x1d540, 0x1d546, 0x1d54a, 0x1d552, 0x1d6a8, 0x1d6c2, 0x1d6dc, 0x1d6fc, 0x1d716, 0x1d736, 0x1d750, 0x1d770, 0x1d78a, 0x1d7aa, 0x1d7c4, 0x1e900, ) @SharedImmutable private val casedEnd = intArrayOf( - 0x00aa, 0x00ba, 0x02b8, 0x02c1, 0x02e4, 0x0345, 0x037a, 0x1d6a, 0x1d78, 0x1dbf, 0x2071, 0x207f, 0x209c, 0x216f, 0x217f, 0x24cf, 0x24e9, 0x2c7d, 0xa69d, 0xa770, - 0xa7f9, 0xab5f, 0x1044f, 0x104d3, 0x104fb, 0x10cb2, 0x10cf2, 0x118df, 0x16e7f, 0x1d454, 0x1d49c, 0x1d49f, 0x1d4a2, 0x1d4a6, 0x1d4ac, 0x1d4b9, 0x1d4bb, 0x1d4c3, 0x1d505, 0x1d50a, - 0x1d514, 0x1d51c, 0x1d539, 0x1d53e, 0x1d544, 0x1d546, 0x1d550, 0x1d6a5, 0x1d6c0, 0x1d6da, 0x1d6fa, 0x1d714, 0x1d734, 0x1d74e, 0x1d76e, 0x1d788, 0x1d7a8, 0x1d7c2, 0x1d7cb, 0x1e943, - 0x1f149, 0x1f169, 0x1f189, + 0x1044f, 0x104d3, 0x104fb, 0x10cb2, 0x10cf2, 0x118df, 0x16e7f, 0x1d454, 0x1d49c, 0x1d49f, 0x1d4a2, 0x1d4a6, 0x1d4ac, 0x1d4b9, 0x1d4bb, 0x1d4c3, 0x1d505, 0x1d50a, 0x1d514, 0x1d51c, + 0x1d539, 0x1d53e, 0x1d544, 0x1d546, 0x1d550, 0x1d6a5, 0x1d6c0, 0x1d6da, 0x1d6fa, 0x1d714, 0x1d734, 0x1d74e, 0x1d76e, 0x1d788, 0x1d7a8, 0x1d7c2, 0x1d7cb, 0x1e943, ) // Lu + Ll + Lt + Other_Lowercase + Other_Uppercase (PropList.txt of Unicode Character Database files) @@ -35,6 +31,9 @@ internal fun Int.isCased(): Boolean { CharCategory.TITLECASE_LETTER.value -> return true } } + if (isOtherUppercase() || isOtherLowercase()) { + return true + } val index = binarySearchRange(casedStart, this) return index >= 0 && this <= casedEnd[index] } diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/text/Char.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/text/Char.kt index b671caa9d58..e1ccb919336 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/text/Char.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/text/Char.kt @@ -101,9 +101,10 @@ external public actual fun Char.isISOControl(): Boolean public actual fun Char.isWhitespace(): Boolean = isWhitespaceImpl() /** - * Returns `true` if this character is an upper case letter. + * Returns `true` if this character is upper case. * - * A character is considered to be an upper case letter if its [category] is [CharCategory.UPPERCASE_LETTER]. + * A character is considered to be an upper case character if its [category] is [CharCategory.UPPERCASE_LETTER], + * or it has contributory property Other_Uppercase as defined by the Unicode Standard. * * @sample samples.text.Chars.isUpperCase */ @@ -118,9 +119,10 @@ public actual fun Char.isUpperCase(): Boolean { } /** - * Returns `true` if this character is a lower case letter. + * Returns `true` if this character is lower case. * - * A character is considered to be a lower case letter if its [category] is [CharCategory.LOWERCASE_LETTER]. + * A character is considered to be a lower case character if its [category] is [CharCategory.LOWERCASE_LETTER], + * or it has contributory property Other_Lowercase as defined by the Unicode Standard. * * @sample samples.text.Chars.isLowerCase */ diff --git a/libraries/stdlib/js-ir/src/generated/_LetterChars.kt b/libraries/stdlib/js-ir/src/generated/_LetterChars.kt index 6fa6dd97d75..1fc593c5ec8 100644 --- a/libraries/stdlib/js-ir/src/generated/_LetterChars.kt +++ b/libraries/stdlib/js-ir/src/generated/_LetterChars.kt @@ -51,17 +51,17 @@ internal fun Char.isLetterImpl(): Boolean { } /** - * Returns `true` if this character is a lower case letter. + * Returns `true` if this character is a lower case letter, or it has contributory property Other_Lowercase. */ internal fun Char.isLowerCaseImpl(): Boolean { - return getLetterType() == 1 + return getLetterType() == 1 || code.isOtherLowercase() } /** - * Returns `true` if this character is an upper case letter. + * Returns `true` if this character is an upper case letter, or it has contributory property Other_Uppercase. */ internal fun Char.isUpperCaseImpl(): Boolean { - return getLetterType() == 2 + return getLetterType() == 2 || code.isOtherUppercase() } /** diff --git a/libraries/stdlib/js-ir/src/generated/_OtherLowercaseChars.kt b/libraries/stdlib/js-ir/src/generated/_OtherLowercaseChars.kt new file mode 100644 index 00000000000..08cddc0315f --- /dev/null +++ b/libraries/stdlib/js-ir/src/generated/_OtherLowercaseChars.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package kotlin.text + +// +// NOTE: THIS FILE IS AUTO-GENERATED by the GenerateUnicodeData.kt +// See: https://github.com/JetBrains/kotlin/tree/master/libraries/stdlib +// + +private object OtherLowercase { + internal val otherLowerStart = intArrayOf( + 0x00aa, 0x00ba, 0x02b0, 0x02c0, 0x02e0, 0x0345, 0x037a, 0x1d2c, 0x1d78, 0x1d9b, 0x2071, 0x207f, 0x2090, 0x2170, 0x24d0, 0x2c7c, 0xa69c, 0xa770, 0xa7f8, 0xab5c, + ) + internal val otherLowerLength = intArrayOf( + 1, 1, 9, 2, 5, 1, 1, 63, 1, 37, 1, 1, 13, 16, 26, 2, 2, 1, 2, 4, + ) +} + +internal fun Int.isOtherLowercase(): Boolean { + val index = binarySearchRange(OtherLowercase.otherLowerStart, this) + return index >= 0 && this < OtherLowercase.otherLowerStart[index] + OtherLowercase.otherLowerLength[index] +} diff --git a/libraries/stdlib/js-ir/src/generated/_OtherUppercaseChars.kt b/libraries/stdlib/js-ir/src/generated/_OtherUppercaseChars.kt new file mode 100644 index 00000000000..d26518cb207 --- /dev/null +++ b/libraries/stdlib/js-ir/src/generated/_OtherUppercaseChars.kt @@ -0,0 +1,16 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package kotlin.text + +// +// NOTE: THIS FILE IS AUTO-GENERATED by the GenerateUnicodeData.kt +// See: https://github.com/JetBrains/kotlin/tree/master/libraries/stdlib +// + +internal fun Int.isOtherUppercase(): Boolean { + return this in 0x2160..0x216f + || this in 0x24b6..0x24cf +} diff --git a/libraries/stdlib/js/src/generated/_LetterChars.kt b/libraries/stdlib/js/src/generated/_LetterChars.kt index 6fa6dd97d75..1fc593c5ec8 100644 --- a/libraries/stdlib/js/src/generated/_LetterChars.kt +++ b/libraries/stdlib/js/src/generated/_LetterChars.kt @@ -51,17 +51,17 @@ internal fun Char.isLetterImpl(): Boolean { } /** - * Returns `true` if this character is a lower case letter. + * Returns `true` if this character is a lower case letter, or it has contributory property Other_Lowercase. */ internal fun Char.isLowerCaseImpl(): Boolean { - return getLetterType() == 1 + return getLetterType() == 1 || code.isOtherLowercase() } /** - * Returns `true` if this character is an upper case letter. + * Returns `true` if this character is an upper case letter, or it has contributory property Other_Uppercase. */ internal fun Char.isUpperCaseImpl(): Boolean { - return getLetterType() == 2 + return getLetterType() == 2 || code.isOtherUppercase() } /** diff --git a/libraries/stdlib/js/src/generated/_OtherLowercaseChars.kt b/libraries/stdlib/js/src/generated/_OtherLowercaseChars.kt new file mode 100644 index 00000000000..08cddc0315f --- /dev/null +++ b/libraries/stdlib/js/src/generated/_OtherLowercaseChars.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package kotlin.text + +// +// NOTE: THIS FILE IS AUTO-GENERATED by the GenerateUnicodeData.kt +// See: https://github.com/JetBrains/kotlin/tree/master/libraries/stdlib +// + +private object OtherLowercase { + internal val otherLowerStart = intArrayOf( + 0x00aa, 0x00ba, 0x02b0, 0x02c0, 0x02e0, 0x0345, 0x037a, 0x1d2c, 0x1d78, 0x1d9b, 0x2071, 0x207f, 0x2090, 0x2170, 0x24d0, 0x2c7c, 0xa69c, 0xa770, 0xa7f8, 0xab5c, + ) + internal val otherLowerLength = intArrayOf( + 1, 1, 9, 2, 5, 1, 1, 63, 1, 37, 1, 1, 13, 16, 26, 2, 2, 1, 2, 4, + ) +} + +internal fun Int.isOtherLowercase(): Boolean { + val index = binarySearchRange(OtherLowercase.otherLowerStart, this) + return index >= 0 && this < OtherLowercase.otherLowerStart[index] + OtherLowercase.otherLowerLength[index] +} diff --git a/libraries/stdlib/js/src/generated/_OtherUppercaseChars.kt b/libraries/stdlib/js/src/generated/_OtherUppercaseChars.kt new file mode 100644 index 00000000000..d26518cb207 --- /dev/null +++ b/libraries/stdlib/js/src/generated/_OtherUppercaseChars.kt @@ -0,0 +1,16 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package kotlin.text + +// +// NOTE: THIS FILE IS AUTO-GENERATED by the GenerateUnicodeData.kt +// See: https://github.com/JetBrains/kotlin/tree/master/libraries/stdlib +// + +internal fun Int.isOtherUppercase(): Boolean { + return this in 0x2160..0x216f + || this in 0x24b6..0x24cf +} diff --git a/libraries/stdlib/js/src/kotlin/text/char.kt b/libraries/stdlib/js/src/kotlin/text/char.kt index 59c027ea384..a9a3fd5b7c9 100644 --- a/libraries/stdlib/js/src/kotlin/text/char.kt +++ b/libraries/stdlib/js/src/kotlin/text/char.kt @@ -181,9 +181,10 @@ public actual fun Char.isDigit(): Boolean { } /** - * Returns `true` if this character is an upper case letter. + * Returns `true` if this character is upper case. * - * A character is considered to be an upper case letter if its [category] is [CharCategory.UPPERCASE_LETTER]. + * A character is considered to be an upper case character if its [category] is [CharCategory.UPPERCASE_LETTER], + * or it has contributory property Other_Uppercase as defined by the Unicode Standard. * * @sample samples.text.Chars.isUpperCase */ @@ -199,9 +200,10 @@ public actual fun Char.isUpperCase(): Boolean { } /** - * Returns `true` if this character is a lower case letter. + * Returns `true` if this character is lower case. * - * A character is considered to be a lower case letter if its [category] is [CharCategory.LOWERCASE_LETTER]. + * A character is considered to be a lower case character if its [category] is [CharCategory.LOWERCASE_LETTER], + * or it has contributory property Other_Lowercase as defined by the Unicode Standard. * * @sample samples.text.Chars.isLowerCase */ diff --git a/libraries/stdlib/jvm/src/kotlin/text/CharJVM.kt b/libraries/stdlib/jvm/src/kotlin/text/CharJVM.kt index c57ab7623d8..bb54173f3f7 100644 --- a/libraries/stdlib/jvm/src/kotlin/text/CharJVM.kt +++ b/libraries/stdlib/jvm/src/kotlin/text/CharJVM.kt @@ -100,7 +100,8 @@ public actual fun Char.isWhitespace(): Boolean = Character.isWhitespace(this) || /** * Returns `true` if this character is upper case. * - * @see Character.isUpperCase + * A character is considered to be an upper case character if its [category] is [CharCategory.UPPERCASE_LETTER], + * or it has contributory property Other_Uppercase as defined by the Unicode Standard. * * @sample samples.text.Chars.isUpperCase */ @@ -110,7 +111,8 @@ public actual inline fun Char.isUpperCase(): Boolean = Character.isUpperCase(thi /** * Returns `true` if this character is lower case. * - * @see Character.isLowerCase + * A character is considered to be a lower case character if its [category] is [CharCategory.LOWERCASE_LETTER], + * or it has contributory property Other_Lowercase as defined by the Unicode Standard. * * @sample samples.text.Chars.isLowerCase */ diff --git a/libraries/stdlib/src/kotlin/text/Char.kt b/libraries/stdlib/src/kotlin/text/Char.kt index 86befea97d6..433a66e4080 100644 --- a/libraries/stdlib/src/kotlin/text/Char.kt +++ b/libraries/stdlib/src/kotlin/text/Char.kt @@ -296,9 +296,10 @@ public expect fun Char.isLetterOrDigit(): Boolean public expect fun Char.isDigit(): Boolean /** - * Returns `true` if this character is an upper case letter. + * Returns `true` if this character is upper case. * - * A character is considered to be an upper case letter if its [category] is [CharCategory.UPPERCASE_LETTER]. + * A character is considered to be an upper case character if its [category] is [CharCategory.UPPERCASE_LETTER], + * or it has contributory property Other_Uppercase as defined by the Unicode Standard. * * @sample samples.text.Chars.isUpperCase */ @@ -306,9 +307,10 @@ public expect fun Char.isDigit(): Boolean public expect fun Char.isUpperCase(): Boolean /** - * Returns `true` if this character is a lower case letter. + * Returns `true` if this character is lower case. * - * A character is considered to be a lower case letter if its [category] is [CharCategory.LOWERCASE_LETTER]. + * A character is considered to be a lower case character if its [category] is [CharCategory.LOWERCASE_LETTER], + * or it has contributory property Other_Lowercase as defined by the Unicode Standard. * * @sample samples.text.Chars.isLowerCase */ diff --git a/libraries/stdlib/test/text/CharTest.kt b/libraries/stdlib/test/text/CharTest.kt index 1c485ef6ac5..9b41396a06c 100644 --- a/libraries/stdlib/test/text/CharTest.kt +++ b/libraries/stdlib/test/text/CharTest.kt @@ -5,6 +5,7 @@ package test.text +import test.testOnNonJvm6And7 import kotlin.test.* class CharTest { @@ -521,4 +522,34 @@ class CharTest { assertEquals("\uFFFF", '\uFFFF'.titlecase()) } + + @Test + fun otherLowercaseProperty() { + testOnNonJvm6And7 { + val feminineOrdinalIndicator = '\u00AA' + assertTrue(feminineOrdinalIndicator.isLowerCase()) + assertTrue(feminineOrdinalIndicator.isLetter()) + assertFalse(feminineOrdinalIndicator.isUpperCase()) + + val circledLatinSmallLetterA = '\u24D0' + assertTrue(circledLatinSmallLetterA.isLowerCase()) + assertFalse(circledLatinSmallLetterA.isLetter()) + assertFalse(circledLatinSmallLetterA.isUpperCase()) + } + } + + @Test + fun otherUppercaseProperty() { + testOnNonJvm6And7 { + val romanNumberOne = '\u2160' + assertTrue(romanNumberOne.isUpperCase()) + assertFalse(romanNumberOne.isLetter()) + assertFalse(romanNumberOne.isLowerCase()) + + val circledLatinCapitalLetterZ = '\u24CF' + assertTrue(circledLatinCapitalLetterZ.isUpperCase()) + assertFalse(circledLatinCapitalLetterZ.isLetter()) + assertFalse(circledLatinCapitalLetterZ.isLowerCase()) + } + } } diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/FileWriterUtils.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/FileWriterUtils.kt index f8d1bcca978..6429e9bce20 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/FileWriterUtils.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/FileWriterUtils.kt @@ -19,8 +19,9 @@ internal fun FileWriter.writeHeader(file: File, pkg: String) { appendLine(autoGeneratedWarning("GenerateUnicodeData.kt")) } -internal fun FileWriter.writeIntArray(name: String, elements: List, strategy: RangesWritingStrategy) { - writeCollection(name, "intArrayOf", elements.map { it.toHexIntLiteral() }, strategy) +internal fun FileWriter.writeIntArray(name: String, elements: List, strategy: RangesWritingStrategy, useHex: Boolean = true) { + val toString = if (useHex) Int::toHexIntLiteral else Int::toString + writeCollection(name, "intArrayOf", elements.map(toString), strategy) } private fun FileWriter.writeCollection( @@ -98,3 +99,16 @@ internal fun String.hexToInt(): Int { internal fun List.hexCharsToStringLiteral(): String { return "\"${joinToString(separator = "") { "\\u$it" }}\"" } + +internal fun IntRange.rangeCheck(ch: String, indent: String): String { + val firstHex = first.toHexIntLiteral() + val lastHex = last.toHexIntLiteral() + return when (first) { + last -> + "$ch == $firstHex" + last - 1 -> + "$ch == $firstHex\n$indent|| $ch == $lastHex" + else -> + "$ch in $firstHex..$lastHex" + } +} diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/GenerateUnicodeData.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/GenerateUnicodeData.kt index f90b919765a..2af9f68ce87 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/GenerateUnicodeData.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/GenerateUnicodeData.kt @@ -12,6 +12,8 @@ import generators.unicode.ranges.RangesGenerator import generators.unicode.mappings.string.StringCasingTestGenerator import generators.unicode.mappings.string.StringLowercaseGenerator import generators.unicode.mappings.string.StringUppercaseGenerator +import generators.unicode.ranges.OtherLowercaseRangesGenerator +import generators.unicode.ranges.OtherUppercaseRangesGenerator import templates.COPYRIGHT_NOTICE import templates.KotlinTarget import templates.readCopyrightNoticeFromProfile @@ -61,6 +63,8 @@ fun main(args: Array) { } val categoryRangesGenerators = mutableListOf() + val otherLowercaseGenerators = mutableListOf() + val otherUppercaseGenerators = mutableListOf() fun addRangesGenerators(generatedDir: File, target: KotlinTarget) { val category = RangesGenerator.forCharCategory(generatedDir.resolve("_CharCategories.kt"), target) @@ -71,6 +75,9 @@ fun main(args: Array) { categoryRangesGenerators.add(digit) categoryRangesGenerators.add(letter) categoryRangesGenerators.add(whitespace) + + otherLowercaseGenerators.add(OtherLowercaseRangesGenerator(generatedDir.resolve("_OtherLowercaseChars.kt"), target)) + otherUppercaseGenerators.add(OtherUppercaseRangesGenerator(generatedDir.resolve("_OtherUppercaseChars.kt"), target)) } val oneToOneMappingsGenerators = mutableListOf() @@ -154,9 +161,18 @@ fun main(args: Array) { bmpUnicodeDataLines.forEach { line -> it.appendLine(line) } it.generate() } + otherLowercaseGenerators.forEach { + propListLines.forEach { line -> it.appendLine(line) } + it.generate() + } + otherUppercaseGenerators.forEach { + propListLines.forEach { line -> it.appendLine(line) } + it.generate() + } categoryTestGenerator.let { bmpUnicodeDataLines.forEach { line -> it.appendLine(line) } + propListLines.forEach { line -> it.appendPropertyLine(line) } it.generate() } @@ -176,7 +192,6 @@ fun main(args: Array) { } stringLowercaseGenerator.let { specialCasingLines.forEach { line -> it.appendSpecialCasingLine(line) } - propListLines.forEach { line -> it.appendPropListLine(line) } wordBreakPropertyLines.forEach { line -> it.appendWordBreakPropertyLine(line) } it.generate() } diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/PropertyLine.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/PropertyLine.kt index 83818dd5e7d..536dc431d1b 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/PropertyLine.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/PropertyLine.kt @@ -14,6 +14,14 @@ internal class PropertyLine(properties: List) { val rangeEnd: String = properties[0].split("..").last() val property: String = properties[1].takeWhile { it != ' ' } + fun intRange(): IntRange { + return rangeStart.hexToInt()..rangeEnd.hexToInt() + } + + fun hexIntRangeLiteral(): String { + return "${rangeStart.hexToInt().toHexIntLiteral()}..${rangeEnd.hexToInt().toHexIntLiteral()}" + } + override fun toString(): String { return "PropertyLine{rangeStart=$rangeStart" + ", rangeEnd=$rangeEnd" + diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/string/StringCasingTestGenerator.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/string/StringCasingTestGenerator.kt index 2c1c2fa18ad..a359ed49ed9 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/string/StringCasingTestGenerator.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/string/StringCasingTestGenerator.kt @@ -57,9 +57,7 @@ internal class StringCasingTestGenerator(private val outputDir: File) { writer.appendLine("@SharedImmutable") writer.appendLine("private val $rangesArrayName = arrayOf(") ranges.forEach { - val start = it.rangeStart.hexToInt().toHexIntLiteral() - val end = it.rangeEnd.hexToInt().toHexIntLiteral() - writer.appendLine(" $start..$end,") + writer.appendLine(" ${it.hexIntRangeLiteral()},") } writer.appendLine(")") writer.appendLine() diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/string/StringLowercaseGenerator.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/string/StringLowercaseGenerator.kt index 435871c2783..853c7670d40 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/string/StringLowercaseGenerator.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/string/StringLowercaseGenerator.kt @@ -44,18 +44,11 @@ internal class StringLowercaseGenerator( override fun UnicodeDataLine.mapping(): String = lowercaseMapping - fun appendPropListLine(line: PropertyLine) { - when (line.property) { - "Other_Lowercase", - "Other_Uppercase" -> casedRanges.add(line.rangeStart.hexToInt()..line.rangeEnd.hexToInt()) - } - } - fun appendWordBreakPropertyLine(line: PropertyLine) { when (line.property) { "MidLetter", "MidNumLet", - "Single_Quote" -> caseIgnorableRanges.add(line.rangeStart.hexToInt()..line.rangeEnd.hexToInt()) + "Single_Quote" -> caseIgnorableRanges.add(line.intRange()) } } @@ -101,6 +94,9 @@ internal class StringLowercaseGenerator( CharCategory.TITLECASE_LETTER.value -> return true } } + if (isOtherUppercase() || isOtherLowercase()) { + return true + } val index = binarySearchRange(casedStart, this) return index >= 0 && this <= casedEnd[index] } diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/CharCategoryTestGenerator.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/CharCategoryTestGenerator.kt index 8c6b71c9cd4..61efbd83b62 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/CharCategoryTestGenerator.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/CharCategoryTestGenerator.kt @@ -5,6 +5,7 @@ package generators.unicode.ranges +import generators.unicode.PropertyLine import generators.unicode.UnicodeDataLine import generators.unicode.writeHeader import java.io.File @@ -38,6 +39,16 @@ internal class CharCategoryTestGenerator(private val outputFile: File) { } } + private val otherLowercaseRanges = mutableListOf() + private val otherUppercaseRanges = mutableListOf() + + fun appendPropertyLine(line: PropertyLine) { + when (line.property) { + "Other_Lowercase" -> otherLowercaseRanges.add(line) + "Other_Uppercase" -> otherUppercaseRanges.add(line) + } + } + fun generate() { writer?.appendLine(")") writer?.close() @@ -111,10 +122,10 @@ class CharCategoryTest { val expectedIsLetterOrDigit = expectedIsLetter || expectedIsDigit test(expectedIsLetterOrDigit, char.isLetterOrDigit(), "isLetterOrDigit()") - val expectedIsLowerCase = isLowerCase(expectedCategoryCode) + val expectedIsLowerCase = isLowerCase(char, expectedCategoryCode) test(expectedIsLowerCase, char.isLowerCase(), "isLowerCase()") - val expectedIsUpperCase = isUpperCase(expectedCategoryCode) + val expectedIsUpperCase = isUpperCase(char, expectedCategoryCode) test(expectedIsUpperCase, char.isUpperCase(), "isUpperCase()") val expectedIsWhitespace = isWhitespace(char, expectedCategoryCode) @@ -136,12 +147,20 @@ class CharCategoryTest { ).map { it.code } } - private fun isLowerCase(categoryCode: String): Boolean { - return categoryCode == CharCategory.LOWERCASE_LETTER.code + private val otherLowerChars = listOf( + ${otherLowercaseRanges.joinToString { it.hexIntRangeLiteral() }} + ).flatten().toHashSet() + + private fun isLowerCase(char: Char, categoryCode: String): Boolean { + return categoryCode == CharCategory.LOWERCASE_LETTER.code || otherLowerChars.contains(char.code) } - private fun isUpperCase(categoryCode: String): Boolean { - return categoryCode == CharCategory.UPPERCASE_LETTER.code + private val otherUpperChars = listOf( + ${otherUppercaseRanges.joinToString { it.hexIntRangeLiteral() }} + ).flatten().toHashSet() + + private fun isUpperCase(char: Char, categoryCode: String): Boolean { + return categoryCode == CharCategory.UPPERCASE_LETTER.code || otherUpperChars.contains(char.code) } private fun isWhitespace(char: Char, categoryCode: String): Boolean { diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/OtherLowercaseRangesGenerator.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/OtherLowercaseRangesGenerator.kt new file mode 100644 index 00000000000..e5c0b500da3 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/OtherLowercaseRangesGenerator.kt @@ -0,0 +1,52 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.ranges + +import generators.unicode.PropertyLine +import generators.unicode.hexToInt +import generators.unicode.writeHeader +import generators.unicode.writeIntArray +import templates.KotlinTarget +import java.io.File +import java.io.FileWriter + +internal class OtherLowercaseRangesGenerator( + private val outputFile: File, + private val target: KotlinTarget +) { + private val otherLowerRanges = mutableListOf() + + fun appendLine(line: PropertyLine) { + // In Native the Other_Lowercase code points are also used to perform String.lowercase() + if (target != KotlinTarget.Native && line.rangeStart.hexToInt() > 0xFFFF) return + + if (line.property == "Other_Lowercase") { + otherLowerRanges.add(line.intRange()) + } + } + + fun generate() { + val strategy = RangesWritingStrategy.of(target, "OtherLowercase") + + FileWriter(outputFile).use { writer -> + writer.writeHeader(outputFile, "kotlin.text") + writer.appendLine() + strategy.beforeWritingRanges(writer) + writer.writeIntArray("otherLowerStart", otherLowerRanges.map { it.first }, strategy) + writer.writeIntArray("otherLowerLength", otherLowerRanges.map { it.last - it.first + 1 }, strategy, useHex = false) + strategy.afterWritingRanges(writer) + writer.appendLine() + writer.appendLine(isOtherLowercaseImpl(strategy)) + } + } + + fun isOtherLowercaseImpl(strategy: RangesWritingStrategy) = """ + internal fun Int.isOtherLowercase(): Boolean { + val index = binarySearchRange(${strategy.rangeRef("otherLowerStart")}, this) + return index >= 0 && this < ${strategy.rangeRef("otherLowerStart")}[index] + ${strategy.rangeRef("otherLowerLength")}[index] + } + """.trimIndent() +} \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/OtherUppercaseRangesGenerator.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/OtherUppercaseRangesGenerator.kt new file mode 100644 index 00000000000..8fc794977ad --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/OtherUppercaseRangesGenerator.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.ranges + +import generators.unicode.PropertyLine +import generators.unicode.hexToInt +import generators.unicode.rangeCheck +import generators.unicode.writeHeader +import templates.KotlinTarget +import java.io.File +import java.io.FileWriter + +internal class OtherUppercaseRangesGenerator( + private val outputFile: File, + private val target: KotlinTarget +) { + private val otherUpperRanges = mutableListOf() + + fun appendLine(line: PropertyLine) { + // In Native the Other_Uppercase code points are also used to perform String.lowercase() + if (target != KotlinTarget.Native && line.rangeStart.hexToInt() > 0xFFFF) return + + if (line.property == "Other_Uppercase") { + otherUpperRanges.add(line.intRange()) + } + } + + fun generate() { + FileWriter(outputFile).use { writer -> + writer.writeHeader(outputFile, "kotlin.text") + writer.appendLine() + writer.appendLine(isOtherUppercaseImpl()) + } + } + + fun isOtherUppercaseImpl(): String { + val indent = " ".repeat(5) + val builder = StringBuilder() + + for (i in otherUpperRanges.indices) { + if (i != 0) { + builder.appendLine().append(indent).append("|| ") + } + builder.append(otherUpperRanges[i].rangeCheck("this", indent)) + } + + return """ + internal fun Int.isOtherUppercase(): Boolean { + return $builder + } + """.trimIndent() + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/LetterRangesWriter.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/LetterRangesWriter.kt index 731d5c86483..872ab111f1c 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/LetterRangesWriter.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/LetterRangesWriter.kt @@ -60,17 +60,17 @@ internal open class LetterRangesWriter(protected val strategy: RangesWritingStra } /** - * Returns `true` if this character is a lower case letter. + * Returns `true` if this character is a lower case letter, or it has contributory property Other_Lowercase. */ internal fun Char.isLowerCaseImpl(): Boolean { - return getLetterType() == 1 + return getLetterType() == 1 || code.isOtherLowercase() } /** - * Returns `true` if this character is an upper case letter. + * Returns `true` if this character is an upper case letter, or it has contributory property Other_Uppercase. */ internal fun Char.isUpperCaseImpl(): Boolean { - return getLetterType() == 2 + return getLetterType() == 2 || code.isOtherUppercase() } /** diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/WhitespaceRangesWriter.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/WhitespaceRangesWriter.kt index 520b70bfd53..0715037ebf6 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/WhitespaceRangesWriter.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/WhitespaceRangesWriter.kt @@ -5,6 +5,7 @@ package generators.unicode.ranges.writers +import generators.unicode.rangeCheck import generators.unicode.toHexIntLiteral import java.io.FileWriter @@ -38,24 +39,12 @@ internal class WhitespaceRangesWriter : RangesWriter { val start = rangeStart[i] val end = rangeEnd[i] - when (start) { - end -> { - if (start > 0x1000 && tabCount == 5) { - builder.appendLine("$ch > 0x1000 && (") - tabCount = 6 - builder.append(tab.repeat(tabCount)) - } - builder.appendLine("$ch == ${start.toHexIntLiteral()}") - } - end - 1 -> { - builder.appendLine("$ch == ${start.toHexIntLiteral()}") - builder.append(tab.repeat(tabCount)).append("|| ") - builder.appendLine("$ch == ${end.toHexIntLiteral()}") - } - else -> { - builder.appendLine("$ch in ${start.toHexIntLiteral()}..${end.toHexIntLiteral()}") - } + if (start > 0x1000 && tabCount == 5) { + builder.appendLine("$ch > 0x1000 && (") + tabCount = 6 + builder.append(tab.repeat(tabCount)) } + builder.appendLine((start..end).rangeCheck(ch, tab.repeat(tabCount))) } return builder.append(tab.repeat(5)).append(")").toString()