Equivalize isLowerCase and isUpperCase behavior in all platforms #KT-46184

2021-05-25 05:32:29 +03:00
parent 0b10f255d7
commit d934c97bf5
25 changed files with 386 additions and 75 deletions
@@ -19,8 +19,9 @@ internal fun FileWriter.writeHeader(file: File, pkg: String) {
    appendLine(autoGeneratedWarning("GenerateUnicodeData.kt"))
 }

-internal fun FileWriter.writeIntArray(name: String, elements: List<Int>, strategy: RangesWritingStrategy) {
-    writeCollection(name, "intArrayOf", elements.map { it.toHexIntLiteral() }, strategy)
+internal fun FileWriter.writeIntArray(name: String, elements: List<Int>, strategy: RangesWritingStrategy, useHex: Boolean = true) {
+    val toString = if (useHex) Int::toHexIntLiteral else Int::toString
+    writeCollection(name, "intArrayOf", elements.map(toString), strategy)
 }

 private fun FileWriter.writeCollection(
@@ -98,3 +99,16 @@ internal fun String.hexToInt(): Int {
 internal fun List<String>.hexCharsToStringLiteral(): String {
    return "\"${joinToString(separator = "") { "\\u$it" }}\""
 }
+
+internal fun IntRange.rangeCheck(ch: String, indent: String): String {
+    val firstHex = first.toHexIntLiteral()
+    val lastHex = last.toHexIntLiteral()
+    return when (first) {
+        last ->
+            "$ch == $firstHex"
+        last - 1 ->
+            "$ch == $firstHex\n$indent|| $ch == $lastHex"
+        else ->
+            "$ch in $firstHex..$lastHex"
+    }
+}
@@ -12,6 +12,8 @@ import generators.unicode.ranges.RangesGenerator
 import generators.unicode.mappings.string.StringCasingTestGenerator
 import generators.unicode.mappings.string.StringLowercaseGenerator
 import generators.unicode.mappings.string.StringUppercaseGenerator
+import generators.unicode.ranges.OtherLowercaseRangesGenerator
+import generators.unicode.ranges.OtherUppercaseRangesGenerator
 import templates.COPYRIGHT_NOTICE
 import templates.KotlinTarget
 import templates.readCopyrightNoticeFromProfile
@@ -61,6 +63,8 @@ fun main(args: Array<String>) {
    }

    val categoryRangesGenerators = mutableListOf<RangesGenerator>()
+    val otherLowercaseGenerators = mutableListOf<OtherLowercaseRangesGenerator>()
+    val otherUppercaseGenerators = mutableListOf<OtherUppercaseRangesGenerator>()

    fun addRangesGenerators(generatedDir: File, target: KotlinTarget) {
        val category = RangesGenerator.forCharCategory(generatedDir.resolve("_CharCategories.kt"), target)
@@ -71,6 +75,9 @@ fun main(args: Array<String>) {
        categoryRangesGenerators.add(digit)
        categoryRangesGenerators.add(letter)
        categoryRangesGenerators.add(whitespace)
+
+        otherLowercaseGenerators.add(OtherLowercaseRangesGenerator(generatedDir.resolve("_OtherLowercaseChars.kt"), target))
+        otherUppercaseGenerators.add(OtherUppercaseRangesGenerator(generatedDir.resolve("_OtherUppercaseChars.kt"), target))
    }

    val oneToOneMappingsGenerators = mutableListOf<MappingsGenerator>()
@@ -154,9 +161,18 @@ fun main(args: Array<String>) {
        bmpUnicodeDataLines.forEach { line -> it.appendLine(line) }
        it.generate()
    }
+    otherLowercaseGenerators.forEach {
+        propListLines.forEach { line -> it.appendLine(line) }
+        it.generate()
+    }
+    otherUppercaseGenerators.forEach {
+        propListLines.forEach { line -> it.appendLine(line) }
+        it.generate()
+    }

    categoryTestGenerator.let {
        bmpUnicodeDataLines.forEach { line -> it.appendLine(line) }
+        propListLines.forEach { line -> it.appendPropertyLine(line) }
        it.generate()
    }

@@ -176,7 +192,6 @@ fun main(args: Array<String>) {
    }
    stringLowercaseGenerator.let {
        specialCasingLines.forEach { line -> it.appendSpecialCasingLine(line) }
-        propListLines.forEach { line -> it.appendPropListLine(line) }
        wordBreakPropertyLines.forEach { line -> it.appendWordBreakPropertyLine(line) }
        it.generate()
    }
@@ -14,6 +14,14 @@ internal class PropertyLine(properties: List<String>) {
    val rangeEnd: String = properties[0].split("..").last()
    val property: String = properties[1].takeWhile { it != ' ' }

+    fun intRange(): IntRange {
+        return rangeStart.hexToInt()..rangeEnd.hexToInt()
+    }
+
+    fun hexIntRangeLiteral(): String {
+        return "${rangeStart.hexToInt().toHexIntLiteral()}..${rangeEnd.hexToInt().toHexIntLiteral()}"
+    }
+
    override fun toString(): String {
        return "PropertyLine{rangeStart=$rangeStart" +
                ", rangeEnd=$rangeEnd" +
@@ -57,9 +57,7 @@ internal class StringCasingTestGenerator(private val outputDir: File) {
            writer.appendLine("@SharedImmutable")
            writer.appendLine("private val $rangesArrayName = arrayOf<IntRange>(")
            ranges.forEach {
-                val start = it.rangeStart.hexToInt().toHexIntLiteral()
-                val end = it.rangeEnd.hexToInt().toHexIntLiteral()
-                writer.appendLine("    $start..$end,")
+                writer.appendLine("    ${it.hexIntRangeLiteral()},")
            }
            writer.appendLine(")")
            writer.appendLine()
@@ -44,18 +44,11 @@ internal class StringLowercaseGenerator(

    override fun UnicodeDataLine.mapping(): String = lowercaseMapping

-    fun appendPropListLine(line: PropertyLine) {
-        when (line.property) {
-            "Other_Lowercase",
-            "Other_Uppercase" -> casedRanges.add(line.rangeStart.hexToInt()..line.rangeEnd.hexToInt())
-        }
-    }
-
    fun appendWordBreakPropertyLine(line: PropertyLine) {
        when (line.property) {
            "MidLetter",
            "MidNumLet",
-            "Single_Quote" -> caseIgnorableRanges.add(line.rangeStart.hexToInt()..line.rangeEnd.hexToInt())
+            "Single_Quote" -> caseIgnorableRanges.add(line.intRange())
        }
    }

@@ -101,6 +94,9 @@ internal class StringLowercaseGenerator(
                    CharCategory.TITLECASE_LETTER.value -> return true
                }
            }
+            if (isOtherUppercase() || isOtherLowercase()) {
+                return true
+            }
            val index = binarySearchRange(casedStart, this)
            return index >= 0 && this <= casedEnd[index]
        }
@@ -5,6 +5,7 @@

 package generators.unicode.ranges

+import generators.unicode.PropertyLine
 import generators.unicode.UnicodeDataLine
 import generators.unicode.writeHeader
 import java.io.File
@@ -38,6 +39,16 @@ internal class CharCategoryTestGenerator(private val outputFile: File) {
        }
    }

+    private val otherLowercaseRanges = mutableListOf<PropertyLine>()
+    private val otherUppercaseRanges = mutableListOf<PropertyLine>()
+
+    fun appendPropertyLine(line: PropertyLine) {
+        when (line.property) {
+            "Other_Lowercase" -> otherLowercaseRanges.add(line)
+            "Other_Uppercase" -> otherUppercaseRanges.add(line)
+        }
+    }
+
    fun generate() {
        writer?.appendLine(")")
        writer?.close()
@@ -111,10 +122,10 @@ class CharCategoryTest {
            val expectedIsLetterOrDigit = expectedIsLetter || expectedIsDigit
            test(expectedIsLetterOrDigit, char.isLetterOrDigit(), "isLetterOrDigit()")
            
-            val expectedIsLowerCase = isLowerCase(expectedCategoryCode)
+            val expectedIsLowerCase = isLowerCase(char, expectedCategoryCode)
            test(expectedIsLowerCase, char.isLowerCase(), "isLowerCase()")

-            val expectedIsUpperCase = isUpperCase(expectedCategoryCode)
+            val expectedIsUpperCase = isUpperCase(char, expectedCategoryCode)
            test(expectedIsUpperCase, char.isUpperCase(), "isUpperCase()")

            val expectedIsWhitespace = isWhitespace(char, expectedCategoryCode)
@@ -136,12 +147,20 @@ class CharCategoryTest {
        ).map { it.code }
    }

-    private fun isLowerCase(categoryCode: String): Boolean {
-        return categoryCode == CharCategory.LOWERCASE_LETTER.code
+    private val otherLowerChars = listOf<IntRange>(
+        ${otherLowercaseRanges.joinToString { it.hexIntRangeLiteral() }}
+    ).flatten().toHashSet()
+
+    private fun isLowerCase(char: Char, categoryCode: String): Boolean {
+        return categoryCode == CharCategory.LOWERCASE_LETTER.code || otherLowerChars.contains(char.code)
    }

-    private fun isUpperCase(categoryCode: String): Boolean {
-        return categoryCode == CharCategory.UPPERCASE_LETTER.code
+    private val otherUpperChars = listOf<IntRange>(
+        ${otherUppercaseRanges.joinToString { it.hexIntRangeLiteral() }}
+    ).flatten().toHashSet()
+
+    private fun isUpperCase(char: Char, categoryCode: String): Boolean {
+        return categoryCode == CharCategory.UPPERCASE_LETTER.code || otherUpperChars.contains(char.code)
    }

    private fun isWhitespace(char: Char, categoryCode: String): Boolean {
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.ranges
+
+import generators.unicode.PropertyLine
+import generators.unicode.hexToInt
+import generators.unicode.writeHeader
+import generators.unicode.writeIntArray
+import templates.KotlinTarget
+import java.io.File
+import java.io.FileWriter
+
+internal class OtherLowercaseRangesGenerator(
+    private val outputFile: File,
+    private val target: KotlinTarget
+) {
+    private val otherLowerRanges = mutableListOf<IntRange>()
+
+    fun appendLine(line: PropertyLine) {
+        // In Native the Other_Lowercase code points are also used to perform String.lowercase()
+        if (target != KotlinTarget.Native && line.rangeStart.hexToInt() > 0xFFFF) return
+
+        if (line.property == "Other_Lowercase") {
+            otherLowerRanges.add(line.intRange())
+        }
+    }
+
+    fun generate() {
+        val strategy = RangesWritingStrategy.of(target, "OtherLowercase")
+
+        FileWriter(outputFile).use { writer ->
+            writer.writeHeader(outputFile, "kotlin.text")
+            writer.appendLine()
+            strategy.beforeWritingRanges(writer)
+            writer.writeIntArray("otherLowerStart", otherLowerRanges.map { it.first }, strategy)
+            writer.writeIntArray("otherLowerLength", otherLowerRanges.map { it.last - it.first + 1 }, strategy, useHex = false)
+            strategy.afterWritingRanges(writer)
+            writer.appendLine()
+            writer.appendLine(isOtherLowercaseImpl(strategy))
+        }
+    }
+
+    fun isOtherLowercaseImpl(strategy: RangesWritingStrategy) = """
+        internal fun Int.isOtherLowercase(): Boolean {
+            val index = binarySearchRange(${strategy.rangeRef("otherLowerStart")}, this)
+            return index >= 0 && this < ${strategy.rangeRef("otherLowerStart")}[index] + ${strategy.rangeRef("otherLowerLength")}[index]
+        }
+    """.trimIndent()
+}
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.ranges
+
+import generators.unicode.PropertyLine
+import generators.unicode.hexToInt
+import generators.unicode.rangeCheck
+import generators.unicode.writeHeader
+import templates.KotlinTarget
+import java.io.File
+import java.io.FileWriter
+
+internal class OtherUppercaseRangesGenerator(
+    private val outputFile: File,
+    private val target: KotlinTarget
+) {
+    private val otherUpperRanges = mutableListOf<IntRange>()
+
+    fun appendLine(line: PropertyLine) {
+        // In Native the Other_Uppercase code points are also used to perform String.lowercase()
+        if (target != KotlinTarget.Native && line.rangeStart.hexToInt() > 0xFFFF) return
+
+        if (line.property == "Other_Uppercase") {
+            otherUpperRanges.add(line.intRange())
+        }
+    }
+
+    fun generate() {
+        FileWriter(outputFile).use { writer ->
+            writer.writeHeader(outputFile, "kotlin.text")
+            writer.appendLine()
+            writer.appendLine(isOtherUppercaseImpl())
+        }
+    }
+
+    fun isOtherUppercaseImpl(): String {
+        val indent = "    ".repeat(5)
+        val builder = StringBuilder()
+
+        for (i in otherUpperRanges.indices) {
+            if (i != 0) {
+                builder.appendLine().append(indent).append("|| ")
+            }
+            builder.append(otherUpperRanges[i].rangeCheck("this", indent))
+        }
+
+        return """
+        internal fun Int.isOtherUppercase(): Boolean {
+            return $builder
+        }
+        """.trimIndent()
+    }
+}
@@ -60,17 +60,17 @@ internal open class LetterRangesWriter(protected val strategy: RangesWritingStra
        }

        /**
-         * Returns `true` if this character is a lower case letter.
+         * Returns `true` if this character is a lower case letter, or it has contributory property Other_Lowercase.
         */
        internal fun Char.isLowerCaseImpl(): Boolean {
-            return getLetterType() == 1
+            return getLetterType() == 1 || code.isOtherLowercase()
        }

        /**
-         * Returns `true` if this character is an upper case letter.
+         * Returns `true` if this character is an upper case letter, or it has contributory property Other_Uppercase.
         */
        internal fun Char.isUpperCaseImpl(): Boolean {
-            return getLetterType() == 2
+            return getLetterType() == 2 || code.isOtherUppercase()
        }

        /**
@@ -5,6 +5,7 @@

 package generators.unicode.ranges.writers

+import generators.unicode.rangeCheck
 import generators.unicode.toHexIntLiteral
 import java.io.FileWriter

@@ -38,24 +39,12 @@ internal class WhitespaceRangesWriter : RangesWriter {

            val start = rangeStart[i]
            val end = rangeEnd[i]
-            when (start) {
-                end -> {
-                    if (start > 0x1000 && tabCount == 5) {
-                        builder.appendLine("$ch > 0x1000 && (")
-                        tabCount = 6
-                        builder.append(tab.repeat(tabCount))
-                    }
-                    builder.appendLine("$ch == ${start.toHexIntLiteral()}")
-                }
-                end - 1 -> {
-                    builder.appendLine("$ch == ${start.toHexIntLiteral()}")
-                    builder.append(tab.repeat(tabCount)).append("|| ")
-                    builder.appendLine("$ch == ${end.toHexIntLiteral()}")
-                }
-                else -> {
-                    builder.appendLine("$ch in ${start.toHexIntLiteral()}..${end.toHexIntLiteral()}")
-                }
+            if (start > 0x1000 && tabCount == 5) {
+                builder.appendLine("$ch > 0x1000 && (")
+                tabCount = 6
+                builder.append(tab.repeat(tabCount))
            }
+            builder.appendLine((start..end).rangeCheck(ch, tab.repeat(tabCount)))
        }

        return builder.append(tab.repeat(5)).append(")").toString()