Implement Char one-to-one and one-to-many case conversion generators #KT-45213

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/FileWriterUtils.kt

@@ -1,9 +1,9 @@
 /*
- * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
-package generators.unicode.ranges.writers
+package generators.unicode
 
 import generators.unicode.ranges.RangesWritingStrategy
 import templates.COPYRIGHT_NOTICE
@@ -19,9 +19,14 @@ internal fun FileWriter.writeHeader(file: File, pkg: String) {
     appendLine(autoGeneratedWarning("GenerateUnicodeData.kt"))
 }
 
-internal fun FileWriter.writeIntArray(
+internal fun FileWriter.writeIntArray(name: String, elements: List<Int>, strategy: RangesWritingStrategy) {
+    writeCollection(name, "intArrayOf", elements.map { it.toHexIntLiteral() }, strategy)
+}
+
+private fun FileWriter.writeCollection(
     name: String,
-    elements: List<Int>,
+    constructingFun: String,
+    elements: List<String>,
     strategy: RangesWritingStrategy
 ) {
     fun appendWithIndentation(string: String) {
@@ -29,19 +34,26 @@ internal fun FileWriter.writeIntArray(
     }
 
     append(strategy.rangesAnnotation)
-    appendWithIndentation("${strategy.rangesVisibilityModifier} val $name = intArrayOf(")
+    appendWithIndentation("${strategy.rangesVisibilityModifier} val $name = $constructingFun(")
     for (i in elements.indices) {
         if (i % 20 == 0) {
             appendLine()
             appendWithIndentation("    ")
         }
-        append(elements[i].hex() + ", ")
+        append(elements[i] + ", ")
     }
     appendLine()
     appendWithIndentation(")")
     appendLine()
 }
 
+internal fun FileWriter.writeMappings(mappings: Map<Int, List<String>>, strategy: RangesWritingStrategy) {
+    val keys = mappings.keys.map { it.toHexIntLiteral() }
+    writeCollection("keys", "intArrayOf", keys, strategy)
+    val values = mappings.values.map { it.hexCharsToStringLiteral() }
+    writeCollection("values", "arrayOf", values, strategy)
+}
+
 internal const val TO_BASE64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
 
 internal fun List<Int>.toVarLenBase64(): String {
@@ -67,10 +79,22 @@ private fun Int.to6Bits(): List<Int> {
     return result
 }
 
-internal fun Int.hex(): String {
+internal fun Int.toHexIntLiteral(): String {
     val result = toString(radix = 16)
     if (result.first() == '-') {
         return "-0x" + result.substring(startIndex = 1).padStart(4, '0')
     }
     return "0x" + result.padStart(4, '0')
 }
+
+internal fun Int.toHexCharLiteral(): String {
+    return "'\\u${toString(radix = 16).padStart(4, '0')}'"
+}
+
+internal fun String.hexToInt(): Int {
+    return toInt(radix = 16)
+}
+
+internal fun List<String>.hexCharsToStringLiteral(): String {
+    return "\"${joinToString(separator = "") { "\\u$it" }}\""
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/GenerateUnicodeData.kt

@@ -5,6 +5,8 @@
 
 package generators.unicode
 
+import generators.unicode.mappings.oneToOne.MappingsGenerator
+import generators.unicode.mappings.oneToMany.OneToManyMappingsGenerator
 import generators.unicode.ranges.CharCategoryTestGenerator
 import generators.unicode.ranges.RangesGenerator
 import templates.COPYRIGHT_NOTICE
@@ -16,10 +18,12 @@ import kotlin.system.exitProcess
 
 
 // Go to https://www.unicode.org/versions/latest/ to find out the latest public version of the Unicode Character Database files.
-private const val unicodeDataUrl = "https://www.unicode.org/Public/13.0.0/ucd/UnicodeData.txt"
+private const val unicodeVersion = "13.0.0"
+private const val unicodeDataUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/UnicodeData.txt"
+private const val specialCasingUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/SpecialCasing.txt"
 
 /**
- * This program generates sources related to UnicodeData.txt.
+ * This program generates sources related to UnicodeData.txt and SpecialCasing.txt.
  * There are two ways to run the program.
  * 1. Pass the root directory of the project to generate sources for js and js-ir.
  *  _CharCategoryTest.kt and supporting files are also generated to test the generated sources.
@@ -30,27 +34,57 @@ private const val unicodeDataUrl = "https://www.unicode.org/Public/13.0.0/ucd/Un
 fun main(args: Array<String>) {
 
     val unicodeDataLines = URL(unicodeDataUrl).openStream().reader().readLines()
+    val bmpUnicodeDataLines = unicodeDataLines.mapNotNull { line ->
+        UnicodeDataLine(line.split(";")).takeIf { it.char.length <= 4 } // Basic Multilingual Plane (BMP)
+    }
+    val specialCasingLines = URL(specialCasingUrl).openStream().reader().readLines()
+    val nonCommentSpecialCasingLines = specialCasingLines.mapNotNull { line ->
+        if (line.isEmpty() || line.startsWith("#")) null else SpecialCasingLine(line.split("; "))
+    }
 
-    val generators = mutableListOf<UnicodeDataGenerator>()
+    val categoryRangesGenerators = mutableListOf<RangesGenerator>()
 
     fun addRangesGenerators(generatedDir: File, target: KotlinTarget) {
-        val categoryRangesGenerator = RangesGenerator.forCharCategory(generatedDir.resolve("_CharCategories.kt"), target)
+        val category = RangesGenerator.forCharCategory(generatedDir.resolve("_CharCategories.kt"), target)
-        val digitRangesGenerator = RangesGenerator.forDigit(generatedDir.resolve("_DigitChars.kt"), target)
+        val digit = RangesGenerator.forDigit(generatedDir.resolve("_DigitChars.kt"), target)
-        val letterRangesGenerator = RangesGenerator.forLetter(generatedDir.resolve("_LetterChars.kt"), target)
+        val letter = RangesGenerator.forLetter(generatedDir.resolve("_LetterChars.kt"), target)
-        val whitespaceRangesGenerator = RangesGenerator.forWhitespace(generatedDir.resolve("_WhitespaceChars.kt"))
+        val whitespace = RangesGenerator.forWhitespace(generatedDir.resolve("_WhitespaceChars.kt"))
-        generators.add(categoryRangesGenerator)
+        categoryRangesGenerators.add(category)
-        generators.add(digitRangesGenerator)
+        categoryRangesGenerators.add(digit)
-        generators.add(letterRangesGenerator)
+        categoryRangesGenerators.add(letter)
-        generators.add(whitespaceRangesGenerator)
+        categoryRangesGenerators.add(whitespace)
     }
 
+    val oneToOneMappingsGenerators = mutableListOf<MappingsGenerator>()
+
+    fun addOneToOneMappingsGenerators(generatedDir: File, target: KotlinTarget) {
+        val uppercase = MappingsGenerator.forUppercase(generatedDir.resolve("_UppercaseMappings.kt"), target)
+        val lowercase = MappingsGenerator.forLowercase(generatedDir.resolve("_LowercaseMappings.kt"), target)
+        val titlecase = MappingsGenerator.forTitlecase(generatedDir.resolve("_TitlecaseMappings.kt"))
+        oneToOneMappingsGenerators.add(uppercase)
+        oneToOneMappingsGenerators.add(lowercase)
+        oneToOneMappingsGenerators.add(titlecase)
+    }
+
+    val oneToManyMappingsGenerators = mutableListOf<OneToManyMappingsGenerator>()
+
+    fun addOneToManyMappingsGenerators(generatedDir: File, target: KotlinTarget) {
+        val uppercase = OneToManyMappingsGenerator.forUppercase(generatedDir.resolve("_OneToManyUppercaseMappings.kt"), target, bmpUnicodeDataLines)
+        val lowercase = OneToManyMappingsGenerator.forLowercase(generatedDir.resolve("_OneToManyLowercaseMappings.kt"), target, bmpUnicodeDataLines)
+        val titlecase = OneToManyMappingsGenerator.forTitlecase(generatedDir.resolve("_OneToManyTitlecaseMappings.kt"), target, bmpUnicodeDataLines)
+        oneToManyMappingsGenerators.add(uppercase)
+        oneToManyMappingsGenerators.add(lowercase)
+        oneToManyMappingsGenerators.add(titlecase)
+    }
+
+    var categoryTestGenerator: CharCategoryTestGenerator? = null
+
     when (args.size) {
         1 -> {
             val baseDir = File(args.first())
 
             val categoryTestFile = baseDir.resolve("libraries/stdlib/js/test/text/unicodeData/_CharCategoryTest.kt")
-            val categoryTestGenerator = CharCategoryTestGenerator(categoryTestFile)
+            categoryTestGenerator = CharCategoryTestGenerator(categoryTestFile)
-            generators.add(categoryTestGenerator)
 
             val jsGeneratedDir = baseDir.resolve("libraries/stdlib/js/src/generated/")
             addRangesGenerators(jsGeneratedDir, KotlinTarget.JS)
@@ -59,8 +93,12 @@ fun main(args: Array<String>) {
             addRangesGenerators(jsIrGeneratedDir, KotlinTarget.JS_IR)
 
             // For debugging. To see the file content
-            val unicodeDataFile = baseDir.resolve("libraries/tools/kotlin-stdlib-gen/src/generators/unicode/UnicodeData.txt")
+            fun writeContent(lines: List<String>, fileName: String) {
-            unicodeDataFile.writeText(unicodeDataLines.joinToString(separator = "\n"))
+                val file = baseDir.resolve("libraries/tools/kotlin-stdlib-gen/src/generators/unicode/$fileName")
+                file.writeText(lines.joinToString(separator = "\n"))
+            }
+            writeContent(unicodeDataLines, "UnicodeData.txt")
+            writeContent(specialCasingLines, "SpecialCasing.txt")
         }
         2 -> {
             val (targetName, targetDir) = args
@@ -68,7 +106,13 @@ fun main(args: Array<String>) {
             val target = KotlinTarget.values.singleOrNull { it.name.equals(targetName, ignoreCase = true) }
                 ?: error("Invalid target: $targetName")
 
-            addRangesGenerators(File(targetDir), target)
+            val generatedDir = File(targetDir)
+            addRangesGenerators(generatedDir, target)
+
+            if (target == KotlinTarget.Native) {
+                addOneToOneMappingsGenerators(generatedDir, target)
+                addOneToManyMappingsGenerators(generatedDir, target)
+            }
         }
         else -> {
             println(
@@ -84,16 +128,23 @@ fun main(args: Array<String>) {
     COPYRIGHT_NOTICE =
         readCopyrightNoticeFromProfile { Thread.currentThread().contextClassLoader.getResourceAsStream("apache.xml").reader() }
 
-    unicodeDataLines.forEach { line ->
+    categoryRangesGenerators.forEach {
-        val parts = line.split(";")
+        bmpUnicodeDataLines.forEach { line -> it.appendLine(line) }
-        if (parts[0].length <= 4) {
+        it.generate()
-            generators.forEach { it.appendChar(parts[0], parts[1], parts[2]) }
-        }
     }
-    generators.forEach { it.close() }
-}
 
-internal interface UnicodeDataGenerator {
+    categoryTestGenerator?.let {
-    fun appendChar(char: String, name: String, categoryCode: String)
+        bmpUnicodeDataLines.forEach { line -> it.appendLine(line) }
-    fun close()
+        it.generate()
+    }
+
+    oneToOneMappingsGenerators.forEach {
+        unicodeDataLines.forEach { line -> it.appendLine(line) }
+        it.generate()
+    }
+
+    oneToManyMappingsGenerators.forEach {
+        nonCommentSpecialCasingLines.forEach { line -> it.appendLine(line) }
+        it.generate()
+    }
 }

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/SpecialCasingLine.kt

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode
+
+internal class SpecialCasingLine(properties: List<String>) {
+    init {
+        require(properties.size in 5..6)
+    }
+
+    val char: String = properties[0]
+    val lowercaseMapping: List<String> = properties[1].split(" ")
+    val titlecaseMapping: List<String> = properties[2].split(" ")
+    val uppercaseMapping: List<String> = properties[3].split(" ")
+    val conditionList: List<String> = if (properties.size == 6) properties[4].split(" ") else emptyList()
+
+    override fun toString(): String {
+        return "SpecialCasingLine{char=$char" +
+                ", lowercaseMapping=$lowercaseMapping" +
+                ", uppercaseMapping=$uppercaseMapping" +
+                ", titlecaseMapping=$titlecaseMapping" +
+                ", conditionList=$conditionList" +
+                "}"
+    }
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/UnicodeDataLine.kt

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode
+
+internal class UnicodeDataLine(properties: List<String>) {
+    init {
+        require(properties.size == 15)
+    }
+
+    val char: String = properties[0]
+    val name: String = properties[1]
+    val categoryCode: String = properties[2]
+    val uppercaseMapping: String = properties[12]
+    val lowercaseMapping: String = properties[13]
+    val titlecaseMapping: String = properties[14]
+
+    override fun toString(): String {
+        return "UnicodeDataLine{char=$char" +
+                ", categoryCode=$categoryCode" +
+                ", uppercaseMapping=$uppercaseMapping" +
+                ", lowercaseMapping=$lowercaseMapping" +
+                ", titlecaseMapping=$titlecaseMapping" +
+                ", name=$name" +
+                "}"
+    }
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/OneToManyMappingsGenerator.kt

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToMany
+
+import generators.unicode.SpecialCasingLine
+import generators.unicode.UnicodeDataLine
+import generators.unicode.mappings.oneToMany.builders.*
+import generators.unicode.mappings.oneToMany.writers.*
+import generators.unicode.ranges.RangesWritingStrategy
+import generators.unicode.writeHeader
+import templates.KotlinTarget
+import java.io.File
+import java.io.FileWriter
+
+internal class OneToManyMappingsGenerator private constructor(
+    private val outputFile: File,
+    private val mappingsBuilder: OneToManyMappingsBuilder,
+    private val mappingsWriter: OneToManyMappingsWriter
+) {
+    fun appendLine(line: SpecialCasingLine) {
+        mappingsBuilder.append(line)
+    }
+
+    fun generate() {
+        val mappings = mappingsBuilder.build()
+
+        FileWriter(outputFile).use { writer ->
+            writer.writeHeader(outputFile, "kotlin.text")
+            writer.appendLine()
+            writer.appendLine("// ${mappings.size} mappings totally")
+
+            mappingsWriter.write(mappings, writer)
+        }
+    }
+
+
+    companion object {
+        fun forUppercase(outputFile: File, target: KotlinTarget, bmpUnicodeDataLines: List<UnicodeDataLine>): OneToManyMappingsGenerator {
+            val builder = OneToManyUppercaseMappingsBuilder(bmpUnicodeDataLines)
+            val writer = OneToManyUppercaseMappingsWriter(RangesWritingStrategy.of(target, "OneToManyUppercase"))
+            return OneToManyMappingsGenerator(outputFile, builder, writer)
+        }
+
+        fun forLowercase(outputFile: File, target: KotlinTarget, bmpUnicodeDataLines: List<UnicodeDataLine>): OneToManyMappingsGenerator {
+            val builder = OneToManyLowercaseMappingsBuilder(bmpUnicodeDataLines)
+            val writer = OneToManyLowercaseMappingsWriter(RangesWritingStrategy.of(target, "OneToManyLowercase"))
+            return OneToManyMappingsGenerator(outputFile, builder, writer)
+        }
+
+        fun forTitlecase(outputFile: File, bmpUnicodeDataLines: List<UnicodeDataLine>): OneToManyMappingsGenerator {
+            val builder = OneToManyTitlecaseMappingsBuilder(bmpUnicodeDataLines)
+            val writer = OneToManyTitlecaseMappingsWriter()
+            return OneToManyMappingsGenerator(outputFile, builder, writer)
+        }
+    }
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyLowercaseMappingsBuilder.kt

@@ -0,0 +1,14 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToMany.builders
+
+import generators.unicode.SpecialCasingLine
+import generators.unicode.UnicodeDataLine
+
+internal class OneToManyLowercaseMappingsBuilder(bmpUnicodeDataLines: List<UnicodeDataLine>) : OneToManyMappingsBuilder(bmpUnicodeDataLines) {
+    override fun SpecialCasingLine.mapping(): List<String> = lowercaseMapping
+    override fun UnicodeDataLine.mapping(): String = lowercaseMapping
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyMappingsBuilder.kt

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToMany.builders
+
+import generators.unicode.SpecialCasingLine
+import generators.unicode.UnicodeDataLine
+import generators.unicode.hexToInt
+import java.util.TreeMap
+
+internal abstract class OneToManyMappingsBuilder(bmpUnicodeDataLines: List<UnicodeDataLine>) {
+    private val bmpUnicodeDataLines = bmpUnicodeDataLines.associateBy { it.char.hexToInt() }
+    private val mappings = TreeMap<Int, List<String>>()
+
+    fun append(line: SpecialCasingLine) {
+        if (line.conditionList.isNotEmpty()) return
+
+        val charCode = line.char.hexToInt()
+
+        check(charCode <= Char.MAX_VALUE.toInt()) { "Handle special casing for the supplementary code point: $line" }
+
+        val mapping = mapping(charCode, line) ?: return
+
+        mappings[charCode] = mapping
+    }
+
+    fun build(): Map<Int, List<String>> {
+//        println(mappings)
+//        println("${this.javaClass} # ${mappings.size}")
+        return mappings
+    }
+
+    private fun mapping(charCode: Int, line: SpecialCasingLine): List<String>? {
+        val mapping = line.mapping()
+
+        check(mapping.isNotEmpty() && mapping.all { it.isNotEmpty() })
+
+        if (mapping.size == 1) {
+            val specialCasingMapping = mapping.first()
+
+            val unicodeLine = bmpUnicodeDataLines[charCode]
+            val unicodeDataMapping = unicodeLine?.mapping()?.takeIf { it.isNotEmpty() } ?: line.char
+
+            check(unicodeDataMapping == specialCasingMapping) {
+                "UnicodeData.txt and SpecialCasing.txt files have different single char case conversion"
+            }
+
+            return null
+        }
+
+        return mapping
+    }
+
+    abstract fun SpecialCasingLine.mapping(): List<String>
+    abstract fun UnicodeDataLine.mapping(): String
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyTitlecaseMappingsBuilder.kt

@@ -0,0 +1,14 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToMany.builders
+
+import generators.unicode.SpecialCasingLine
+import generators.unicode.UnicodeDataLine
+
+internal class OneToManyTitlecaseMappingsBuilder(bmpUnicodeDataLines: List<UnicodeDataLine>) : OneToManyMappingsBuilder(bmpUnicodeDataLines) {
+    override fun SpecialCasingLine.mapping(): List<String> = titlecaseMapping
+    override fun UnicodeDataLine.mapping(): String = titlecaseMapping
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyUppercaseMappingsBuilder.kt

@@ -0,0 +1,14 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToMany.builders
+
+import generators.unicode.SpecialCasingLine
+import generators.unicode.UnicodeDataLine
+
+internal class OneToManyUppercaseMappingsBuilder(bmpUnicodeDataLines: List<UnicodeDataLine>) : OneToManyMappingsBuilder(bmpUnicodeDataLines) {
+    override fun SpecialCasingLine.mapping(): List<String> = uppercaseMapping
+    override fun UnicodeDataLine.mapping(): String = uppercaseMapping
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyLowercaseMappingsWriter.kt

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToMany.writers
+
+import generators.unicode.ranges.RangesWritingStrategy
+import generators.unicode.toHexCharLiteral
+import generators.unicode.hexCharsToStringLiteral
+import java.io.FileWriter
+
+internal class OneToManyLowercaseMappingsWriter(private val strategy: RangesWritingStrategy) : OneToManyMappingsWriter {
+    override fun write(mappings: Map<Int, List<String>>, writer: FileWriter) {
+        check(mappings.size == 1) { "Number of multi-char lowercase mappings has changed." }
+
+        val (key, value) = mappings.entries.single()
+        val char = key.toHexCharLiteral()
+        val result = value.hexCharsToStringLiteral()
+
+        writer.appendLine(lowercaseImpl(char, result))
+    }
+
+    private fun lowercaseImpl(char: String, oneToManyResult: String): String = """
+        internal fun Char.lowercaseImpl(): String {
+            if (this == $char) {
+                return $oneToManyResult
+            }
+            return lowercaseCharImpl().toString()
+        }
+    """.trimIndent()
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyMappingsWriter.kt

@@ -0,0 +1,12 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToMany.writers
+
+import java.io.FileWriter
+
+internal interface OneToManyMappingsWriter {
+    fun write(mappings: Map<Int, List<String>>, writer: FileWriter)
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyTitlecaseMappingsWriter.kt

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToMany.writers
+
+import generators.unicode.ranges.RangesWritingStrategy
+import generators.unicode.writeMappings
+import java.io.FileWriter
+
+internal class OneToManyTitlecaseMappingsWriter(private val strategy: RangesWritingStrategy) : OneToManyMappingsWriter {
+    override fun write(mappings: Map<Int, List<String>>, writer: FileWriter) {
+        strategy.beforeWritingRanges(writer)
+        writer.writeMappings(mappings, strategy)
+        strategy.afterWritingRanges(writer)
+        writer.appendLine()
+        writer.appendLine(titlecaseImpl())
+    }
+
+    private fun titlecaseImpl(): String = """
+        internal fun Char.titlecaseImpl(): String {
+            val code = this.toInt()
+            val index = binarySearchRange(keys, code)
+            if (index >= 0 && keys[index] == code) {
+                return values[index]
+            }
+            return titlecaseCharImpl().toString()
+        }
+    """.trimIndent()
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyUppercaseMappingsWriter.kt

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToMany.writers
+
+import generators.unicode.ranges.RangesWritingStrategy
+import generators.unicode.toHexCharLiteral
+import generators.unicode.writeMappings
+import java.io.FileWriter
+
+internal class OneToManyUppercaseMappingsWriter(private val strategy: RangesWritingStrategy) : OneToManyMappingsWriter {
+    override fun write(mappings: Map<Int, List<String>>, writer: FileWriter) {
+        strategy.beforeWritingRanges(writer)
+        writer.writeMappings(mappings, strategy)
+        strategy.afterWritingRanges(writer)
+        writer.appendLine()
+        writer.appendLine(oneToManyUppercase(mappings.keys.first()))
+        writer.appendLine()
+        writer.appendLine(uppercaseImpl())
+    }
+
+    private fun oneToManyUppercase(smallestOneToMany: Int): String = """
+        internal fun Char.oneToManyUppercase(): String? {
+            if (this < ${smallestOneToMany.toHexCharLiteral()}) {
+                return null
+            }
+
+            val code = this.toInt()
+            val index = binarySearchRange(keys, code)
+            if (keys[index] == code) {
+                return values[index]
+            }
+            return null
+        }
+    """.trimIndent()
+
+    private fun uppercaseImpl(): String = """
+        internal fun Char.uppercaseImpl(): String {
+            return oneToManyUppercase() ?: uppercaseCharImpl().toString()
+        }
+    """.trimIndent()
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/MappingsGenerator.kt

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToOne
+
+import generators.requireExistingDir
+import generators.unicode.UnicodeDataLine
+import generators.unicode.mappings.oneToOne.builders.*
+import generators.unicode.mappings.oneToOne.writers.*
+import generators.unicode.ranges.RangesWritingStrategy
+import generators.unicode.writeHeader
+import templates.KotlinTarget
+import java.io.File
+import java.io.FileWriter
+
+internal class MappingsGenerator private constructor(
+    private val outputFile: File,
+    private val mappingsBuilder: MappingsBuilder,
+    private val mappingsWriter: MappingsWriter,
+) {
+
+    init {
+        outputFile.parentFile.requireExistingDir()
+    }
+
+    fun appendLine(line: UnicodeDataLine) {
+        mappingsBuilder.append(line)
+    }
+
+    fun generate() {
+        val mappings = mappingsBuilder.build()
+
+        FileWriter(outputFile).use { writer ->
+            writer.writeHeader(outputFile, "kotlin.text")
+            writer.appendLine()
+            writer.appendLine("// ${mappings.size} ranges totally")
+
+            mappingsWriter.write(mappings, writer)
+        }
+    }
+
+    companion object {
+        fun forUppercase(outputFile: File, target: KotlinTarget): MappingsGenerator {
+            val builder = UppercaseMappingsBuilder()
+            val writer = UppercaseMappingsWriter(RangesWritingStrategy.of(target, "Uppercase"))
+            return MappingsGenerator(outputFile, builder, writer)
+        }
+
+        fun forLowercase(outputFile: File, target: KotlinTarget): MappingsGenerator {
+            val builder = LowercaseMappingsBuilder()
+            val writer = LowercaseMappingsWriter(RangesWritingStrategy.of(target, "Lowercase"))
+            return MappingsGenerator(outputFile, builder, writer)
+        }
+
+        fun forTitlecase(outputFile: File): MappingsGenerator {
+            val builder = TitlecaseMappingsBuilder()
+            val writer = TitlecaseMappingsWriter()
+            return MappingsGenerator(outputFile, builder, writer)
+        }
+    }
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/LowercaseMappingsBuilder.kt

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToOne.builders
+
+import generators.unicode.UnicodeDataLine
+
+internal class LowercaseMappingsBuilder : MappingsBuilder() {
+    override fun mappingEquivalent(line: UnicodeDataLine): String? {
+        if (line.lowercaseMapping.isEmpty()) return null
+        check(line.char != line.lowercaseMapping) { "UnicodeData.txt format has changed!" }
+        return line.lowercaseMapping
+    }
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/MappingsBuilder.kt

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToOne.builders
+
+import generators.unicode.UnicodeDataLine
+import generators.unicode.mappings.oneToOne.patterns.EqualDistanceMappingPattern
+import generators.unicode.mappings.oneToOne.patterns.MappingPattern
+import generators.unicode.hexToInt
+
+/**
+ * The base class of character mappings builders.
+ */
+internal abstract class MappingsBuilder {
+    private val patterns = mutableListOf<MappingPattern>()
+
+    /**
+     * Appends a line from the UnicodeData.txt file.
+     */
+    fun append(line: UnicodeDataLine) {
+        val charCode = line.char.hexToInt()
+        val equivalent = mappingEquivalent(line)?.hexToInt() ?: return
+        val mapping = equivalent - charCode
+
+        check((charCode > Char.MAX_VALUE.toInt()) == (equivalent > Char.MAX_VALUE.toInt())) { "Handle when equivalent mapping is out of BMP." }
+
+        if (patterns.isEmpty()) {
+            patterns.add(createPattern(charCode, line.categoryCode, mapping))
+            return
+        }
+
+        val lastPattern = patterns.last()
+
+        if (!lastPattern.append(charCode, line.categoryCode, mapping)) {
+            val newLastPattern = evolveLastPattern(lastPattern, charCode, line.categoryCode, mapping)
+            if (newLastPattern != null) {
+                patterns[patterns.lastIndex] = newLastPattern
+            } else {
+                patterns.add(createPattern(charCode, line.categoryCode, mapping))
+            }
+        }
+    }
+
+    /**
+     * Returns the resulting mapping patterns.
+     */
+    fun build(): List<MappingPattern> {
+//        println(patterns.joinToString(separator = "\n"))
+//        println("${this.javaClass} # ${patterns.size}")
+        return patterns
+    }
+
+    /**
+     * Returns the mapping equivalent this builder is responsible for.
+     */
+    abstract fun mappingEquivalent(line: UnicodeDataLine): String?
+
+    /**
+     * Appends the [charCode] with the specified [categoryCode] and [mapping] to the [lastPattern] and returns the resulting pattern,
+     * or returns `null` if the [charCode] can't be appended to the [lastPattern].
+     * The [lastPattern] can be transformed to another pattern type to accommodate the [charCode].
+     */
+    protected open fun evolveLastPattern(lastPattern: MappingPattern, charCode: Int, categoryCode: String, mapping: Int): MappingPattern? {
+        return null
+    }
+
+    private fun createPattern(charCode: Int, categoryCode: String, mapping: Int): MappingPattern {
+        return EqualDistanceMappingPattern.from(charCode, categoryCode, mapping)
+    }
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/TitlecaseMappingsBuilder.kt

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToOne.builders
+
+import generators.unicode.UnicodeDataLine
+import generators.unicode.mappings.oneToOne.patterns.LuLtLlMappingPattern
+import generators.unicode.mappings.oneToOne.patterns.MappingPattern
+
+internal class TitlecaseMappingsBuilder : MappingsBuilder() {
+
+    override fun mappingEquivalent(line: UnicodeDataLine): String? {
+        if (line.titlecaseMapping == line.uppercaseMapping) return null
+        check(line.titlecaseMapping.isNotEmpty()) { "UnicodeData.txt format has changed!" }
+        return line.titlecaseMapping
+    }
+
+    override fun evolveLastPattern(lastPattern: MappingPattern, charCode: Int, categoryCode: String, mapping: Int): MappingPattern? {
+        return LuLtLlMappingPattern.from(lastPattern, charCode, categoryCode, mapping)
+    }
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/UppercaseMappingsBuilder.kt

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToOne.builders
+
+import generators.unicode.UnicodeDataLine
+
+internal class UppercaseMappingsBuilder : MappingsBuilder() {
+    override fun mappingEquivalent(line: UnicodeDataLine): String? {
+        if (line.uppercaseMapping.isEmpty()) return null
+        check(line.char != line.uppercaseMapping) { "UnicodeData.txt format has changed!" }
+        return line.uppercaseMapping
+    }
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/patterns/EqualDistanceMappingPattern.kt

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToOne.patterns
+
+import generators.unicode.toHexIntLiteral
+
+/**
+ * A range of consequent chars where all chars with code `start + k * distance` have the same [mapping] for case conversion.
+ *
+ * @param charCode the start of this range
+ * @param categoryCode the category code of the char with the specified [charCode]
+ * @param mapping the difference between [charCode] and the char it converts to.
+ */
+internal class EqualDistanceMappingPattern private constructor(
+    charCode: Int,
+    val categoryCode: String,
+    val mapping: Int
+) : MappingPattern {
+    override val start = charCode
+    override var end = charCode
+        private set
+    var distance = 1
+        private set
+
+    override fun append(charCode: Int, categoryCode: String, mapping: Int): Boolean {
+        require(charCode > end)
+
+        val distance = charCode - end
+        if (mapping == this.mapping
+            && length < MAX_LENGTH
+            && distance <= MAX_DISTANCE
+            && (distance == this.distance || length == 1)
+        ) {
+            // if length is equal to 1, distance can be changed
+            this.distance = distance
+            end = charCode
+            return true
+        }
+        return false
+    }
+
+    override fun toString(): String {
+        return "EqualDistanceMappingPattern{" +
+                "start=" + start.toHexIntLiteral() +
+                ", end=" + end.toHexIntLiteral() +
+                ", length=" + length +
+                ", distance=" + distance +
+                ", count=" + ((length + distance - 1) / distance) +
+                ", mapping=" + mapping +
+                "}"
+    }
+
+    companion object {
+        private const val MAX_DISTANCE = 15
+        private const val MAX_LENGTH = 255
+
+        fun from(charCode: Int, categoryCode: String, mapping: Int): EqualDistanceMappingPattern {
+            return EqualDistanceMappingPattern(charCode, categoryCode, mapping)
+        }
+    }
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/patterns/LuLtLlMappingPattern.kt

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToOne.patterns
+
+import generators.unicode.toHexIntLiteral
+
+/**
+ * A range of consequent chars that repeat <Lu, Lt, Ll> sequence of `categoryCode` and <1, 0, -1> sequence of `mapping`.
+ *
+ * @param charCode the start of this range
+ * @param categoryCode the category code of the char with the specified [charCode]
+ * @param mapping the difference between the [charCode] and the char it converts to.
+ */
+internal class LuLtLlMappingPattern private constructor(
+    charCode: Int,
+    categoryCode: String,
+    mapping: Int
+) : MappingPattern {
+    override val start = charCode
+    override var end = charCode
+        private set
+    val distance = 1
+
+    init {
+        require(categoryCode == "Lu" && mapping == 1)
+    }
+
+    override fun append(charCode: Int, categoryCode: String, mapping: Int): Boolean {
+        require(charCode > end)
+
+        val distance = charCode - end
+        val modThree = (charCode - start) % 3
+        val expectedMapping = expectedMapping(categoryCode)
+        val expectedModThree = expectedModThree(categoryCode)
+        if (distance == this.distance && mapping == expectedMapping && modThree == expectedModThree) {
+            end = charCode
+            return true
+        }
+        return false
+    }
+
+    private fun expectedModThree(categoryCode: String): Int = when (categoryCode) {
+        "Lu" -> 0
+        "Lt" -> 1
+        "Ll" -> 2
+        else -> error("Unexpected category: $categoryCode")
+    }
+
+    private fun expectedMapping(categoryCode: String): Int = when (categoryCode) {
+        "Lu" -> 1
+        "Lt" -> 0
+        "Ll" -> -1
+        else -> error("Unexpected category: $categoryCode")
+    }
+
+    override fun toString(): String {
+        return "LuLtLlMappingPattern{" +
+                "start=" + start.toHexIntLiteral() +
+                ", end=" + end.toHexIntLiteral() +
+                ", length=" + length +
+                ", distance=" + distance +
+                ", count=" + ((length + distance - 1) / distance) +
+                "}"
+    }
+
+    companion object {
+        fun from(pattern: MappingPattern, charCode: Int, categoryCode: String, mapping: Int): LuLtLlMappingPattern? {
+            if (pattern is EqualDistanceMappingPattern
+                && pattern.length == 1
+                && pattern.categoryCode == "Lu"
+                && pattern.mapping == 1
+            ) {
+                val newPattern = LuLtLlMappingPattern(pattern.start, pattern.categoryCode, pattern.mapping)
+                if (newPattern.append(charCode, categoryCode, mapping)) {
+                    return newPattern
+                }
+            }
+            return null
+        }
+    }
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/patterns/MappingPattern.kt

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToOne.patterns
+
+/**
+ * A range of consequent chars that fit within a particular mapping pattern.
+ */
+internal interface MappingPattern {
+    /**
+     * Char code of the first char in this range.
+     */
+    val start: Int
+
+    /**
+     * Char code of the last char in this range.
+     */
+    val end: Int
+
+    /**
+     * Appends the [charCode] to this range pattern.
+     * Returns true if the [charCode] with the specified [categoryCode] and [mapping] was accommodated within this pattern.
+     * Returns false otherwise.
+     *
+     * @param mapping the difference between the [charCode] and the char it converts to.
+     */
+    fun append(charCode: Int, categoryCode: String, mapping: Int): Boolean
+}
+
+internal val MappingPattern.length: Int get() = end - start + 1

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/LowercaseMappingsWriter.kt

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToOne.writers
+
+import generators.unicode.mappings.oneToOne.patterns.EqualDistanceMappingPattern
+import generators.unicode.mappings.oneToOne.patterns.length
+import generators.unicode.mappings.oneToOne.patterns.MappingPattern
+import generators.unicode.ranges.RangesWritingStrategy
+import generators.unicode.writeIntArray
+import java.io.FileWriter
+
+internal class LowercaseMappingsWriter(private val strategy: RangesWritingStrategy) : MappingsWriter {
+    override fun write(mappings: List<MappingPattern>, writer: FileWriter) {
+        @Suppress("UNCHECKED_CAST")
+        val distanceMappings = mappings as List<EqualDistanceMappingPattern>
+
+        val start = distanceMappings.map { it.start }
+        val length = distanceMappings.map { (it.mapping shl 12) or (it.distance shl 8) or it.length }
+
+        strategy.beforeWritingRanges(writer)
+        writer.writeIntArray("rangeStart", start, strategy)
+        writer.appendLine()
+        writer.writeIntArray("rangeLength", length, strategy)
+        strategy.afterWritingRanges(writer)
+        writer.appendLine()
+        writer.appendLine(lowercaseCodePoint())
+        writer.appendLine()
+        writer.appendLine(lowercaseCharImpl())
+    }
+
+    private fun lowercaseCodePoint(): String = """
+        internal fun Int.lowercaseCodePoint(): Int {
+            if (this in 0x41..0x5a) {
+                return this + 32
+            }
+            if (this < 0x80) {
+                return this
+            }
+            val index = binarySearchRange(rangeStart, this)
+            return equalDistanceMapping(this, rangeStart[index], rangeLength[index])
+        }
+    """.trimIndent()
+
+    private fun lowercaseCharImpl(): String = """
+        internal fun Char.lowercaseCharImpl(): Char {
+            return toInt().lowercaseCodePoint().toChar()
+        }
+    """.trimIndent()
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/MappingsWriter.kt

@@ -0,0 +1,13 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToOne.writers
+
+import generators.unicode.mappings.oneToOne.patterns.MappingPattern
+import java.io.FileWriter
+
+internal interface MappingsWriter {
+    fun write(mappings: List<MappingPattern>, writer: FileWriter)
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/TitlecaseMappingsWriter.kt

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToOne.writers
+
+import generators.unicode.mappings.oneToOne.patterns.EqualDistanceMappingPattern
+import generators.unicode.mappings.oneToOne.patterns.LuLtLlMappingPattern
+import generators.unicode.mappings.oneToOne.patterns.MappingPattern
+import generators.unicode.toHexIntLiteral
+import java.io.FileWriter
+
+internal class TitlecaseMappingsWriter : MappingsWriter {
+
+    override fun write(mappings: List<MappingPattern>, writer: FileWriter) {
+        val LuLtLlMappings = mappings.filterIsInstance<LuLtLlMappingPattern>()
+        val zeroMappings = mappings.filterIsInstance<EqualDistanceMappingPattern>().filter { it.distance == 1 && it.mapping == 0 }
+
+        check(LuLtLlMappings.size + zeroMappings.size == mappings.size) { "Handle new types of titlecase mapping." }
+        check(LuLtLlMappings.all { it.start % 3 == 2 }) { "Handle when code of the Lt char is not multiple of 3." }
+
+        writer.append(
+            """
+            internal fun Char.titlecaseCharImpl(): Char {
+                val code = this.toInt()
+                if (${rangeChecks(LuLtLlMappings, "code")}) {
+                    return (3 * ((code + 1) / 3)).toChar()
+                }
+                if (${rangeChecks(zeroMappings, "code")}) {
+                    return this
+                }
+                return uppercaseCharImpl()
+            }
+            """.trimIndent()
+        )
+    }
+
+    private fun rangeChecks(mappings: List<MappingPattern>, code: String): String {
+        return mappings.joinToString(separator = " || ") { "$code in ${it.start.toHexIntLiteral()}..${it.end.toHexIntLiteral()}" }
+    }
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/UppercaseMappingsWriter.kt

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package generators.unicode.mappings.oneToOne.writers
+
+import generators.unicode.mappings.oneToOne.patterns.EqualDistanceMappingPattern
+import generators.unicode.mappings.oneToOne.patterns.length
+import generators.unicode.mappings.oneToOne.patterns.MappingPattern
+import generators.unicode.ranges.RangesWritingStrategy
+import generators.unicode.writeIntArray
+import java.io.FileWriter
+
+internal class UppercaseMappingsWriter(private val strategy: RangesWritingStrategy) : MappingsWriter {
+    override fun write(mappings: List<MappingPattern>, writer: FileWriter) {
+        @Suppress("UNCHECKED_CAST")
+        val distanceMappings = mappings as List<EqualDistanceMappingPattern>
+
+        val start = distanceMappings.map { it.start }
+        val length = distanceMappings.map { (it.mapping shl 12) or (it.distance shl 8) or it.length }
+
+        strategy.beforeWritingRanges(writer)
+        writer.writeIntArray("rangeStart", start, strategy)
+        writer.appendLine()
+        writer.writeIntArray("rangeLength", length, strategy)
+        strategy.afterWritingRanges(writer)
+        writer.appendLine()
+        writer.appendLine(equalDistanceMapping())
+        writer.appendLine()
+        writer.appendLine(uppercaseCodePoint())
+        writer.appendLine()
+        writer.appendLine(uppercaseCharImpl())
+    }
+
+    private fun equalDistanceMapping(): String = """
+        internal fun equalDistanceMapping(code: Int, start: Int, pattern: Int): Int {
+            val diff = code - start
+
+            val length = pattern and 0xff
+            if (diff >= length) {
+                return code
+            }
+
+            val distance = (pattern shr 8) and 0xf
+            if (diff % distance != 0) {
+                return code
+            }
+
+            val mapping = pattern shr 12
+            return code + mapping
+        }
+    """.trimIndent()
+
+    private fun uppercaseCodePoint(): String = """
+        internal fun Int.uppercaseCodePoint(): Int {
+            if (this in 0x61..0x7a) {
+                return this - 32
+            }
+            if (this < 0x80) {
+                return this
+            }
+            val index = binarySearchRange(rangeStart, this)
+            return equalDistanceMapping(this, rangeStart[index], rangeLength[index])
+        }
+    """.trimIndent()
+
+    private fun uppercaseCharImpl(): String = """
+        internal fun Char.uppercaseCharImpl(): Char {
+            return toInt().uppercaseCodePoint().toChar()
+        }
+    """.trimIndent()
+}

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/CharCategoryTestGenerator.kt

@@ -5,12 +5,12 @@
 
 package generators.unicode.ranges
 
-import generators.unicode.UnicodeDataGenerator
+import generators.unicode.UnicodeDataLine
-import generators.unicode.ranges.writers.writeHeader
+import generators.unicode.writeHeader
 import java.io.File
 import java.io.FileWriter
 
-internal class CharCategoryTestGenerator(private val outputFile: File) : UnicodeDataGenerator {
+internal class CharCategoryTestGenerator(private val outputFile: File) {
     private var arrayIndex = 0
     private var arraySize = 0
     private var writer: FileWriter? = null
@@ -19,7 +19,7 @@ internal class CharCategoryTestGenerator(private val outputFile: File) : Unicode
         outputFile.parentFile.mkdirs()
     }
 
-    override fun appendChar(char: String, name: String, categoryCode: String) {
+    fun appendLine(line: UnicodeDataLine) {
         if (arraySize == 0) {
             writer?.appendLine(")")
             writer?.close()
@@ -27,9 +27,9 @@ internal class CharCategoryTestGenerator(private val outputFile: File) : Unicode
             generateUnicodeDataHeader(arrayIndex)
         }
 
-        val isStart = name.endsWith(", First>")
+        val isStart = line.name.endsWith(", First>")
 
-        writer?.appendLine("    CharProperties(char = '\\u$char', isStartOfARange = $isStart, categoryCode = \"$categoryCode\"),")
+        writer?.appendLine("    CharProperties(char = '\\u${line.char}', isStartOfARange = $isStart, categoryCode = \"${line.categoryCode}\"),")
 
         arraySize++
         if (arraySize == 2048) {
@@ -38,7 +38,7 @@ internal class CharCategoryTestGenerator(private val outputFile: File) : Unicode
         }
     }
 
-    override fun close() {
+    fun generate() {
         writer?.appendLine(")")
         writer?.close()
 

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/RangesGenerator.kt

@@ -6,9 +6,10 @@
 package generators.unicode.ranges
 
 import generators.requireExistingDir
-import generators.unicode.UnicodeDataGenerator
+import generators.unicode.UnicodeDataLine
 import generators.unicode.ranges.builders.*
 import generators.unicode.ranges.writers.*
+import generators.unicode.writeHeader
 import templates.KotlinTarget
 import templates.Platform
 import java.io.File
@@ -18,17 +19,17 @@ internal class RangesGenerator private constructor(
     private val outputFile: File,
     private val rangesBuilder: RangesBuilder,
     private val rangesWriter: RangesWriter,
-) : UnicodeDataGenerator {
+) {
 
     init {
         outputFile.parentFile.requireExistingDir()
     }
 
-    override fun appendChar(char: String, name: String, categoryCode: String) {
+    fun appendLine(line: UnicodeDataLine) {
-        rangesBuilder.append(char, name, categoryCode)
+        rangesBuilder.append(line.char, line.name, line.categoryCode)
     }
 
-    override fun close() {
+    fun generate() {
         val (rangeStart, rangeEnd, rangeCategory) = rangesBuilder.build()
 
         FileWriter(outputFile).use { writer ->

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/builders/DigitRangesBuilder.kt

@@ -5,8 +5,6 @@
 
 package generators.unicode.ranges.builders
 
-import generators.unicode.ranges.patterns.RangePattern
-
 internal class DigitRangesBuilder : RangesBuilder() {
     override fun categoryId(categoryCode: String): String {
         return categoryCode
@@ -15,11 +13,4 @@ internal class DigitRangesBuilder : RangesBuilder() {
     override fun shouldSkip(categoryId: String): Boolean {
         return categoryId != CharCategory.DECIMAL_DIGIT_NUMBER.code
     }
-
-    override val makeOnePeriodCategory: (Array<String>) -> Int
-        get() = { 0 }
-
-    override fun evolveLastRange(lastRange: RangePattern, charCode: Int, categoryId: String): RangePattern? {
-        return null
-    }
 }

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/builders/RangesBuilder.kt

@@ -8,6 +8,7 @@ package generators.unicode.ranges.builders
 import generators.unicode.ranges.patterns.PeriodicRangePattern
 import generators.unicode.ranges.patterns.RangePattern
 import generators.unicode.ranges.patterns.rangeLength
+import generators.unicode.hexToInt
 
 /**
  * The base class of character ranges builders.
@@ -20,7 +21,7 @@ internal abstract class RangesBuilder {
      * Appends a line from the UnicodeData.txt file.
      */
     fun append(char: String, name: String, categoryCode: String) {
-        val charCode = char.toInt(radix = 16)
+        val charCode = char.hexToInt()
         val categoryId = categoryId(categoryCode)
 
         when {
@@ -177,16 +178,12 @@ internal abstract class RangesBuilder {
     /**
      * The function to use to transform periodic ranges with period equal to 1 to an Int representation.
      */
-    protected abstract val makeOnePeriodCategory: (Array<String>) -> Int
+    protected open val makeOnePeriodCategory: (Array<String>) -> Int = { 0 }
 
     /**
      * Appends the [charCode] with the specified [categoryId] to the [lastRange] and returns the resulting range,
      * or returns `null` if [charCode] can't be appended to the [lastRange].
      * The [lastRange] can be transformed to another range type to accommodate the [charCode].
      */
-    protected abstract fun evolveLastRange(
+    protected open fun evolveLastRange(lastRange: RangePattern, charCode: Int, categoryId: String): RangePattern? = null
-        lastRange: RangePattern,
-        charCode: Int,
-        categoryId: String
-    ): RangePattern?
 }

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/builders/WhitespaceRangesBuilder.kt

@@ -5,8 +5,6 @@
 
 package generators.unicode.ranges.builders
 
-import generators.unicode.ranges.patterns.RangePattern
-
 internal class WhitespaceRangesBuilder : RangesBuilder() {
 
     init {
@@ -24,13 +22,6 @@ internal class WhitespaceRangesBuilder : RangesBuilder() {
     override fun shouldSkip(categoryId: String): Boolean {
         return categoryId == NOT_WS
     }
-
-    override val makeOnePeriodCategory: (Array<String>) -> Int
-        get() = { 0 }
-
-    override fun evolveLastRange(lastRange: RangePattern, charCode: Int, categoryId: String): RangePattern? {
-        return null
-    }
 }
 
 private const val WS = "WS"

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/patterns/GapRangePattern.kt

@@ -5,7 +5,7 @@
 
 package generators.unicode.ranges.patterns
 
-import generators.unicode.ranges.writers.hex
+import generators.unicode.toHexIntLiteral
 
 /**
  * A range of consequent chars that starts with a letter and ends with a letter, and contains multiple ranges of consequent not-letter chars.
@@ -95,8 +95,8 @@ internal class GapRangePattern private constructor(
 
     override fun toString(): String {
         return "GapPattern{" +
-                "start=" + start.hex() +
+                "start=" + start.toHexIntLiteral() +
-                ", end=" + end.hex() +
+                ", end=" + end.toHexIntLiteral() +
                 ", length=" + rangeLength() +
                 ", gaps=" + gaps +
                 ", categoryId=" + categoryId +

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/patterns/PeriodicRangePattern.kt

@@ -5,7 +5,7 @@
 
 package generators.unicode.ranges.patterns
 
-import generators.unicode.ranges.writers.hex
+import generators.unicode.toHexIntLiteral
 
 /**
  * A range of consequent chars.
@@ -76,7 +76,7 @@ internal class PeriodicRangePattern private constructor(
 
     override fun categoryIdOf(charCode: Int): String {
         if (charCode !in start..end) {
-            throw IllegalArgumentException("Char code ${charCode.hex()} is not in $this")
+            throw IllegalArgumentException("Char code ${charCode.toHexIntLiteral()} is not in $this")
         }
         val categoryId = bag.categoryIdOf(charCode)
         check(categoryId != null)
@@ -85,8 +85,8 @@ internal class PeriodicRangePattern private constructor(
 
     override fun toString(): String {
         return "PeriodicRangePattern{" +
-                "start=" + start.hex() +
+                "start=" + start.toHexIntLiteral() +
-                ", end=" + end.hex() +
+                ", end=" + end.toHexIntLiteral() +
                 ", length=" + rangeLength() +
                 ", orderedCategoryIds=" + orderedCategoryIds().contentToString() +
                 ", bag=" + bag +

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/CategoryRangesWriter.kt

@@ -5,8 +5,11 @@
 
 package generators.unicode.ranges.writers
 
+import generators.unicode.TO_BASE64
 import generators.unicode.ranges.RangesWritingStrategy
 import generators.unicode.ranges.builders.UNASSIGNED_CATEGORY_VALUE_REPLACEMENT
+import generators.unicode.toVarLenBase64
+import generators.unicode.writeIntArray
 import java.io.FileWriter
 
 internal open class CategoryRangesWriter(protected val strategy: RangesWritingStrategy) : RangesWriter {

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/DigitRangesWriter.kt

@@ -6,6 +6,7 @@
 package generators.unicode.ranges.writers
 
 import generators.unicode.ranges.RangesWritingStrategy
+import generators.unicode.writeIntArray
 import java.io.FileWriter
 
 internal class DigitRangesWriter(private val strategy: RangesWritingStrategy) : RangesWriter {

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/LetterRangesWriter.kt

@@ -5,8 +5,11 @@
 
 package generators.unicode.ranges.writers
 
+import generators.unicode.TO_BASE64
 import generators.unicode.ranges.RangesWritingStrategy
 import generators.unicode.ranges.patterns.GapRangePattern
+import generators.unicode.toVarLenBase64
+import generators.unicode.writeIntArray
 import java.io.FileWriter
 
 internal open class LetterRangesWriter(protected val strategy: RangesWritingStrategy) : RangesWriter {

libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/WhitespaceRangesWriter.kt

@@ -5,6 +5,7 @@
 
 package generators.unicode.ranges.writers
 
+import generators.unicode.toHexIntLiteral
 import java.io.FileWriter
 
 internal class WhitespaceRangesWriter : RangesWriter {
@@ -44,15 +45,15 @@ internal class WhitespaceRangesWriter : RangesWriter {
                         tabCount = 6
                         builder.append(tab.repeat(tabCount))
                     }
-                    builder.appendLine("$ch == ${start.hex()}")
+                    builder.appendLine("$ch == ${start.toHexIntLiteral()}")
                 }
                 end - 1 -> {
-                    builder.appendLine("$ch == ${start.hex()}")
+                    builder.appendLine("$ch == ${start.toHexIntLiteral()}")
                     builder.append(tab.repeat(tabCount)).append("|| ")
-                    builder.appendLine("$ch == ${end.hex()}")
+                    builder.appendLine("$ch == ${end.toHexIntLiteral()}")
                 }
                 else -> {
-                    builder.appendLine("$ch in ${start.hex()}..${end.hex()}")
+                    builder.appendLine("$ch in ${start.toHexIntLiteral()}..${end.toHexIntLiteral()}")
                 }
             }
         }