From 1066de1be6c5fd9f2193a762bb963d067a7c7574 Mon Sep 17 00:00:00 2001 From: Abduqodiri Qurbonzoda Date: Wed, 27 Jan 2021 15:54:11 +0300 Subject: [PATCH] Implement Char one-to-one and one-to-many case conversion generators #KT-45213 --- .../{ranges/writers => }/FileWriterUtils.kt | 40 +++++-- .../generators/unicode/GenerateUnicodeData.kt | 103 +++++++++++++----- .../generators/unicode/SpecialCasingLine.kt | 27 +++++ .../src/generators/unicode/UnicodeDataLine.kt | 29 +++++ .../oneToMany/OneToManyMappingsGenerator.kt | 59 ++++++++++ .../OneToManyLowercaseMappingsBuilder.kt | 14 +++ .../builders/OneToManyMappingsBuilder.kt | 58 ++++++++++ .../OneToManyTitlecaseMappingsBuilder.kt | 14 +++ .../OneToManyUppercaseMappingsBuilder.kt | 14 +++ .../OneToManyLowercaseMappingsWriter.kt | 32 ++++++ .../writers/OneToManyMappingsWriter.kt | 12 ++ .../OneToManyTitlecaseMappingsWriter.kt | 31 ++++++ .../OneToManyUppercaseMappingsWriter.kt | 44 ++++++++ .../mappings/oneToOne/MappingsGenerator.kt | 63 +++++++++++ .../builders/LowercaseMappingsBuilder.kt | 16 +++ .../oneToOne/builders/MappingsBuilder.kt | 72 ++++++++++++ .../builders/TitlecaseMappingsBuilder.kt | 23 ++++ .../builders/UppercaseMappingsBuilder.kt | 16 +++ .../patterns/EqualDistanceMappingPattern.kt | 64 +++++++++++ .../oneToOne/patterns/LuLtLlMappingPattern.kt | 84 ++++++++++++++ .../oneToOne/patterns/MappingPattern.kt | 32 ++++++ .../writers/LowercaseMappingsWriter.kt | 52 +++++++++ .../oneToOne/writers/MappingsWriter.kt | 13 +++ .../writers/TitlecaseMappingsWriter.kt | 42 +++++++ .../writers/UppercaseMappingsWriter.kt | 73 +++++++++++++ .../ranges/CharCategoryTestGenerator.kt | 14 +-- .../unicode/ranges/RangesGenerator.kt | 11 +- .../ranges/builders/DigitRangesBuilder.kt | 9 -- .../unicode/ranges/builders/RangesBuilder.kt | 11 +- .../builders/WhitespaceRangesBuilder.kt | 9 -- .../ranges/patterns/GapRangePattern.kt | 6 +- .../ranges/patterns/PeriodicRangePattern.kt | 8 +- .../ranges/writers/CategoryRangesWriter.kt | 3 + .../ranges/writers/DigitRangesWriter.kt | 1 + .../ranges/writers/LetterRangesWriter.kt | 3 + .../ranges/writers/WhitespaceRangesWriter.kt | 9 +- 36 files changed, 1029 insertions(+), 82 deletions(-) rename libraries/tools/kotlin-stdlib-gen/src/generators/unicode/{ranges/writers => }/FileWriterUtils.kt (61%) create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/SpecialCasingLine.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/UnicodeDataLine.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/OneToManyMappingsGenerator.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyLowercaseMappingsBuilder.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyMappingsBuilder.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyTitlecaseMappingsBuilder.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyUppercaseMappingsBuilder.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyLowercaseMappingsWriter.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyMappingsWriter.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyTitlecaseMappingsWriter.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyUppercaseMappingsWriter.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/MappingsGenerator.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/LowercaseMappingsBuilder.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/MappingsBuilder.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/TitlecaseMappingsBuilder.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/UppercaseMappingsBuilder.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/patterns/EqualDistanceMappingPattern.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/patterns/LuLtLlMappingPattern.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/patterns/MappingPattern.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/LowercaseMappingsWriter.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/MappingsWriter.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/TitlecaseMappingsWriter.kt create mode 100644 libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/UppercaseMappingsWriter.kt diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/FileWriterUtils.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/FileWriterUtils.kt similarity index 61% rename from libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/FileWriterUtils.kt rename to libraries/tools/kotlin-stdlib-gen/src/generators/unicode/FileWriterUtils.kt index c8d321c6b0f..f8d1bcca978 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/FileWriterUtils.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/FileWriterUtils.kt @@ -1,9 +1,9 @@ /* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ -package generators.unicode.ranges.writers +package generators.unicode import generators.unicode.ranges.RangesWritingStrategy import templates.COPYRIGHT_NOTICE @@ -19,9 +19,14 @@ internal fun FileWriter.writeHeader(file: File, pkg: String) { appendLine(autoGeneratedWarning("GenerateUnicodeData.kt")) } -internal fun FileWriter.writeIntArray( +internal fun FileWriter.writeIntArray(name: String, elements: List, strategy: RangesWritingStrategy) { + writeCollection(name, "intArrayOf", elements.map { it.toHexIntLiteral() }, strategy) +} + +private fun FileWriter.writeCollection( name: String, - elements: List, + constructingFun: String, + elements: List, strategy: RangesWritingStrategy ) { fun appendWithIndentation(string: String) { @@ -29,19 +34,26 @@ internal fun FileWriter.writeIntArray( } append(strategy.rangesAnnotation) - appendWithIndentation("${strategy.rangesVisibilityModifier} val $name = intArrayOf(") + appendWithIndentation("${strategy.rangesVisibilityModifier} val $name = $constructingFun(") for (i in elements.indices) { if (i % 20 == 0) { appendLine() appendWithIndentation(" ") } - append(elements[i].hex() + ", ") + append(elements[i] + ", ") } appendLine() appendWithIndentation(")") appendLine() } +internal fun FileWriter.writeMappings(mappings: Map>, strategy: RangesWritingStrategy) { + val keys = mappings.keys.map { it.toHexIntLiteral() } + writeCollection("keys", "intArrayOf", keys, strategy) + val values = mappings.values.map { it.hexCharsToStringLiteral() } + writeCollection("values", "arrayOf", values, strategy) +} + internal const val TO_BASE64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" internal fun List.toVarLenBase64(): String { @@ -67,10 +79,22 @@ private fun Int.to6Bits(): List { return result } -internal fun Int.hex(): String { +internal fun Int.toHexIntLiteral(): String { val result = toString(radix = 16) if (result.first() == '-') { return "-0x" + result.substring(startIndex = 1).padStart(4, '0') } return "0x" + result.padStart(4, '0') -} \ No newline at end of file +} + +internal fun Int.toHexCharLiteral(): String { + return "'\\u${toString(radix = 16).padStart(4, '0')}'" +} + +internal fun String.hexToInt(): Int { + return toInt(radix = 16) +} + +internal fun List.hexCharsToStringLiteral(): String { + return "\"${joinToString(separator = "") { "\\u$it" }}\"" +} diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/GenerateUnicodeData.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/GenerateUnicodeData.kt index 41f057d784c..970c9386390 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/GenerateUnicodeData.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/GenerateUnicodeData.kt @@ -5,6 +5,8 @@ package generators.unicode +import generators.unicode.mappings.oneToOne.MappingsGenerator +import generators.unicode.mappings.oneToMany.OneToManyMappingsGenerator import generators.unicode.ranges.CharCategoryTestGenerator import generators.unicode.ranges.RangesGenerator import templates.COPYRIGHT_NOTICE @@ -16,10 +18,12 @@ import kotlin.system.exitProcess // Go to https://www.unicode.org/versions/latest/ to find out the latest public version of the Unicode Character Database files. -private const val unicodeDataUrl = "https://www.unicode.org/Public/13.0.0/ucd/UnicodeData.txt" +private const val unicodeVersion = "13.0.0" +private const val unicodeDataUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/UnicodeData.txt" +private const val specialCasingUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/SpecialCasing.txt" /** - * This program generates sources related to UnicodeData.txt. + * This program generates sources related to UnicodeData.txt and SpecialCasing.txt. * There are two ways to run the program. * 1. Pass the root directory of the project to generate sources for js and js-ir. * _CharCategoryTest.kt and supporting files are also generated to test the generated sources. @@ -30,27 +34,57 @@ private const val unicodeDataUrl = "https://www.unicode.org/Public/13.0.0/ucd/Un fun main(args: Array) { val unicodeDataLines = URL(unicodeDataUrl).openStream().reader().readLines() + val bmpUnicodeDataLines = unicodeDataLines.mapNotNull { line -> + UnicodeDataLine(line.split(";")).takeIf { it.char.length <= 4 } // Basic Multilingual Plane (BMP) + } + val specialCasingLines = URL(specialCasingUrl).openStream().reader().readLines() + val nonCommentSpecialCasingLines = specialCasingLines.mapNotNull { line -> + if (line.isEmpty() || line.startsWith("#")) null else SpecialCasingLine(line.split("; ")) + } - val generators = mutableListOf() + val categoryRangesGenerators = mutableListOf() fun addRangesGenerators(generatedDir: File, target: KotlinTarget) { - val categoryRangesGenerator = RangesGenerator.forCharCategory(generatedDir.resolve("_CharCategories.kt"), target) - val digitRangesGenerator = RangesGenerator.forDigit(generatedDir.resolve("_DigitChars.kt"), target) - val letterRangesGenerator = RangesGenerator.forLetter(generatedDir.resolve("_LetterChars.kt"), target) - val whitespaceRangesGenerator = RangesGenerator.forWhitespace(generatedDir.resolve("_WhitespaceChars.kt")) - generators.add(categoryRangesGenerator) - generators.add(digitRangesGenerator) - generators.add(letterRangesGenerator) - generators.add(whitespaceRangesGenerator) + val category = RangesGenerator.forCharCategory(generatedDir.resolve("_CharCategories.kt"), target) + val digit = RangesGenerator.forDigit(generatedDir.resolve("_DigitChars.kt"), target) + val letter = RangesGenerator.forLetter(generatedDir.resolve("_LetterChars.kt"), target) + val whitespace = RangesGenerator.forWhitespace(generatedDir.resolve("_WhitespaceChars.kt")) + categoryRangesGenerators.add(category) + categoryRangesGenerators.add(digit) + categoryRangesGenerators.add(letter) + categoryRangesGenerators.add(whitespace) } + val oneToOneMappingsGenerators = mutableListOf() + + fun addOneToOneMappingsGenerators(generatedDir: File, target: KotlinTarget) { + val uppercase = MappingsGenerator.forUppercase(generatedDir.resolve("_UppercaseMappings.kt"), target) + val lowercase = MappingsGenerator.forLowercase(generatedDir.resolve("_LowercaseMappings.kt"), target) + val titlecase = MappingsGenerator.forTitlecase(generatedDir.resolve("_TitlecaseMappings.kt")) + oneToOneMappingsGenerators.add(uppercase) + oneToOneMappingsGenerators.add(lowercase) + oneToOneMappingsGenerators.add(titlecase) + } + + val oneToManyMappingsGenerators = mutableListOf() + + fun addOneToManyMappingsGenerators(generatedDir: File, target: KotlinTarget) { + val uppercase = OneToManyMappingsGenerator.forUppercase(generatedDir.resolve("_OneToManyUppercaseMappings.kt"), target, bmpUnicodeDataLines) + val lowercase = OneToManyMappingsGenerator.forLowercase(generatedDir.resolve("_OneToManyLowercaseMappings.kt"), target, bmpUnicodeDataLines) + val titlecase = OneToManyMappingsGenerator.forTitlecase(generatedDir.resolve("_OneToManyTitlecaseMappings.kt"), target, bmpUnicodeDataLines) + oneToManyMappingsGenerators.add(uppercase) + oneToManyMappingsGenerators.add(lowercase) + oneToManyMappingsGenerators.add(titlecase) + } + + var categoryTestGenerator: CharCategoryTestGenerator? = null + when (args.size) { 1 -> { val baseDir = File(args.first()) val categoryTestFile = baseDir.resolve("libraries/stdlib/js/test/text/unicodeData/_CharCategoryTest.kt") - val categoryTestGenerator = CharCategoryTestGenerator(categoryTestFile) - generators.add(categoryTestGenerator) + categoryTestGenerator = CharCategoryTestGenerator(categoryTestFile) val jsGeneratedDir = baseDir.resolve("libraries/stdlib/js/src/generated/") addRangesGenerators(jsGeneratedDir, KotlinTarget.JS) @@ -59,8 +93,12 @@ fun main(args: Array) { addRangesGenerators(jsIrGeneratedDir, KotlinTarget.JS_IR) // For debugging. To see the file content - val unicodeDataFile = baseDir.resolve("libraries/tools/kotlin-stdlib-gen/src/generators/unicode/UnicodeData.txt") - unicodeDataFile.writeText(unicodeDataLines.joinToString(separator = "\n")) + fun writeContent(lines: List, fileName: String) { + val file = baseDir.resolve("libraries/tools/kotlin-stdlib-gen/src/generators/unicode/$fileName") + file.writeText(lines.joinToString(separator = "\n")) + } + writeContent(unicodeDataLines, "UnicodeData.txt") + writeContent(specialCasingLines, "SpecialCasing.txt") } 2 -> { val (targetName, targetDir) = args @@ -68,7 +106,13 @@ fun main(args: Array) { val target = KotlinTarget.values.singleOrNull { it.name.equals(targetName, ignoreCase = true) } ?: error("Invalid target: $targetName") - addRangesGenerators(File(targetDir), target) + val generatedDir = File(targetDir) + addRangesGenerators(generatedDir, target) + + if (target == KotlinTarget.Native) { + addOneToOneMappingsGenerators(generatedDir, target) + addOneToManyMappingsGenerators(generatedDir, target) + } } else -> { println( @@ -84,16 +128,23 @@ fun main(args: Array) { COPYRIGHT_NOTICE = readCopyrightNoticeFromProfile { Thread.currentThread().contextClassLoader.getResourceAsStream("apache.xml").reader() } - unicodeDataLines.forEach { line -> - val parts = line.split(";") - if (parts[0].length <= 4) { - generators.forEach { it.appendChar(parts[0], parts[1], parts[2]) } - } + categoryRangesGenerators.forEach { + bmpUnicodeDataLines.forEach { line -> it.appendLine(line) } + it.generate() } - generators.forEach { it.close() } -} -internal interface UnicodeDataGenerator { - fun appendChar(char: String, name: String, categoryCode: String) - fun close() + categoryTestGenerator?.let { + bmpUnicodeDataLines.forEach { line -> it.appendLine(line) } + it.generate() + } + + oneToOneMappingsGenerators.forEach { + unicodeDataLines.forEach { line -> it.appendLine(line) } + it.generate() + } + + oneToManyMappingsGenerators.forEach { + nonCommentSpecialCasingLines.forEach { line -> it.appendLine(line) } + it.generate() + } } diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/SpecialCasingLine.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/SpecialCasingLine.kt new file mode 100644 index 00000000000..98b668643cb --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/SpecialCasingLine.kt @@ -0,0 +1,27 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode + +internal class SpecialCasingLine(properties: List) { + init { + require(properties.size in 5..6) + } + + val char: String = properties[0] + val lowercaseMapping: List = properties[1].split(" ") + val titlecaseMapping: List = properties[2].split(" ") + val uppercaseMapping: List = properties[3].split(" ") + val conditionList: List = if (properties.size == 6) properties[4].split(" ") else emptyList() + + override fun toString(): String { + return "SpecialCasingLine{char=$char" + + ", lowercaseMapping=$lowercaseMapping" + + ", uppercaseMapping=$uppercaseMapping" + + ", titlecaseMapping=$titlecaseMapping" + + ", conditionList=$conditionList" + + "}" + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/UnicodeDataLine.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/UnicodeDataLine.kt new file mode 100644 index 00000000000..a1276da062b --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/UnicodeDataLine.kt @@ -0,0 +1,29 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode + +internal class UnicodeDataLine(properties: List) { + init { + require(properties.size == 15) + } + + val char: String = properties[0] + val name: String = properties[1] + val categoryCode: String = properties[2] + val uppercaseMapping: String = properties[12] + val lowercaseMapping: String = properties[13] + val titlecaseMapping: String = properties[14] + + override fun toString(): String { + return "UnicodeDataLine{char=$char" + + ", categoryCode=$categoryCode" + + ", uppercaseMapping=$uppercaseMapping" + + ", lowercaseMapping=$lowercaseMapping" + + ", titlecaseMapping=$titlecaseMapping" + + ", name=$name" + + "}" + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/OneToManyMappingsGenerator.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/OneToManyMappingsGenerator.kt new file mode 100644 index 00000000000..6561b85c5e6 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/OneToManyMappingsGenerator.kt @@ -0,0 +1,59 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToMany + +import generators.unicode.SpecialCasingLine +import generators.unicode.UnicodeDataLine +import generators.unicode.mappings.oneToMany.builders.* +import generators.unicode.mappings.oneToMany.writers.* +import generators.unicode.ranges.RangesWritingStrategy +import generators.unicode.writeHeader +import templates.KotlinTarget +import java.io.File +import java.io.FileWriter + +internal class OneToManyMappingsGenerator private constructor( + private val outputFile: File, + private val mappingsBuilder: OneToManyMappingsBuilder, + private val mappingsWriter: OneToManyMappingsWriter +) { + fun appendLine(line: SpecialCasingLine) { + mappingsBuilder.append(line) + } + + fun generate() { + val mappings = mappingsBuilder.build() + + FileWriter(outputFile).use { writer -> + writer.writeHeader(outputFile, "kotlin.text") + writer.appendLine() + writer.appendLine("// ${mappings.size} mappings totally") + + mappingsWriter.write(mappings, writer) + } + } + + + companion object { + fun forUppercase(outputFile: File, target: KotlinTarget, bmpUnicodeDataLines: List): OneToManyMappingsGenerator { + val builder = OneToManyUppercaseMappingsBuilder(bmpUnicodeDataLines) + val writer = OneToManyUppercaseMappingsWriter(RangesWritingStrategy.of(target, "OneToManyUppercase")) + return OneToManyMappingsGenerator(outputFile, builder, writer) + } + + fun forLowercase(outputFile: File, target: KotlinTarget, bmpUnicodeDataLines: List): OneToManyMappingsGenerator { + val builder = OneToManyLowercaseMappingsBuilder(bmpUnicodeDataLines) + val writer = OneToManyLowercaseMappingsWriter(RangesWritingStrategy.of(target, "OneToManyLowercase")) + return OneToManyMappingsGenerator(outputFile, builder, writer) + } + + fun forTitlecase(outputFile: File, bmpUnicodeDataLines: List): OneToManyMappingsGenerator { + val builder = OneToManyTitlecaseMappingsBuilder(bmpUnicodeDataLines) + val writer = OneToManyTitlecaseMappingsWriter() + return OneToManyMappingsGenerator(outputFile, builder, writer) + } + } +} diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyLowercaseMappingsBuilder.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyLowercaseMappingsBuilder.kt new file mode 100644 index 00000000000..97f07030956 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyLowercaseMappingsBuilder.kt @@ -0,0 +1,14 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToMany.builders + +import generators.unicode.SpecialCasingLine +import generators.unicode.UnicodeDataLine + +internal class OneToManyLowercaseMappingsBuilder(bmpUnicodeDataLines: List) : OneToManyMappingsBuilder(bmpUnicodeDataLines) { + override fun SpecialCasingLine.mapping(): List = lowercaseMapping + override fun UnicodeDataLine.mapping(): String = lowercaseMapping +} \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyMappingsBuilder.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyMappingsBuilder.kt new file mode 100644 index 00000000000..9f22e55a6e2 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyMappingsBuilder.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToMany.builders + +import generators.unicode.SpecialCasingLine +import generators.unicode.UnicodeDataLine +import generators.unicode.hexToInt +import java.util.TreeMap + +internal abstract class OneToManyMappingsBuilder(bmpUnicodeDataLines: List) { + private val bmpUnicodeDataLines = bmpUnicodeDataLines.associateBy { it.char.hexToInt() } + private val mappings = TreeMap>() + + fun append(line: SpecialCasingLine) { + if (line.conditionList.isNotEmpty()) return + + val charCode = line.char.hexToInt() + + check(charCode <= Char.MAX_VALUE.toInt()) { "Handle special casing for the supplementary code point: $line" } + + val mapping = mapping(charCode, line) ?: return + + mappings[charCode] = mapping + } + + fun build(): Map> { +// println(mappings) +// println("${this.javaClass} # ${mappings.size}") + return mappings + } + + private fun mapping(charCode: Int, line: SpecialCasingLine): List? { + val mapping = line.mapping() + + check(mapping.isNotEmpty() && mapping.all { it.isNotEmpty() }) + + if (mapping.size == 1) { + val specialCasingMapping = mapping.first() + + val unicodeLine = bmpUnicodeDataLines[charCode] + val unicodeDataMapping = unicodeLine?.mapping()?.takeIf { it.isNotEmpty() } ?: line.char + + check(unicodeDataMapping == specialCasingMapping) { + "UnicodeData.txt and SpecialCasing.txt files have different single char case conversion" + } + + return null + } + + return mapping + } + + abstract fun SpecialCasingLine.mapping(): List + abstract fun UnicodeDataLine.mapping(): String +} \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyTitlecaseMappingsBuilder.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyTitlecaseMappingsBuilder.kt new file mode 100644 index 00000000000..8652e382d5e --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyTitlecaseMappingsBuilder.kt @@ -0,0 +1,14 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToMany.builders + +import generators.unicode.SpecialCasingLine +import generators.unicode.UnicodeDataLine + +internal class OneToManyTitlecaseMappingsBuilder(bmpUnicodeDataLines: List) : OneToManyMappingsBuilder(bmpUnicodeDataLines) { + override fun SpecialCasingLine.mapping(): List = titlecaseMapping + override fun UnicodeDataLine.mapping(): String = titlecaseMapping +} diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyUppercaseMappingsBuilder.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyUppercaseMappingsBuilder.kt new file mode 100644 index 00000000000..e514b0d6dff --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/builders/OneToManyUppercaseMappingsBuilder.kt @@ -0,0 +1,14 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToMany.builders + +import generators.unicode.SpecialCasingLine +import generators.unicode.UnicodeDataLine + +internal class OneToManyUppercaseMappingsBuilder(bmpUnicodeDataLines: List) : OneToManyMappingsBuilder(bmpUnicodeDataLines) { + override fun SpecialCasingLine.mapping(): List = uppercaseMapping + override fun UnicodeDataLine.mapping(): String = uppercaseMapping +} diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyLowercaseMappingsWriter.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyLowercaseMappingsWriter.kt new file mode 100644 index 00000000000..754e383b31e --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyLowercaseMappingsWriter.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToMany.writers + +import generators.unicode.ranges.RangesWritingStrategy +import generators.unicode.toHexCharLiteral +import generators.unicode.hexCharsToStringLiteral +import java.io.FileWriter + +internal class OneToManyLowercaseMappingsWriter(private val strategy: RangesWritingStrategy) : OneToManyMappingsWriter { + override fun write(mappings: Map>, writer: FileWriter) { + check(mappings.size == 1) { "Number of multi-char lowercase mappings has changed." } + + val (key, value) = mappings.entries.single() + val char = key.toHexCharLiteral() + val result = value.hexCharsToStringLiteral() + + writer.appendLine(lowercaseImpl(char, result)) + } + + private fun lowercaseImpl(char: String, oneToManyResult: String): String = """ + internal fun Char.lowercaseImpl(): String { + if (this == $char) { + return $oneToManyResult + } + return lowercaseCharImpl().toString() + } + """.trimIndent() +} \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyMappingsWriter.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyMappingsWriter.kt new file mode 100644 index 00000000000..bbfc477d362 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyMappingsWriter.kt @@ -0,0 +1,12 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToMany.writers + +import java.io.FileWriter + +internal interface OneToManyMappingsWriter { + fun write(mappings: Map>, writer: FileWriter) +} diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyTitlecaseMappingsWriter.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyTitlecaseMappingsWriter.kt new file mode 100644 index 00000000000..72a02c1eb5c --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyTitlecaseMappingsWriter.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToMany.writers + +import generators.unicode.ranges.RangesWritingStrategy +import generators.unicode.writeMappings +import java.io.FileWriter + +internal class OneToManyTitlecaseMappingsWriter(private val strategy: RangesWritingStrategy) : OneToManyMappingsWriter { + override fun write(mappings: Map>, writer: FileWriter) { + strategy.beforeWritingRanges(writer) + writer.writeMappings(mappings, strategy) + strategy.afterWritingRanges(writer) + writer.appendLine() + writer.appendLine(titlecaseImpl()) + } + + private fun titlecaseImpl(): String = """ + internal fun Char.titlecaseImpl(): String { + val code = this.toInt() + val index = binarySearchRange(keys, code) + if (index >= 0 && keys[index] == code) { + return values[index] + } + return titlecaseCharImpl().toString() + } + """.trimIndent() +} diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyUppercaseMappingsWriter.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyUppercaseMappingsWriter.kt new file mode 100644 index 00000000000..e5287959b00 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToMany/writers/OneToManyUppercaseMappingsWriter.kt @@ -0,0 +1,44 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToMany.writers + +import generators.unicode.ranges.RangesWritingStrategy +import generators.unicode.toHexCharLiteral +import generators.unicode.writeMappings +import java.io.FileWriter + +internal class OneToManyUppercaseMappingsWriter(private val strategy: RangesWritingStrategy) : OneToManyMappingsWriter { + override fun write(mappings: Map>, writer: FileWriter) { + strategy.beforeWritingRanges(writer) + writer.writeMappings(mappings, strategy) + strategy.afterWritingRanges(writer) + writer.appendLine() + writer.appendLine(oneToManyUppercase(mappings.keys.first())) + writer.appendLine() + writer.appendLine(uppercaseImpl()) + } + + private fun oneToManyUppercase(smallestOneToMany: Int): String = """ + internal fun Char.oneToManyUppercase(): String? { + if (this < ${smallestOneToMany.toHexCharLiteral()}) { + return null + } + + val code = this.toInt() + val index = binarySearchRange(keys, code) + if (keys[index] == code) { + return values[index] + } + return null + } + """.trimIndent() + + private fun uppercaseImpl(): String = """ + internal fun Char.uppercaseImpl(): String { + return oneToManyUppercase() ?: uppercaseCharImpl().toString() + } + """.trimIndent() +} diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/MappingsGenerator.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/MappingsGenerator.kt new file mode 100644 index 00000000000..cdcf62fbc84 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/MappingsGenerator.kt @@ -0,0 +1,63 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToOne + +import generators.requireExistingDir +import generators.unicode.UnicodeDataLine +import generators.unicode.mappings.oneToOne.builders.* +import generators.unicode.mappings.oneToOne.writers.* +import generators.unicode.ranges.RangesWritingStrategy +import generators.unicode.writeHeader +import templates.KotlinTarget +import java.io.File +import java.io.FileWriter + +internal class MappingsGenerator private constructor( + private val outputFile: File, + private val mappingsBuilder: MappingsBuilder, + private val mappingsWriter: MappingsWriter, +) { + + init { + outputFile.parentFile.requireExistingDir() + } + + fun appendLine(line: UnicodeDataLine) { + mappingsBuilder.append(line) + } + + fun generate() { + val mappings = mappingsBuilder.build() + + FileWriter(outputFile).use { writer -> + writer.writeHeader(outputFile, "kotlin.text") + writer.appendLine() + writer.appendLine("// ${mappings.size} ranges totally") + + mappingsWriter.write(mappings, writer) + } + } + + companion object { + fun forUppercase(outputFile: File, target: KotlinTarget): MappingsGenerator { + val builder = UppercaseMappingsBuilder() + val writer = UppercaseMappingsWriter(RangesWritingStrategy.of(target, "Uppercase")) + return MappingsGenerator(outputFile, builder, writer) + } + + fun forLowercase(outputFile: File, target: KotlinTarget): MappingsGenerator { + val builder = LowercaseMappingsBuilder() + val writer = LowercaseMappingsWriter(RangesWritingStrategy.of(target, "Lowercase")) + return MappingsGenerator(outputFile, builder, writer) + } + + fun forTitlecase(outputFile: File): MappingsGenerator { + val builder = TitlecaseMappingsBuilder() + val writer = TitlecaseMappingsWriter() + return MappingsGenerator(outputFile, builder, writer) + } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/LowercaseMappingsBuilder.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/LowercaseMappingsBuilder.kt new file mode 100644 index 00000000000..b47de4bdb34 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/LowercaseMappingsBuilder.kt @@ -0,0 +1,16 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToOne.builders + +import generators.unicode.UnicodeDataLine + +internal class LowercaseMappingsBuilder : MappingsBuilder() { + override fun mappingEquivalent(line: UnicodeDataLine): String? { + if (line.lowercaseMapping.isEmpty()) return null + check(line.char != line.lowercaseMapping) { "UnicodeData.txt format has changed!" } + return line.lowercaseMapping + } +} diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/MappingsBuilder.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/MappingsBuilder.kt new file mode 100644 index 00000000000..4da44f18b2e --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/MappingsBuilder.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToOne.builders + +import generators.unicode.UnicodeDataLine +import generators.unicode.mappings.oneToOne.patterns.EqualDistanceMappingPattern +import generators.unicode.mappings.oneToOne.patterns.MappingPattern +import generators.unicode.hexToInt + +/** + * The base class of character mappings builders. + */ +internal abstract class MappingsBuilder { + private val patterns = mutableListOf() + + /** + * Appends a line from the UnicodeData.txt file. + */ + fun append(line: UnicodeDataLine) { + val charCode = line.char.hexToInt() + val equivalent = mappingEquivalent(line)?.hexToInt() ?: return + val mapping = equivalent - charCode + + check((charCode > Char.MAX_VALUE.toInt()) == (equivalent > Char.MAX_VALUE.toInt())) { "Handle when equivalent mapping is out of BMP." } + + if (patterns.isEmpty()) { + patterns.add(createPattern(charCode, line.categoryCode, mapping)) + return + } + + val lastPattern = patterns.last() + + if (!lastPattern.append(charCode, line.categoryCode, mapping)) { + val newLastPattern = evolveLastPattern(lastPattern, charCode, line.categoryCode, mapping) + if (newLastPattern != null) { + patterns[patterns.lastIndex] = newLastPattern + } else { + patterns.add(createPattern(charCode, line.categoryCode, mapping)) + } + } + } + + /** + * Returns the resulting mapping patterns. + */ + fun build(): List { +// println(patterns.joinToString(separator = "\n")) +// println("${this.javaClass} # ${patterns.size}") + return patterns + } + + /** + * Returns the mapping equivalent this builder is responsible for. + */ + abstract fun mappingEquivalent(line: UnicodeDataLine): String? + + /** + * Appends the [charCode] with the specified [categoryCode] and [mapping] to the [lastPattern] and returns the resulting pattern, + * or returns `null` if the [charCode] can't be appended to the [lastPattern]. + * The [lastPattern] can be transformed to another pattern type to accommodate the [charCode]. + */ + protected open fun evolveLastPattern(lastPattern: MappingPattern, charCode: Int, categoryCode: String, mapping: Int): MappingPattern? { + return null + } + + private fun createPattern(charCode: Int, categoryCode: String, mapping: Int): MappingPattern { + return EqualDistanceMappingPattern.from(charCode, categoryCode, mapping) + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/TitlecaseMappingsBuilder.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/TitlecaseMappingsBuilder.kt new file mode 100644 index 00000000000..68a07497727 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/TitlecaseMappingsBuilder.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToOne.builders + +import generators.unicode.UnicodeDataLine +import generators.unicode.mappings.oneToOne.patterns.LuLtLlMappingPattern +import generators.unicode.mappings.oneToOne.patterns.MappingPattern + +internal class TitlecaseMappingsBuilder : MappingsBuilder() { + + override fun mappingEquivalent(line: UnicodeDataLine): String? { + if (line.titlecaseMapping == line.uppercaseMapping) return null + check(line.titlecaseMapping.isNotEmpty()) { "UnicodeData.txt format has changed!" } + return line.titlecaseMapping + } + + override fun evolveLastPattern(lastPattern: MappingPattern, charCode: Int, categoryCode: String, mapping: Int): MappingPattern? { + return LuLtLlMappingPattern.from(lastPattern, charCode, categoryCode, mapping) + } +} diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/UppercaseMappingsBuilder.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/UppercaseMappingsBuilder.kt new file mode 100644 index 00000000000..11bea788227 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/builders/UppercaseMappingsBuilder.kt @@ -0,0 +1,16 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToOne.builders + +import generators.unicode.UnicodeDataLine + +internal class UppercaseMappingsBuilder : MappingsBuilder() { + override fun mappingEquivalent(line: UnicodeDataLine): String? { + if (line.uppercaseMapping.isEmpty()) return null + check(line.char != line.uppercaseMapping) { "UnicodeData.txt format has changed!" } + return line.uppercaseMapping + } +} diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/patterns/EqualDistanceMappingPattern.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/patterns/EqualDistanceMappingPattern.kt new file mode 100644 index 00000000000..ca464ad9c33 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/patterns/EqualDistanceMappingPattern.kt @@ -0,0 +1,64 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToOne.patterns + +import generators.unicode.toHexIntLiteral + +/** + * A range of consequent chars where all chars with code `start + k * distance` have the same [mapping] for case conversion. + * + * @param charCode the start of this range + * @param categoryCode the category code of the char with the specified [charCode] + * @param mapping the difference between [charCode] and the char it converts to. + */ +internal class EqualDistanceMappingPattern private constructor( + charCode: Int, + val categoryCode: String, + val mapping: Int +) : MappingPattern { + override val start = charCode + override var end = charCode + private set + var distance = 1 + private set + + override fun append(charCode: Int, categoryCode: String, mapping: Int): Boolean { + require(charCode > end) + + val distance = charCode - end + if (mapping == this.mapping + && length < MAX_LENGTH + && distance <= MAX_DISTANCE + && (distance == this.distance || length == 1) + ) { + // if length is equal to 1, distance can be changed + this.distance = distance + end = charCode + return true + } + return false + } + + override fun toString(): String { + return "EqualDistanceMappingPattern{" + + "start=" + start.toHexIntLiteral() + + ", end=" + end.toHexIntLiteral() + + ", length=" + length + + ", distance=" + distance + + ", count=" + ((length + distance - 1) / distance) + + ", mapping=" + mapping + + "}" + } + + companion object { + private const val MAX_DISTANCE = 15 + private const val MAX_LENGTH = 255 + + fun from(charCode: Int, categoryCode: String, mapping: Int): EqualDistanceMappingPattern { + return EqualDistanceMappingPattern(charCode, categoryCode, mapping) + } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/patterns/LuLtLlMappingPattern.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/patterns/LuLtLlMappingPattern.kt new file mode 100644 index 00000000000..417126d31ae --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/patterns/LuLtLlMappingPattern.kt @@ -0,0 +1,84 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToOne.patterns + +import generators.unicode.toHexIntLiteral + +/** + * A range of consequent chars that repeat sequence of `categoryCode` and <1, 0, -1> sequence of `mapping`. + * + * @param charCode the start of this range + * @param categoryCode the category code of the char with the specified [charCode] + * @param mapping the difference between the [charCode] and the char it converts to. + */ +internal class LuLtLlMappingPattern private constructor( + charCode: Int, + categoryCode: String, + mapping: Int +) : MappingPattern { + override val start = charCode + override var end = charCode + private set + val distance = 1 + + init { + require(categoryCode == "Lu" && mapping == 1) + } + + override fun append(charCode: Int, categoryCode: String, mapping: Int): Boolean { + require(charCode > end) + + val distance = charCode - end + val modThree = (charCode - start) % 3 + val expectedMapping = expectedMapping(categoryCode) + val expectedModThree = expectedModThree(categoryCode) + if (distance == this.distance && mapping == expectedMapping && modThree == expectedModThree) { + end = charCode + return true + } + return false + } + + private fun expectedModThree(categoryCode: String): Int = when (categoryCode) { + "Lu" -> 0 + "Lt" -> 1 + "Ll" -> 2 + else -> error("Unexpected category: $categoryCode") + } + + private fun expectedMapping(categoryCode: String): Int = when (categoryCode) { + "Lu" -> 1 + "Lt" -> 0 + "Ll" -> -1 + else -> error("Unexpected category: $categoryCode") + } + + override fun toString(): String { + return "LuLtLlMappingPattern{" + + "start=" + start.toHexIntLiteral() + + ", end=" + end.toHexIntLiteral() + + ", length=" + length + + ", distance=" + distance + + ", count=" + ((length + distance - 1) / distance) + + "}" + } + + companion object { + fun from(pattern: MappingPattern, charCode: Int, categoryCode: String, mapping: Int): LuLtLlMappingPattern? { + if (pattern is EqualDistanceMappingPattern + && pattern.length == 1 + && pattern.categoryCode == "Lu" + && pattern.mapping == 1 + ) { + val newPattern = LuLtLlMappingPattern(pattern.start, pattern.categoryCode, pattern.mapping) + if (newPattern.append(charCode, categoryCode, mapping)) { + return newPattern + } + } + return null + } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/patterns/MappingPattern.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/patterns/MappingPattern.kt new file mode 100644 index 00000000000..c79f0d59e36 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/patterns/MappingPattern.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToOne.patterns + +/** + * A range of consequent chars that fit within a particular mapping pattern. + */ +internal interface MappingPattern { + /** + * Char code of the first char in this range. + */ + val start: Int + + /** + * Char code of the last char in this range. + */ + val end: Int + + /** + * Appends the [charCode] to this range pattern. + * Returns true if the [charCode] with the specified [categoryCode] and [mapping] was accommodated within this pattern. + * Returns false otherwise. + * + * @param mapping the difference between the [charCode] and the char it converts to. + */ + fun append(charCode: Int, categoryCode: String, mapping: Int): Boolean +} + +internal val MappingPattern.length: Int get() = end - start + 1 \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/LowercaseMappingsWriter.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/LowercaseMappingsWriter.kt new file mode 100644 index 00000000000..82ec9227ff1 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/LowercaseMappingsWriter.kt @@ -0,0 +1,52 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToOne.writers + +import generators.unicode.mappings.oneToOne.patterns.EqualDistanceMappingPattern +import generators.unicode.mappings.oneToOne.patterns.length +import generators.unicode.mappings.oneToOne.patterns.MappingPattern +import generators.unicode.ranges.RangesWritingStrategy +import generators.unicode.writeIntArray +import java.io.FileWriter + +internal class LowercaseMappingsWriter(private val strategy: RangesWritingStrategy) : MappingsWriter { + override fun write(mappings: List, writer: FileWriter) { + @Suppress("UNCHECKED_CAST") + val distanceMappings = mappings as List + + val start = distanceMappings.map { it.start } + val length = distanceMappings.map { (it.mapping shl 12) or (it.distance shl 8) or it.length } + + strategy.beforeWritingRanges(writer) + writer.writeIntArray("rangeStart", start, strategy) + writer.appendLine() + writer.writeIntArray("rangeLength", length, strategy) + strategy.afterWritingRanges(writer) + writer.appendLine() + writer.appendLine(lowercaseCodePoint()) + writer.appendLine() + writer.appendLine(lowercaseCharImpl()) + } + + private fun lowercaseCodePoint(): String = """ + internal fun Int.lowercaseCodePoint(): Int { + if (this in 0x41..0x5a) { + return this + 32 + } + if (this < 0x80) { + return this + } + val index = binarySearchRange(rangeStart, this) + return equalDistanceMapping(this, rangeStart[index], rangeLength[index]) + } + """.trimIndent() + + private fun lowercaseCharImpl(): String = """ + internal fun Char.lowercaseCharImpl(): Char { + return toInt().lowercaseCodePoint().toChar() + } + """.trimIndent() +} \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/MappingsWriter.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/MappingsWriter.kt new file mode 100644 index 00000000000..b7038bb4771 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/MappingsWriter.kt @@ -0,0 +1,13 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToOne.writers + +import generators.unicode.mappings.oneToOne.patterns.MappingPattern +import java.io.FileWriter + +internal interface MappingsWriter { + fun write(mappings: List, writer: FileWriter) +} \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/TitlecaseMappingsWriter.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/TitlecaseMappingsWriter.kt new file mode 100644 index 00000000000..87cecc2f5f4 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/TitlecaseMappingsWriter.kt @@ -0,0 +1,42 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToOne.writers + +import generators.unicode.mappings.oneToOne.patterns.EqualDistanceMappingPattern +import generators.unicode.mappings.oneToOne.patterns.LuLtLlMappingPattern +import generators.unicode.mappings.oneToOne.patterns.MappingPattern +import generators.unicode.toHexIntLiteral +import java.io.FileWriter + +internal class TitlecaseMappingsWriter : MappingsWriter { + + override fun write(mappings: List, writer: FileWriter) { + val LuLtLlMappings = mappings.filterIsInstance() + val zeroMappings = mappings.filterIsInstance().filter { it.distance == 1 && it.mapping == 0 } + + check(LuLtLlMappings.size + zeroMappings.size == mappings.size) { "Handle new types of titlecase mapping." } + check(LuLtLlMappings.all { it.start % 3 == 2 }) { "Handle when code of the Lt char is not multiple of 3." } + + writer.append( + """ + internal fun Char.titlecaseCharImpl(): Char { + val code = this.toInt() + if (${rangeChecks(LuLtLlMappings, "code")}) { + return (3 * ((code + 1) / 3)).toChar() + } + if (${rangeChecks(zeroMappings, "code")}) { + return this + } + return uppercaseCharImpl() + } + """.trimIndent() + ) + } + + private fun rangeChecks(mappings: List, code: String): String { + return mappings.joinToString(separator = " || ") { "$code in ${it.start.toHexIntLiteral()}..${it.end.toHexIntLiteral()}" } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/UppercaseMappingsWriter.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/UppercaseMappingsWriter.kt new file mode 100644 index 00000000000..ecf63848c81 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/mappings/oneToOne/writers/UppercaseMappingsWriter.kt @@ -0,0 +1,73 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package generators.unicode.mappings.oneToOne.writers + +import generators.unicode.mappings.oneToOne.patterns.EqualDistanceMappingPattern +import generators.unicode.mappings.oneToOne.patterns.length +import generators.unicode.mappings.oneToOne.patterns.MappingPattern +import generators.unicode.ranges.RangesWritingStrategy +import generators.unicode.writeIntArray +import java.io.FileWriter + +internal class UppercaseMappingsWriter(private val strategy: RangesWritingStrategy) : MappingsWriter { + override fun write(mappings: List, writer: FileWriter) { + @Suppress("UNCHECKED_CAST") + val distanceMappings = mappings as List + + val start = distanceMappings.map { it.start } + val length = distanceMappings.map { (it.mapping shl 12) or (it.distance shl 8) or it.length } + + strategy.beforeWritingRanges(writer) + writer.writeIntArray("rangeStart", start, strategy) + writer.appendLine() + writer.writeIntArray("rangeLength", length, strategy) + strategy.afterWritingRanges(writer) + writer.appendLine() + writer.appendLine(equalDistanceMapping()) + writer.appendLine() + writer.appendLine(uppercaseCodePoint()) + writer.appendLine() + writer.appendLine(uppercaseCharImpl()) + } + + private fun equalDistanceMapping(): String = """ + internal fun equalDistanceMapping(code: Int, start: Int, pattern: Int): Int { + val diff = code - start + + val length = pattern and 0xff + if (diff >= length) { + return code + } + + val distance = (pattern shr 8) and 0xf + if (diff % distance != 0) { + return code + } + + val mapping = pattern shr 12 + return code + mapping + } + """.trimIndent() + + private fun uppercaseCodePoint(): String = """ + internal fun Int.uppercaseCodePoint(): Int { + if (this in 0x61..0x7a) { + return this - 32 + } + if (this < 0x80) { + return this + } + val index = binarySearchRange(rangeStart, this) + return equalDistanceMapping(this, rangeStart[index], rangeLength[index]) + } + """.trimIndent() + + private fun uppercaseCharImpl(): String = """ + internal fun Char.uppercaseCharImpl(): Char { + return toInt().uppercaseCodePoint().toChar() + } + """.trimIndent() +} \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/CharCategoryTestGenerator.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/CharCategoryTestGenerator.kt index 67a3ef246d7..584369816cc 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/CharCategoryTestGenerator.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/CharCategoryTestGenerator.kt @@ -5,12 +5,12 @@ package generators.unicode.ranges -import generators.unicode.UnicodeDataGenerator -import generators.unicode.ranges.writers.writeHeader +import generators.unicode.UnicodeDataLine +import generators.unicode.writeHeader import java.io.File import java.io.FileWriter -internal class CharCategoryTestGenerator(private val outputFile: File) : UnicodeDataGenerator { +internal class CharCategoryTestGenerator(private val outputFile: File) { private var arrayIndex = 0 private var arraySize = 0 private var writer: FileWriter? = null @@ -19,7 +19,7 @@ internal class CharCategoryTestGenerator(private val outputFile: File) : Unicode outputFile.parentFile.mkdirs() } - override fun appendChar(char: String, name: String, categoryCode: String) { + fun appendLine(line: UnicodeDataLine) { if (arraySize == 0) { writer?.appendLine(")") writer?.close() @@ -27,9 +27,9 @@ internal class CharCategoryTestGenerator(private val outputFile: File) : Unicode generateUnicodeDataHeader(arrayIndex) } - val isStart = name.endsWith(", First>") + val isStart = line.name.endsWith(", First>") - writer?.appendLine(" CharProperties(char = '\\u$char', isStartOfARange = $isStart, categoryCode = \"$categoryCode\"),") + writer?.appendLine(" CharProperties(char = '\\u${line.char}', isStartOfARange = $isStart, categoryCode = \"${line.categoryCode}\"),") arraySize++ if (arraySize == 2048) { @@ -38,7 +38,7 @@ internal class CharCategoryTestGenerator(private val outputFile: File) : Unicode } } - override fun close() { + fun generate() { writer?.appendLine(")") writer?.close() diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/RangesGenerator.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/RangesGenerator.kt index b3dcb5a62d0..3443f451ce5 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/RangesGenerator.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/RangesGenerator.kt @@ -6,9 +6,10 @@ package generators.unicode.ranges import generators.requireExistingDir -import generators.unicode.UnicodeDataGenerator +import generators.unicode.UnicodeDataLine import generators.unicode.ranges.builders.* import generators.unicode.ranges.writers.* +import generators.unicode.writeHeader import templates.KotlinTarget import templates.Platform import java.io.File @@ -18,17 +19,17 @@ internal class RangesGenerator private constructor( private val outputFile: File, private val rangesBuilder: RangesBuilder, private val rangesWriter: RangesWriter, -) : UnicodeDataGenerator { +) { init { outputFile.parentFile.requireExistingDir() } - override fun appendChar(char: String, name: String, categoryCode: String) { - rangesBuilder.append(char, name, categoryCode) + fun appendLine(line: UnicodeDataLine) { + rangesBuilder.append(line.char, line.name, line.categoryCode) } - override fun close() { + fun generate() { val (rangeStart, rangeEnd, rangeCategory) = rangesBuilder.build() FileWriter(outputFile).use { writer -> diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/builders/DigitRangesBuilder.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/builders/DigitRangesBuilder.kt index ae7f7d30e9d..229b2dbb1ca 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/builders/DigitRangesBuilder.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/builders/DigitRangesBuilder.kt @@ -5,8 +5,6 @@ package generators.unicode.ranges.builders -import generators.unicode.ranges.patterns.RangePattern - internal class DigitRangesBuilder : RangesBuilder() { override fun categoryId(categoryCode: String): String { return categoryCode @@ -15,11 +13,4 @@ internal class DigitRangesBuilder : RangesBuilder() { override fun shouldSkip(categoryId: String): Boolean { return categoryId != CharCategory.DECIMAL_DIGIT_NUMBER.code } - - override val makeOnePeriodCategory: (Array) -> Int - get() = { 0 } - - override fun evolveLastRange(lastRange: RangePattern, charCode: Int, categoryId: String): RangePattern? { - return null - } } \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/builders/RangesBuilder.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/builders/RangesBuilder.kt index 330b70b77a2..47f0837aaf5 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/builders/RangesBuilder.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/builders/RangesBuilder.kt @@ -8,6 +8,7 @@ package generators.unicode.ranges.builders import generators.unicode.ranges.patterns.PeriodicRangePattern import generators.unicode.ranges.patterns.RangePattern import generators.unicode.ranges.patterns.rangeLength +import generators.unicode.hexToInt /** * The base class of character ranges builders. @@ -20,7 +21,7 @@ internal abstract class RangesBuilder { * Appends a line from the UnicodeData.txt file. */ fun append(char: String, name: String, categoryCode: String) { - val charCode = char.toInt(radix = 16) + val charCode = char.hexToInt() val categoryId = categoryId(categoryCode) when { @@ -177,16 +178,12 @@ internal abstract class RangesBuilder { /** * The function to use to transform periodic ranges with period equal to 1 to an Int representation. */ - protected abstract val makeOnePeriodCategory: (Array) -> Int + protected open val makeOnePeriodCategory: (Array) -> Int = { 0 } /** * Appends the [charCode] with the specified [categoryId] to the [lastRange] and returns the resulting range, * or returns `null` if [charCode] can't be appended to the [lastRange]. * The [lastRange] can be transformed to another range type to accommodate the [charCode]. */ - protected abstract fun evolveLastRange( - lastRange: RangePattern, - charCode: Int, - categoryId: String - ): RangePattern? + protected open fun evolveLastRange(lastRange: RangePattern, charCode: Int, categoryId: String): RangePattern? = null } \ No newline at end of file diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/builders/WhitespaceRangesBuilder.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/builders/WhitespaceRangesBuilder.kt index f296186489a..2f2dfb57410 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/builders/WhitespaceRangesBuilder.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/builders/WhitespaceRangesBuilder.kt @@ -5,8 +5,6 @@ package generators.unicode.ranges.builders -import generators.unicode.ranges.patterns.RangePattern - internal class WhitespaceRangesBuilder : RangesBuilder() { init { @@ -24,13 +22,6 @@ internal class WhitespaceRangesBuilder : RangesBuilder() { override fun shouldSkip(categoryId: String): Boolean { return categoryId == NOT_WS } - - override val makeOnePeriodCategory: (Array) -> Int - get() = { 0 } - - override fun evolveLastRange(lastRange: RangePattern, charCode: Int, categoryId: String): RangePattern? { - return null - } } private const val WS = "WS" diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/patterns/GapRangePattern.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/patterns/GapRangePattern.kt index 5bb592ef2fc..ff2014da76b 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/patterns/GapRangePattern.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/patterns/GapRangePattern.kt @@ -5,7 +5,7 @@ package generators.unicode.ranges.patterns -import generators.unicode.ranges.writers.hex +import generators.unicode.toHexIntLiteral /** * A range of consequent chars that starts with a letter and ends with a letter, and contains multiple ranges of consequent not-letter chars. @@ -95,8 +95,8 @@ internal class GapRangePattern private constructor( override fun toString(): String { return "GapPattern{" + - "start=" + start.hex() + - ", end=" + end.hex() + + "start=" + start.toHexIntLiteral() + + ", end=" + end.toHexIntLiteral() + ", length=" + rangeLength() + ", gaps=" + gaps + ", categoryId=" + categoryId + diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/patterns/PeriodicRangePattern.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/patterns/PeriodicRangePattern.kt index 0e9eca94e22..04deed24652 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/patterns/PeriodicRangePattern.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/patterns/PeriodicRangePattern.kt @@ -5,7 +5,7 @@ package generators.unicode.ranges.patterns -import generators.unicode.ranges.writers.hex +import generators.unicode.toHexIntLiteral /** * A range of consequent chars. @@ -76,7 +76,7 @@ internal class PeriodicRangePattern private constructor( override fun categoryIdOf(charCode: Int): String { if (charCode !in start..end) { - throw IllegalArgumentException("Char code ${charCode.hex()} is not in $this") + throw IllegalArgumentException("Char code ${charCode.toHexIntLiteral()} is not in $this") } val categoryId = bag.categoryIdOf(charCode) check(categoryId != null) @@ -85,8 +85,8 @@ internal class PeriodicRangePattern private constructor( override fun toString(): String { return "PeriodicRangePattern{" + - "start=" + start.hex() + - ", end=" + end.hex() + + "start=" + start.toHexIntLiteral() + + ", end=" + end.toHexIntLiteral() + ", length=" + rangeLength() + ", orderedCategoryIds=" + orderedCategoryIds().contentToString() + ", bag=" + bag + diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/CategoryRangesWriter.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/CategoryRangesWriter.kt index fcae45137cf..495318777f9 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/CategoryRangesWriter.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/CategoryRangesWriter.kt @@ -5,8 +5,11 @@ package generators.unicode.ranges.writers +import generators.unicode.TO_BASE64 import generators.unicode.ranges.RangesWritingStrategy import generators.unicode.ranges.builders.UNASSIGNED_CATEGORY_VALUE_REPLACEMENT +import generators.unicode.toVarLenBase64 +import generators.unicode.writeIntArray import java.io.FileWriter internal open class CategoryRangesWriter(protected val strategy: RangesWritingStrategy) : RangesWriter { diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/DigitRangesWriter.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/DigitRangesWriter.kt index 38b5125eae5..f35a95959ca 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/DigitRangesWriter.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/DigitRangesWriter.kt @@ -6,6 +6,7 @@ package generators.unicode.ranges.writers import generators.unicode.ranges.RangesWritingStrategy +import generators.unicode.writeIntArray import java.io.FileWriter internal class DigitRangesWriter(private val strategy: RangesWritingStrategy) : RangesWriter { diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/LetterRangesWriter.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/LetterRangesWriter.kt index 9f38bbd4b01..669757f5dcc 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/LetterRangesWriter.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/LetterRangesWriter.kt @@ -5,8 +5,11 @@ package generators.unicode.ranges.writers +import generators.unicode.TO_BASE64 import generators.unicode.ranges.RangesWritingStrategy import generators.unicode.ranges.patterns.GapRangePattern +import generators.unicode.toVarLenBase64 +import generators.unicode.writeIntArray import java.io.FileWriter internal open class LetterRangesWriter(protected val strategy: RangesWritingStrategy) : RangesWriter { diff --git a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/WhitespaceRangesWriter.kt b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/WhitespaceRangesWriter.kt index ff20d17e511..db8377662bc 100644 --- a/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/WhitespaceRangesWriter.kt +++ b/libraries/tools/kotlin-stdlib-gen/src/generators/unicode/ranges/writers/WhitespaceRangesWriter.kt @@ -5,6 +5,7 @@ package generators.unicode.ranges.writers +import generators.unicode.toHexIntLiteral import java.io.FileWriter internal class WhitespaceRangesWriter : RangesWriter { @@ -44,15 +45,15 @@ internal class WhitespaceRangesWriter : RangesWriter { tabCount = 6 builder.append(tab.repeat(tabCount)) } - builder.appendLine("$ch == ${start.hex()}") + builder.appendLine("$ch == ${start.toHexIntLiteral()}") } end - 1 -> { - builder.appendLine("$ch == ${start.hex()}") + builder.appendLine("$ch == ${start.toHexIntLiteral()}") builder.append(tab.repeat(tabCount)).append("|| ") - builder.appendLine("$ch == ${end.hex()}") + builder.appendLine("$ch == ${end.toHexIntLiteral()}") } else -> { - builder.appendLine("$ch in ${start.hex()}..${end.hex()}") + builder.appendLine("$ch in ${start.toHexIntLiteral()}..${end.toHexIntLiteral()}") } } }