Implement String case conversion generators #KT-45213
This commit is contained in:
@@ -9,6 +9,9 @@ import generators.unicode.mappings.oneToOne.MappingsGenerator
|
||||
import generators.unicode.mappings.oneToMany.OneToManyMappingsGenerator
|
||||
import generators.unicode.ranges.CharCategoryTestGenerator
|
||||
import generators.unicode.ranges.RangesGenerator
|
||||
import generators.unicode.mappings.string.StringCasingTestGenerator
|
||||
import generators.unicode.mappings.string.StringLowercaseGenerator
|
||||
import generators.unicode.mappings.string.StringUppercaseGenerator
|
||||
import templates.COPYRIGHT_NOTICE
|
||||
import templates.KotlinTarget
|
||||
import templates.readCopyrightNoticeFromProfile
|
||||
@@ -21,6 +24,9 @@ import kotlin.system.exitProcess
|
||||
private const val unicodeVersion = "13.0.0"
|
||||
private const val unicodeDataUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/UnicodeData.txt"
|
||||
private const val specialCasingUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/SpecialCasing.txt"
|
||||
private const val propListUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/PropList.txt"
|
||||
private const val wordBreakPropertyUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/auxiliary/WordBreakProperty.txt"
|
||||
private const val derivedCorePropertiesUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/DerivedCoreProperties.txt"
|
||||
|
||||
/**
|
||||
* This program generates sources related to UnicodeData.txt and SpecialCasing.txt.
|
||||
@@ -32,14 +38,29 @@ private const val specialCasingUrl = "https://www.unicode.org/Public/$unicodeVer
|
||||
* No tests are generated.
|
||||
*/
|
||||
fun main(args: Array<String>) {
|
||||
|
||||
val unicodeDataLines = URL(unicodeDataUrl).openStream().reader().readLines()
|
||||
val bmpUnicodeDataLines = unicodeDataLines.mapNotNull { line ->
|
||||
UnicodeDataLine(line.split(";")).takeIf { it.char.length <= 4 } // Basic Multilingual Plane (BMP)
|
||||
fun readLines(url: String): List<String> {
|
||||
return URL(url).openStream().reader().readLines()
|
||||
}
|
||||
val specialCasingLines = URL(specialCasingUrl).openStream().reader().readLines()
|
||||
val nonCommentSpecialCasingLines = specialCasingLines.mapNotNull { line ->
|
||||
if (line.isEmpty() || line.startsWith("#")) null else SpecialCasingLine(line.split("; "))
|
||||
|
||||
val unicodeDataLines = readLines(unicodeDataUrl).map { line -> UnicodeDataLine(line.split(";")) }
|
||||
val bmpUnicodeDataLines = unicodeDataLines.filter { line -> line.char.length <= 4 } // Basic Multilingual Plane (BMP)
|
||||
|
||||
fun String.isEmptyOrComment(): Boolean = isEmpty() || startsWith("#")
|
||||
|
||||
val specialCasingLines = readLines(specialCasingUrl).filterNot(String::isEmptyOrComment).map { line ->
|
||||
SpecialCasingLine(line.split("; "))
|
||||
}
|
||||
|
||||
val propListLines = readLines(propListUrl).filterNot(String::isEmptyOrComment).map { line ->
|
||||
PropertyLine(line.split("; ").map { it.trim() })
|
||||
}
|
||||
|
||||
val wordBreakPropertyLines = readLines(wordBreakPropertyUrl).filterNot(String::isEmptyOrComment).map { line ->
|
||||
PropertyLine(line.split("; ").map { it.trim() })
|
||||
}
|
||||
|
||||
val derivedCorePropertiesLines = readLines(derivedCorePropertiesUrl).filterNot(String::isEmptyOrComment).map { line ->
|
||||
PropertyLine(line.split("; ").map { it.trim() })
|
||||
}
|
||||
|
||||
val categoryRangesGenerators = mutableListOf<RangesGenerator>()
|
||||
@@ -79,6 +100,11 @@ fun main(args: Array<String>) {
|
||||
|
||||
var categoryTestGenerator: CharCategoryTestGenerator? = null
|
||||
|
||||
var stringUppercaseGenerator: StringUppercaseGenerator? = null
|
||||
var stringLowercaseGenerator: StringLowercaseGenerator? = null
|
||||
|
||||
var stringCasingTestGenerator: StringCasingTestGenerator? = null
|
||||
|
||||
when (args.size) {
|
||||
1 -> {
|
||||
val baseDir = File(args.first())
|
||||
@@ -93,15 +119,16 @@ fun main(args: Array<String>) {
|
||||
addRangesGenerators(jsIrGeneratedDir, KotlinTarget.JS_IR)
|
||||
|
||||
// For debugging. To see the file content
|
||||
fun writeContent(lines: List<String>, fileName: String) {
|
||||
val file = baseDir.resolve("libraries/tools/kotlin-stdlib-gen/src/generators/unicode/$fileName")
|
||||
file.writeText(lines.joinToString(separator = "\n"))
|
||||
fun downloadFile(fromUrl: String) {
|
||||
val fileName = File(fromUrl).name
|
||||
val dest = baseDir.resolve("libraries/tools/kotlin-stdlib-gen/src/generators/unicode/$fileName")
|
||||
dest.writeText(readLines(fromUrl).joinToString(separator = "\n"))
|
||||
}
|
||||
writeContent(unicodeDataLines, "UnicodeData.txt")
|
||||
writeContent(specialCasingLines, "SpecialCasing.txt")
|
||||
downloadFile(unicodeDataUrl)
|
||||
downloadFile(specialCasingUrl)
|
||||
}
|
||||
2 -> {
|
||||
val (targetName, targetDir) = args
|
||||
3 -> {
|
||||
val (targetName, targetDir, testDir) = args
|
||||
|
||||
val target = KotlinTarget.values.singleOrNull { it.name.equals(targetName, ignoreCase = true) }
|
||||
?: error("Invalid target: $targetName")
|
||||
@@ -112,13 +139,17 @@ fun main(args: Array<String>) {
|
||||
if (target == KotlinTarget.Native) {
|
||||
addOneToOneMappingsGenerators(generatedDir, target)
|
||||
addOneToManyMappingsGenerators(generatedDir, target)
|
||||
stringUppercaseGenerator = StringUppercaseGenerator(generatedDir.resolve("_StringUppercase.kt"), unicodeDataLines)
|
||||
stringLowercaseGenerator = StringLowercaseGenerator(generatedDir.resolve("_StringLowercase.kt"), unicodeDataLines)
|
||||
|
||||
stringCasingTestGenerator = StringCasingTestGenerator(File(testDir))
|
||||
}
|
||||
}
|
||||
else -> {
|
||||
println(
|
||||
"""Parameters:
|
||||
<kotlin-base-dir> - generates UnicodeData.txt sources for js and js-ir targets using paths derived from specified base path
|
||||
<UnicodeData.txt-path> <target> <target-dir> - generates UnicodeData.txt sources for the specified target in the specified target directory
|
||||
<kotlin-base-dir> - generates sources for js and js-ir targets using paths derived from specified base path
|
||||
<target> <target-dir> <test-dir> - generates sources for the specified target in the specified target directory
|
||||
"""
|
||||
)
|
||||
exitProcess(1)
|
||||
@@ -144,7 +175,23 @@ fun main(args: Array<String>) {
|
||||
}
|
||||
|
||||
oneToManyMappingsGenerators.forEach {
|
||||
nonCommentSpecialCasingLines.forEach { line -> it.appendLine(line) }
|
||||
specialCasingLines.forEach { line -> it.appendLine(line) }
|
||||
it.generate()
|
||||
}
|
||||
|
||||
stringUppercaseGenerator?.let {
|
||||
specialCasingLines.forEach { line -> it.appendSpecialCasingLine(line) }
|
||||
it.generate()
|
||||
}
|
||||
stringLowercaseGenerator?.let {
|
||||
specialCasingLines.forEach { line -> it.appendSpecialCasingLine(line) }
|
||||
propListLines.forEach { line -> it.appendPropListLine(line) }
|
||||
wordBreakPropertyLines.forEach { line -> it.appendWordBreakPropertyLine(line) }
|
||||
it.generate()
|
||||
}
|
||||
stringCasingTestGenerator?.let {
|
||||
derivedCorePropertiesLines.forEach { line -> it.appendDerivedCorePropertiesLine(line) }
|
||||
it.generate()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
/*
|
||||
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package generators.unicode
|
||||
|
||||
internal class PropertyLine(properties: List<String>) {
|
||||
init {
|
||||
require(properties.size == 2)
|
||||
}
|
||||
|
||||
val rangeStart: String = properties[0].split("..").first()
|
||||
val rangeEnd: String = properties[0].split("..").last()
|
||||
val property: String = properties[1].takeWhile { it != ' ' }
|
||||
|
||||
override fun toString(): String {
|
||||
return "PropertyLine{rangeStart=$rangeStart" +
|
||||
", rangeEnd=$rangeEnd" +
|
||||
", property=$property" +
|
||||
"}"
|
||||
}
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package generators.unicode.mappings.string
|
||||
|
||||
import generators.unicode.SpecialCasingLine
|
||||
import generators.unicode.UnicodeDataLine
|
||||
import generators.unicode.hexToInt
|
||||
|
||||
internal abstract class StringCasingGenerator(
|
||||
unicodeDataLines: List<UnicodeDataLine>
|
||||
) {
|
||||
private val unicodeDataLines = unicodeDataLines.associateBy { it.char.hexToInt() }
|
||||
protected val contextDependentMappings = mutableListOf<SpecialCasingLine>()
|
||||
|
||||
fun appendSpecialCasingLine(line: SpecialCasingLine) {
|
||||
if (line.conditionList.isEmpty()) return
|
||||
|
||||
val isLocaleAgnosticCondition = line.conditionList.all { it.length > 2 }
|
||||
|
||||
val unicodeLine = unicodeDataLines[line.char.hexToInt()]
|
||||
val unicodeDataMapping = unicodeLine?.mapping()?.takeIf { it.isNotEmpty() } ?: line.char
|
||||
|
||||
if (isLocaleAgnosticCondition && line.mapping() != listOf(unicodeDataMapping)) {
|
||||
contextDependentMappings.add(line)
|
||||
}
|
||||
}
|
||||
|
||||
abstract fun SpecialCasingLine.mapping(): List<String>
|
||||
abstract fun UnicodeDataLine.mapping(): String
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
/*
|
||||
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package generators.unicode.mappings.string
|
||||
|
||||
import generators.requireExistingDir
|
||||
import generators.unicode.PropertyLine
|
||||
import generators.unicode.hexToInt
|
||||
import generators.unicode.toHexIntLiteral
|
||||
import generators.unicode.writeHeader
|
||||
import java.io.File
|
||||
import java.io.FileWriter
|
||||
|
||||
internal class StringCasingTestGenerator(private val outputDir: File) {
|
||||
private val casedRanges = mutableListOf<PropertyLine>()
|
||||
private val caseIgnorableRanges = mutableListOf<PropertyLine>()
|
||||
|
||||
init {
|
||||
outputDir.requireExistingDir()
|
||||
}
|
||||
|
||||
fun appendDerivedCorePropertiesLine(line: PropertyLine) {
|
||||
when (line.property) {
|
||||
"Cased" -> casedRanges.add(line)
|
||||
"Case_Ignorable" -> caseIgnorableRanges.add(line)
|
||||
}
|
||||
}
|
||||
|
||||
fun generate() {
|
||||
generateIsCasedTest()
|
||||
generateIsCaseIgnorableTest()
|
||||
}
|
||||
|
||||
private fun generateIsCasedTest() {
|
||||
val file = outputDir.resolve("_IsCasedTest.kt")
|
||||
generateRangesTest(casedRanges, file, "casedRanges", "isCased")
|
||||
}
|
||||
|
||||
private fun generateIsCaseIgnorableTest() {
|
||||
val file = outputDir.resolve("_IsCaseIgnorableTest.kt")
|
||||
generateRangesTest(caseIgnorableRanges, file, "caseIgnorableRanges", "isCaseIgnorable")
|
||||
}
|
||||
|
||||
private fun generateRangesTest(
|
||||
ranges: List<PropertyLine>,
|
||||
file: File,
|
||||
rangesArrayName: String,
|
||||
functionName: String
|
||||
) {
|
||||
FileWriter(file).use { writer ->
|
||||
writer.writeHeader(file, "test.text")
|
||||
writer.appendLine()
|
||||
writer.appendLine("import kotlin.test.*")
|
||||
writer.appendLine()
|
||||
writer.appendLine("@SharedImmutable")
|
||||
writer.appendLine("private val $rangesArrayName = arrayOf<IntRange>(")
|
||||
ranges.forEach {
|
||||
val start = it.rangeStart.hexToInt().toHexIntLiteral()
|
||||
val end = it.rangeEnd.hexToInt().toHexIntLiteral()
|
||||
writer.appendLine(" $start..$end,")
|
||||
}
|
||||
writer.appendLine(")")
|
||||
writer.appendLine()
|
||||
writer.appendLine(test(rangesArrayName, functionName))
|
||||
}
|
||||
}
|
||||
|
||||
@OptIn(ExperimentalStdlibApi::class)
|
||||
private fun test(rangesArrayName: String, functionName: String): String = """
|
||||
class ${functionName.replaceFirstChar { it.uppercase() }}Test {
|
||||
@Test
|
||||
fun $functionName() {
|
||||
var lastChecked = -1
|
||||
for (range in $rangesArrayName) {
|
||||
for (codePoint in lastChecked + 1 until range.first) {
|
||||
assertFalse(codePoint.$functionName())
|
||||
}
|
||||
for (codePoint in range.first..range.last) {
|
||||
assertTrue(codePoint.$functionName())
|
||||
}
|
||||
lastChecked = range.last
|
||||
}
|
||||
for (codePoint in lastChecked + 1..0x10FFFF) {
|
||||
assertFalse(codePoint.$functionName())
|
||||
}
|
||||
}
|
||||
}
|
||||
""".trimIndent()
|
||||
}
|
||||
+246
@@ -0,0 +1,246 @@
|
||||
/*
|
||||
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package generators.unicode.mappings.string
|
||||
|
||||
import generators.unicode.*
|
||||
import generators.unicode.PropertyLine
|
||||
import generators.unicode.SpecialCasingLine
|
||||
import generators.unicode.UnicodeDataLine
|
||||
import generators.unicode.ranges.RangesWritingStrategy
|
||||
import generators.unicode.ranges.builders.RangesBuilder
|
||||
import templates.KotlinTarget
|
||||
import java.io.File
|
||||
import java.io.FileWriter
|
||||
|
||||
internal class StringLowercaseGenerator(
|
||||
private val outputFile: File,
|
||||
unicodeDataLines: List<UnicodeDataLine>,
|
||||
) : StringCasingGenerator(unicodeDataLines) {
|
||||
|
||||
private val casedRanges = mutableListOf<IntRange>()
|
||||
private val caseIgnorableRanges = mutableListOf<IntRange>()
|
||||
|
||||
init {
|
||||
val casedRangesBuilder = CasedRangesBuilder()
|
||||
val caseIgnorableRangesBuilder = CaseIgnorableRangesBuilder()
|
||||
unicodeDataLines.forEach { line ->
|
||||
if (line.char.length > 4) {
|
||||
casedRangesBuilder.append(line.char, line.name, line.categoryCode)
|
||||
caseIgnorableRangesBuilder.append(line.char, line.name, line.categoryCode)
|
||||
}
|
||||
}
|
||||
casedRangesBuilder.build().let { (start, end, _) ->
|
||||
start.indices.forEach { casedRanges.add(start[it]..end[it]) }
|
||||
}
|
||||
caseIgnorableRangesBuilder.build().let { (start, end, _) ->
|
||||
start.indices.forEach { caseIgnorableRanges.add(start[it]..end[it]) }
|
||||
}
|
||||
}
|
||||
|
||||
override fun SpecialCasingLine.mapping(): List<String> = lowercaseMapping
|
||||
|
||||
override fun UnicodeDataLine.mapping(): String = lowercaseMapping
|
||||
|
||||
fun appendPropListLine(line: PropertyLine) {
|
||||
when (line.property) {
|
||||
"Other_Lowercase",
|
||||
"Other_Uppercase" -> casedRanges.add(line.rangeStart.hexToInt()..line.rangeEnd.hexToInt())
|
||||
}
|
||||
}
|
||||
|
||||
fun appendWordBreakPropertyLine(line: PropertyLine) {
|
||||
when (line.property) {
|
||||
"MidLetter",
|
||||
"MidNumLet",
|
||||
"Single_Quote" -> caseIgnorableRanges.add(line.rangeStart.hexToInt()..line.rangeEnd.hexToInt())
|
||||
}
|
||||
}
|
||||
|
||||
fun generate() {
|
||||
check(contextDependentMappings.size == 1 || contextDependentMappings[0].conditionList == listOf("Final_Sigma")) {
|
||||
"The locale-agnostic conditional mappings $contextDependentMappings are not handled."
|
||||
}
|
||||
|
||||
casedRanges.sortBy { it.first }
|
||||
caseIgnorableRanges.sortBy { it.first }
|
||||
|
||||
val strategy = RangesWritingStrategy.of(KotlinTarget.Native)
|
||||
|
||||
FileWriter(outputFile).use { writer ->
|
||||
writer.writeHeader(outputFile, "kotlin.text")
|
||||
writer.appendLine()
|
||||
writer.writeIntArray("casedStart", casedRanges.map { it.first }, strategy)
|
||||
writer.writeIntArray("casedEnd", casedRanges.map { it.last }, strategy)
|
||||
writer.appendLine()
|
||||
writer.appendLine(isCased())
|
||||
writer.appendLine()
|
||||
writer.writeIntArray("caseIgnorableStart", caseIgnorableRanges.map { it.first }, strategy)
|
||||
writer.writeIntArray("caseIgnorableEnd", caseIgnorableRanges.map { it.last }, strategy)
|
||||
writer.appendLine()
|
||||
writer.appendLine(isCaseIgnorable())
|
||||
writer.appendLine()
|
||||
writer.appendLine(codePointBefore())
|
||||
writer.appendLine()
|
||||
writer.appendLine(isFinalSigmaAt())
|
||||
writer.appendLine()
|
||||
writer.appendLine(lowercaseImpl())
|
||||
}
|
||||
}
|
||||
|
||||
private fun isCased(): String = """
|
||||
// Lu + Ll + Lt + Other_Lowercase + Other_Uppercase (PropList.txt of Unicode Character Database files)
|
||||
// Declared internal for testing
|
||||
internal fun Int.isCased(): Boolean {
|
||||
if (this <= Char.MAX_VALUE.toInt()) {
|
||||
when (toChar().getCategoryValue()) {
|
||||
CharCategory.UPPERCASE_LETTER.value,
|
||||
CharCategory.LOWERCASE_LETTER.value,
|
||||
CharCategory.TITLECASE_LETTER.value -> return true
|
||||
}
|
||||
}
|
||||
val index = binarySearchRange(casedStart, this)
|
||||
return index >= 0 && this <= casedEnd[index]
|
||||
}
|
||||
""".trimIndent()
|
||||
|
||||
private fun isCaseIgnorable(): String = """
|
||||
// Mn + Me + Cf + Lm + Sk + Word_Break=MidLetter + Word_Break=MidNumLet + Word_Break=Single_Quote (WordBreakProperty.txt of Unicode Character Database files)
|
||||
// Declared internal for testing
|
||||
internal fun Int.isCaseIgnorable(): Boolean {
|
||||
if (this <= Char.MAX_VALUE.toInt()) {
|
||||
when (toChar().getCategoryValue()) {
|
||||
CharCategory.NON_SPACING_MARK.value,
|
||||
CharCategory.ENCLOSING_MARK.value,
|
||||
CharCategory.FORMAT.value,
|
||||
CharCategory.MODIFIER_LETTER.value,
|
||||
CharCategory.MODIFIER_SYMBOL.value -> return true
|
||||
}
|
||||
}
|
||||
val index = binarySearchRange(caseIgnorableStart, this)
|
||||
return index >= 0 && this <= caseIgnorableEnd[index]
|
||||
}
|
||||
""".trimIndent()
|
||||
|
||||
private fun codePointBefore(): String = """
|
||||
private fun String.codePointBefore(index: Int): Int {
|
||||
val low = this[index]
|
||||
if (low.isLowSurrogate() && index - 1 >= 0) {
|
||||
val high = this[index - 1]
|
||||
if (high.isHighSurrogate()) {
|
||||
return Char.toCodePoint(high, low)
|
||||
}
|
||||
}
|
||||
return low.toInt()
|
||||
}
|
||||
""".trimIndent()
|
||||
|
||||
private fun isFinalSigmaAt(): String = """
|
||||
// \p{cased} (\p{case-ignorable})* Sigma !( (\p{case-ignorable})* \p{cased} )
|
||||
// The regular-expression operator * is "possessive", consuming as many characters as possible, with no backup.
|
||||
// This is significant in the case of Final_Sigma, because the sets of case-ignorable and cased characters are not disjoint.
|
||||
private fun String.isFinalSigmaAt(index: Int): Boolean {
|
||||
if (this[index] == '\u03A3' && index > 0) {
|
||||
var i = index - 1
|
||||
var codePoint: Int = 0
|
||||
while (i >= 0) {
|
||||
codePoint = codePointBefore(i)
|
||||
if (codePoint.isCaseIgnorable()) {
|
||||
i -= codePoint.charCount()
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
if (i >= 0 && codePoint.isCased()) {
|
||||
var j = index + 1
|
||||
while (j < length) {
|
||||
codePoint = codePointAt(j)
|
||||
if (codePoint.isCaseIgnorable()) {
|
||||
j += codePoint.charCount()
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
if (j >= length || !codePoint.isCased()) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
""".trimIndent()
|
||||
|
||||
private fun lowercaseImpl(): String = """
|
||||
internal fun String.lowercaseImpl(): String {
|
||||
var unchangedIndex = 0
|
||||
while (unchangedIndex < this.length) {
|
||||
val codePoint = codePointAt(unchangedIndex)
|
||||
if (codePoint.lowercaseCodePoint() != codePoint) { // '\u0130' and '\u03A3' have lowercase corresponding mapping in UnicodeData.txt, no need to check them separately
|
||||
break
|
||||
}
|
||||
unchangedIndex += codePoint.charCount()
|
||||
}
|
||||
if (unchangedIndex == this.length) {
|
||||
return this
|
||||
}
|
||||
|
||||
val sb = StringBuilder(this.length)
|
||||
sb.appendRange(this, 0, unchangedIndex)
|
||||
|
||||
var index = unchangedIndex
|
||||
|
||||
while (index < this.length) {
|
||||
if (this[index] == '\u0130') {
|
||||
sb.append("\u0069\u0307")
|
||||
index++
|
||||
continue
|
||||
}
|
||||
if (isFinalSigmaAt(index)) {
|
||||
sb.append('\u03C2')
|
||||
index++
|
||||
continue
|
||||
}
|
||||
val codePoint = codePointAt(index)
|
||||
val lowercaseCodePoint = codePoint.lowercaseCodePoint()
|
||||
sb.appendCodePoint(lowercaseCodePoint)
|
||||
index += codePoint.charCount()
|
||||
}
|
||||
|
||||
return sb.toString()
|
||||
}
|
||||
""".trimIndent()
|
||||
}
|
||||
|
||||
private class CasedRangesBuilder : RangesBuilder() {
|
||||
private val id = "Cased"
|
||||
|
||||
override fun categoryId(categoryCode: String): String = when (categoryCode) {
|
||||
CharCategory.UPPERCASE_LETTER.code,
|
||||
CharCategory.LOWERCASE_LETTER.code,
|
||||
CharCategory.TITLECASE_LETTER.code -> id
|
||||
else -> "Else"
|
||||
}
|
||||
|
||||
override fun shouldSkip(categoryId: String): Boolean {
|
||||
return categoryId != id
|
||||
}
|
||||
}
|
||||
|
||||
private class CaseIgnorableRangesBuilder : RangesBuilder() {
|
||||
private val id = "CaseIgnorable"
|
||||
|
||||
override fun categoryId(categoryCode: String): String = when (categoryCode) {
|
||||
CharCategory.NON_SPACING_MARK.code,
|
||||
CharCategory.ENCLOSING_MARK.code,
|
||||
CharCategory.FORMAT.code,
|
||||
CharCategory.MODIFIER_LETTER.code,
|
||||
CharCategory.MODIFIER_SYMBOL.code -> id
|
||||
else -> "Else"
|
||||
}
|
||||
|
||||
override fun shouldSkip(categoryId: String): Boolean {
|
||||
return categoryId != id
|
||||
}
|
||||
}
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package generators.unicode.mappings.string
|
||||
|
||||
import generators.unicode.SpecialCasingLine
|
||||
import generators.unicode.UnicodeDataLine
|
||||
import generators.unicode.writeHeader
|
||||
import java.io.File
|
||||
import java.io.FileWriter
|
||||
|
||||
internal class StringUppercaseGenerator(
|
||||
private val outputFile: File,
|
||||
unicodeDataLines: List<UnicodeDataLine>
|
||||
) : StringCasingGenerator(unicodeDataLines) {
|
||||
|
||||
override fun SpecialCasingLine.mapping(): List<String> = uppercaseMapping
|
||||
|
||||
override fun UnicodeDataLine.mapping(): String = uppercaseMapping
|
||||
|
||||
fun generate() {
|
||||
check(contextDependentMappings.isEmpty()) {
|
||||
"The locale-agnostic conditional mappings $contextDependentMappings are not handled."
|
||||
}
|
||||
|
||||
FileWriter(outputFile).use { writer ->
|
||||
writer.writeHeader(outputFile, "kotlin.text")
|
||||
writer.appendLine()
|
||||
writer.appendLine(codePointAt())
|
||||
writer.appendLine()
|
||||
writer.appendLine(charCount())
|
||||
writer.appendLine()
|
||||
writer.appendLine(appendCodePoint())
|
||||
writer.appendLine()
|
||||
writer.appendLine(uppercaseImpl())
|
||||
}
|
||||
}
|
||||
|
||||
private fun charCount(): String = """
|
||||
internal fun Int.charCount(): Int = if (this >= Char.MIN_SUPPLEMENTARY_CODE_POINT) 2 else 1
|
||||
""".trimIndent()
|
||||
|
||||
private fun codePointAt(): String = """
|
||||
internal fun String.codePointAt(index: Int): Int {
|
||||
val high = this[index]
|
||||
if (high.isHighSurrogate() && index + 1 < this.length) {
|
||||
val low = this[index + 1]
|
||||
if (low.isLowSurrogate()) {
|
||||
return Char.toCodePoint(high, low)
|
||||
}
|
||||
}
|
||||
return high.toInt()
|
||||
}
|
||||
""".trimIndent()
|
||||
|
||||
private fun appendCodePoint(): String = """
|
||||
internal fun StringBuilder.appendCodePoint(codePoint: Int) {
|
||||
if (codePoint < Char.MIN_SUPPLEMENTARY_CODE_POINT) {
|
||||
append(codePoint.toChar())
|
||||
} else {
|
||||
append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10))
|
||||
append(Char.MIN_LOW_SURROGATE + (codePoint and 0x3ff))
|
||||
}
|
||||
}
|
||||
""".trimIndent()
|
||||
|
||||
private fun uppercaseImpl(): String = """
|
||||
internal fun String.uppercaseImpl(): String {
|
||||
var unchangedIndex = 0
|
||||
while (unchangedIndex < this.length) {
|
||||
val codePoint = codePointAt(unchangedIndex)
|
||||
if (this[unchangedIndex].oneToManyUppercase() != null || codePoint.uppercaseCodePoint() != codePoint) {
|
||||
break
|
||||
}
|
||||
unchangedIndex += codePoint.charCount()
|
||||
}
|
||||
if (unchangedIndex == this.length) {
|
||||
return this
|
||||
}
|
||||
|
||||
val sb = StringBuilder(this.length)
|
||||
sb.appendRange(this, 0, unchangedIndex)
|
||||
|
||||
var index = unchangedIndex
|
||||
|
||||
while (index < this.length) {
|
||||
val specialCasing = this[index].oneToManyUppercase()
|
||||
if (specialCasing != null) {
|
||||
sb.append(specialCasing)
|
||||
index++
|
||||
continue
|
||||
}
|
||||
val codePoint = codePointAt(index)
|
||||
val uppercaseCodePoint = codePoint.uppercaseCodePoint()
|
||||
sb.appendCodePoint(uppercaseCodePoint)
|
||||
index += codePoint.charCount()
|
||||
}
|
||||
|
||||
return sb.toString()
|
||||
}
|
||||
""".trimIndent()
|
||||
}
|
||||
+2
-2
@@ -18,9 +18,9 @@ internal sealed class RangesWritingStrategy {
|
||||
abstract fun rangeRef(name: String): String
|
||||
|
||||
companion object {
|
||||
fun of(target: KotlinTarget, wrapperName: String): RangesWritingStrategy {
|
||||
fun of(target: KotlinTarget, wrapperName: String? = null): RangesWritingStrategy {
|
||||
return when (target.platform) {
|
||||
Platform.JS -> JsRangesWritingStrategy(wrapperName)
|
||||
Platform.JS -> JsRangesWritingStrategy(wrapperName!!)
|
||||
else -> NativeRangesWritingStrategy
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user