Implement String case conversion generators #KT-45213
This commit is contained in:
@@ -9,6 +9,9 @@ import generators.unicode.mappings.oneToOne.MappingsGenerator
|
|||||||
import generators.unicode.mappings.oneToMany.OneToManyMappingsGenerator
|
import generators.unicode.mappings.oneToMany.OneToManyMappingsGenerator
|
||||||
import generators.unicode.ranges.CharCategoryTestGenerator
|
import generators.unicode.ranges.CharCategoryTestGenerator
|
||||||
import generators.unicode.ranges.RangesGenerator
|
import generators.unicode.ranges.RangesGenerator
|
||||||
|
import generators.unicode.mappings.string.StringCasingTestGenerator
|
||||||
|
import generators.unicode.mappings.string.StringLowercaseGenerator
|
||||||
|
import generators.unicode.mappings.string.StringUppercaseGenerator
|
||||||
import templates.COPYRIGHT_NOTICE
|
import templates.COPYRIGHT_NOTICE
|
||||||
import templates.KotlinTarget
|
import templates.KotlinTarget
|
||||||
import templates.readCopyrightNoticeFromProfile
|
import templates.readCopyrightNoticeFromProfile
|
||||||
@@ -21,6 +24,9 @@ import kotlin.system.exitProcess
|
|||||||
private const val unicodeVersion = "13.0.0"
|
private const val unicodeVersion = "13.0.0"
|
||||||
private const val unicodeDataUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/UnicodeData.txt"
|
private const val unicodeDataUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/UnicodeData.txt"
|
||||||
private const val specialCasingUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/SpecialCasing.txt"
|
private const val specialCasingUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/SpecialCasing.txt"
|
||||||
|
private const val propListUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/PropList.txt"
|
||||||
|
private const val wordBreakPropertyUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/auxiliary/WordBreakProperty.txt"
|
||||||
|
private const val derivedCorePropertiesUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/DerivedCoreProperties.txt"
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This program generates sources related to UnicodeData.txt and SpecialCasing.txt.
|
* This program generates sources related to UnicodeData.txt and SpecialCasing.txt.
|
||||||
@@ -32,14 +38,29 @@ private const val specialCasingUrl = "https://www.unicode.org/Public/$unicodeVer
|
|||||||
* No tests are generated.
|
* No tests are generated.
|
||||||
*/
|
*/
|
||||||
fun main(args: Array<String>) {
|
fun main(args: Array<String>) {
|
||||||
|
fun readLines(url: String): List<String> {
|
||||||
val unicodeDataLines = URL(unicodeDataUrl).openStream().reader().readLines()
|
return URL(url).openStream().reader().readLines()
|
||||||
val bmpUnicodeDataLines = unicodeDataLines.mapNotNull { line ->
|
|
||||||
UnicodeDataLine(line.split(";")).takeIf { it.char.length <= 4 } // Basic Multilingual Plane (BMP)
|
|
||||||
}
|
}
|
||||||
val specialCasingLines = URL(specialCasingUrl).openStream().reader().readLines()
|
|
||||||
val nonCommentSpecialCasingLines = specialCasingLines.mapNotNull { line ->
|
val unicodeDataLines = readLines(unicodeDataUrl).map { line -> UnicodeDataLine(line.split(";")) }
|
||||||
if (line.isEmpty() || line.startsWith("#")) null else SpecialCasingLine(line.split("; "))
|
val bmpUnicodeDataLines = unicodeDataLines.filter { line -> line.char.length <= 4 } // Basic Multilingual Plane (BMP)
|
||||||
|
|
||||||
|
fun String.isEmptyOrComment(): Boolean = isEmpty() || startsWith("#")
|
||||||
|
|
||||||
|
val specialCasingLines = readLines(specialCasingUrl).filterNot(String::isEmptyOrComment).map { line ->
|
||||||
|
SpecialCasingLine(line.split("; "))
|
||||||
|
}
|
||||||
|
|
||||||
|
val propListLines = readLines(propListUrl).filterNot(String::isEmptyOrComment).map { line ->
|
||||||
|
PropertyLine(line.split("; ").map { it.trim() })
|
||||||
|
}
|
||||||
|
|
||||||
|
val wordBreakPropertyLines = readLines(wordBreakPropertyUrl).filterNot(String::isEmptyOrComment).map { line ->
|
||||||
|
PropertyLine(line.split("; ").map { it.trim() })
|
||||||
|
}
|
||||||
|
|
||||||
|
val derivedCorePropertiesLines = readLines(derivedCorePropertiesUrl).filterNot(String::isEmptyOrComment).map { line ->
|
||||||
|
PropertyLine(line.split("; ").map { it.trim() })
|
||||||
}
|
}
|
||||||
|
|
||||||
val categoryRangesGenerators = mutableListOf<RangesGenerator>()
|
val categoryRangesGenerators = mutableListOf<RangesGenerator>()
|
||||||
@@ -79,6 +100,11 @@ fun main(args: Array<String>) {
|
|||||||
|
|
||||||
var categoryTestGenerator: CharCategoryTestGenerator? = null
|
var categoryTestGenerator: CharCategoryTestGenerator? = null
|
||||||
|
|
||||||
|
var stringUppercaseGenerator: StringUppercaseGenerator? = null
|
||||||
|
var stringLowercaseGenerator: StringLowercaseGenerator? = null
|
||||||
|
|
||||||
|
var stringCasingTestGenerator: StringCasingTestGenerator? = null
|
||||||
|
|
||||||
when (args.size) {
|
when (args.size) {
|
||||||
1 -> {
|
1 -> {
|
||||||
val baseDir = File(args.first())
|
val baseDir = File(args.first())
|
||||||
@@ -93,15 +119,16 @@ fun main(args: Array<String>) {
|
|||||||
addRangesGenerators(jsIrGeneratedDir, KotlinTarget.JS_IR)
|
addRangesGenerators(jsIrGeneratedDir, KotlinTarget.JS_IR)
|
||||||
|
|
||||||
// For debugging. To see the file content
|
// For debugging. To see the file content
|
||||||
fun writeContent(lines: List<String>, fileName: String) {
|
fun downloadFile(fromUrl: String) {
|
||||||
val file = baseDir.resolve("libraries/tools/kotlin-stdlib-gen/src/generators/unicode/$fileName")
|
val fileName = File(fromUrl).name
|
||||||
file.writeText(lines.joinToString(separator = "\n"))
|
val dest = baseDir.resolve("libraries/tools/kotlin-stdlib-gen/src/generators/unicode/$fileName")
|
||||||
|
dest.writeText(readLines(fromUrl).joinToString(separator = "\n"))
|
||||||
}
|
}
|
||||||
writeContent(unicodeDataLines, "UnicodeData.txt")
|
downloadFile(unicodeDataUrl)
|
||||||
writeContent(specialCasingLines, "SpecialCasing.txt")
|
downloadFile(specialCasingUrl)
|
||||||
}
|
}
|
||||||
2 -> {
|
3 -> {
|
||||||
val (targetName, targetDir) = args
|
val (targetName, targetDir, testDir) = args
|
||||||
|
|
||||||
val target = KotlinTarget.values.singleOrNull { it.name.equals(targetName, ignoreCase = true) }
|
val target = KotlinTarget.values.singleOrNull { it.name.equals(targetName, ignoreCase = true) }
|
||||||
?: error("Invalid target: $targetName")
|
?: error("Invalid target: $targetName")
|
||||||
@@ -112,13 +139,17 @@ fun main(args: Array<String>) {
|
|||||||
if (target == KotlinTarget.Native) {
|
if (target == KotlinTarget.Native) {
|
||||||
addOneToOneMappingsGenerators(generatedDir, target)
|
addOneToOneMappingsGenerators(generatedDir, target)
|
||||||
addOneToManyMappingsGenerators(generatedDir, target)
|
addOneToManyMappingsGenerators(generatedDir, target)
|
||||||
|
stringUppercaseGenerator = StringUppercaseGenerator(generatedDir.resolve("_StringUppercase.kt"), unicodeDataLines)
|
||||||
|
stringLowercaseGenerator = StringLowercaseGenerator(generatedDir.resolve("_StringLowercase.kt"), unicodeDataLines)
|
||||||
|
|
||||||
|
stringCasingTestGenerator = StringCasingTestGenerator(File(testDir))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else -> {
|
else -> {
|
||||||
println(
|
println(
|
||||||
"""Parameters:
|
"""Parameters:
|
||||||
<kotlin-base-dir> - generates UnicodeData.txt sources for js and js-ir targets using paths derived from specified base path
|
<kotlin-base-dir> - generates sources for js and js-ir targets using paths derived from specified base path
|
||||||
<UnicodeData.txt-path> <target> <target-dir> - generates UnicodeData.txt sources for the specified target in the specified target directory
|
<target> <target-dir> <test-dir> - generates sources for the specified target in the specified target directory
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
exitProcess(1)
|
exitProcess(1)
|
||||||
@@ -144,7 +175,23 @@ fun main(args: Array<String>) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
oneToManyMappingsGenerators.forEach {
|
oneToManyMappingsGenerators.forEach {
|
||||||
nonCommentSpecialCasingLines.forEach { line -> it.appendLine(line) }
|
specialCasingLines.forEach { line -> it.appendLine(line) }
|
||||||
it.generate()
|
it.generate()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
stringUppercaseGenerator?.let {
|
||||||
|
specialCasingLines.forEach { line -> it.appendSpecialCasingLine(line) }
|
||||||
|
it.generate()
|
||||||
|
}
|
||||||
|
stringLowercaseGenerator?.let {
|
||||||
|
specialCasingLines.forEach { line -> it.appendSpecialCasingLine(line) }
|
||||||
|
propListLines.forEach { line -> it.appendPropListLine(line) }
|
||||||
|
wordBreakPropertyLines.forEach { line -> it.appendWordBreakPropertyLine(line) }
|
||||||
|
it.generate()
|
||||||
|
}
|
||||||
|
stringCasingTestGenerator?.let {
|
||||||
|
derivedCorePropertiesLines.forEach { line -> it.appendDerivedCorePropertiesLine(line) }
|
||||||
|
it.generate()
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||||
|
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package generators.unicode
|
||||||
|
|
||||||
|
internal class PropertyLine(properties: List<String>) {
|
||||||
|
init {
|
||||||
|
require(properties.size == 2)
|
||||||
|
}
|
||||||
|
|
||||||
|
val rangeStart: String = properties[0].split("..").first()
|
||||||
|
val rangeEnd: String = properties[0].split("..").last()
|
||||||
|
val property: String = properties[1].takeWhile { it != ' ' }
|
||||||
|
|
||||||
|
override fun toString(): String {
|
||||||
|
return "PropertyLine{rangeStart=$rangeStart" +
|
||||||
|
", rangeEnd=$rangeEnd" +
|
||||||
|
", property=$property" +
|
||||||
|
"}"
|
||||||
|
}
|
||||||
|
}
|
||||||
+33
@@ -0,0 +1,33 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||||
|
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package generators.unicode.mappings.string
|
||||||
|
|
||||||
|
import generators.unicode.SpecialCasingLine
|
||||||
|
import generators.unicode.UnicodeDataLine
|
||||||
|
import generators.unicode.hexToInt
|
||||||
|
|
||||||
|
internal abstract class StringCasingGenerator(
|
||||||
|
unicodeDataLines: List<UnicodeDataLine>
|
||||||
|
) {
|
||||||
|
private val unicodeDataLines = unicodeDataLines.associateBy { it.char.hexToInt() }
|
||||||
|
protected val contextDependentMappings = mutableListOf<SpecialCasingLine>()
|
||||||
|
|
||||||
|
fun appendSpecialCasingLine(line: SpecialCasingLine) {
|
||||||
|
if (line.conditionList.isEmpty()) return
|
||||||
|
|
||||||
|
val isLocaleAgnosticCondition = line.conditionList.all { it.length > 2 }
|
||||||
|
|
||||||
|
val unicodeLine = unicodeDataLines[line.char.hexToInt()]
|
||||||
|
val unicodeDataMapping = unicodeLine?.mapping()?.takeIf { it.isNotEmpty() } ?: line.char
|
||||||
|
|
||||||
|
if (isLocaleAgnosticCondition && line.mapping() != listOf(unicodeDataMapping)) {
|
||||||
|
contextDependentMappings.add(line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
abstract fun SpecialCasingLine.mapping(): List<String>
|
||||||
|
abstract fun UnicodeDataLine.mapping(): String
|
||||||
|
}
|
||||||
+91
@@ -0,0 +1,91 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||||
|
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package generators.unicode.mappings.string
|
||||||
|
|
||||||
|
import generators.requireExistingDir
|
||||||
|
import generators.unicode.PropertyLine
|
||||||
|
import generators.unicode.hexToInt
|
||||||
|
import generators.unicode.toHexIntLiteral
|
||||||
|
import generators.unicode.writeHeader
|
||||||
|
import java.io.File
|
||||||
|
import java.io.FileWriter
|
||||||
|
|
||||||
|
internal class StringCasingTestGenerator(private val outputDir: File) {
|
||||||
|
private val casedRanges = mutableListOf<PropertyLine>()
|
||||||
|
private val caseIgnorableRanges = mutableListOf<PropertyLine>()
|
||||||
|
|
||||||
|
init {
|
||||||
|
outputDir.requireExistingDir()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun appendDerivedCorePropertiesLine(line: PropertyLine) {
|
||||||
|
when (line.property) {
|
||||||
|
"Cased" -> casedRanges.add(line)
|
||||||
|
"Case_Ignorable" -> caseIgnorableRanges.add(line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun generate() {
|
||||||
|
generateIsCasedTest()
|
||||||
|
generateIsCaseIgnorableTest()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun generateIsCasedTest() {
|
||||||
|
val file = outputDir.resolve("_IsCasedTest.kt")
|
||||||
|
generateRangesTest(casedRanges, file, "casedRanges", "isCased")
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun generateIsCaseIgnorableTest() {
|
||||||
|
val file = outputDir.resolve("_IsCaseIgnorableTest.kt")
|
||||||
|
generateRangesTest(caseIgnorableRanges, file, "caseIgnorableRanges", "isCaseIgnorable")
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun generateRangesTest(
|
||||||
|
ranges: List<PropertyLine>,
|
||||||
|
file: File,
|
||||||
|
rangesArrayName: String,
|
||||||
|
functionName: String
|
||||||
|
) {
|
||||||
|
FileWriter(file).use { writer ->
|
||||||
|
writer.writeHeader(file, "test.text")
|
||||||
|
writer.appendLine()
|
||||||
|
writer.appendLine("import kotlin.test.*")
|
||||||
|
writer.appendLine()
|
||||||
|
writer.appendLine("@SharedImmutable")
|
||||||
|
writer.appendLine("private val $rangesArrayName = arrayOf<IntRange>(")
|
||||||
|
ranges.forEach {
|
||||||
|
val start = it.rangeStart.hexToInt().toHexIntLiteral()
|
||||||
|
val end = it.rangeEnd.hexToInt().toHexIntLiteral()
|
||||||
|
writer.appendLine(" $start..$end,")
|
||||||
|
}
|
||||||
|
writer.appendLine(")")
|
||||||
|
writer.appendLine()
|
||||||
|
writer.appendLine(test(rangesArrayName, functionName))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@OptIn(ExperimentalStdlibApi::class)
|
||||||
|
private fun test(rangesArrayName: String, functionName: String): String = """
|
||||||
|
class ${functionName.replaceFirstChar { it.uppercase() }}Test {
|
||||||
|
@Test
|
||||||
|
fun $functionName() {
|
||||||
|
var lastChecked = -1
|
||||||
|
for (range in $rangesArrayName) {
|
||||||
|
for (codePoint in lastChecked + 1 until range.first) {
|
||||||
|
assertFalse(codePoint.$functionName())
|
||||||
|
}
|
||||||
|
for (codePoint in range.first..range.last) {
|
||||||
|
assertTrue(codePoint.$functionName())
|
||||||
|
}
|
||||||
|
lastChecked = range.last
|
||||||
|
}
|
||||||
|
for (codePoint in lastChecked + 1..0x10FFFF) {
|
||||||
|
assertFalse(codePoint.$functionName())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
""".trimIndent()
|
||||||
|
}
|
||||||
+246
@@ -0,0 +1,246 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||||
|
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package generators.unicode.mappings.string
|
||||||
|
|
||||||
|
import generators.unicode.*
|
||||||
|
import generators.unicode.PropertyLine
|
||||||
|
import generators.unicode.SpecialCasingLine
|
||||||
|
import generators.unicode.UnicodeDataLine
|
||||||
|
import generators.unicode.ranges.RangesWritingStrategy
|
||||||
|
import generators.unicode.ranges.builders.RangesBuilder
|
||||||
|
import templates.KotlinTarget
|
||||||
|
import java.io.File
|
||||||
|
import java.io.FileWriter
|
||||||
|
|
||||||
|
internal class StringLowercaseGenerator(
|
||||||
|
private val outputFile: File,
|
||||||
|
unicodeDataLines: List<UnicodeDataLine>,
|
||||||
|
) : StringCasingGenerator(unicodeDataLines) {
|
||||||
|
|
||||||
|
private val casedRanges = mutableListOf<IntRange>()
|
||||||
|
private val caseIgnorableRanges = mutableListOf<IntRange>()
|
||||||
|
|
||||||
|
init {
|
||||||
|
val casedRangesBuilder = CasedRangesBuilder()
|
||||||
|
val caseIgnorableRangesBuilder = CaseIgnorableRangesBuilder()
|
||||||
|
unicodeDataLines.forEach { line ->
|
||||||
|
if (line.char.length > 4) {
|
||||||
|
casedRangesBuilder.append(line.char, line.name, line.categoryCode)
|
||||||
|
caseIgnorableRangesBuilder.append(line.char, line.name, line.categoryCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
casedRangesBuilder.build().let { (start, end, _) ->
|
||||||
|
start.indices.forEach { casedRanges.add(start[it]..end[it]) }
|
||||||
|
}
|
||||||
|
caseIgnorableRangesBuilder.build().let { (start, end, _) ->
|
||||||
|
start.indices.forEach { caseIgnorableRanges.add(start[it]..end[it]) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun SpecialCasingLine.mapping(): List<String> = lowercaseMapping
|
||||||
|
|
||||||
|
override fun UnicodeDataLine.mapping(): String = lowercaseMapping
|
||||||
|
|
||||||
|
fun appendPropListLine(line: PropertyLine) {
|
||||||
|
when (line.property) {
|
||||||
|
"Other_Lowercase",
|
||||||
|
"Other_Uppercase" -> casedRanges.add(line.rangeStart.hexToInt()..line.rangeEnd.hexToInt())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun appendWordBreakPropertyLine(line: PropertyLine) {
|
||||||
|
when (line.property) {
|
||||||
|
"MidLetter",
|
||||||
|
"MidNumLet",
|
||||||
|
"Single_Quote" -> caseIgnorableRanges.add(line.rangeStart.hexToInt()..line.rangeEnd.hexToInt())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun generate() {
|
||||||
|
check(contextDependentMappings.size == 1 || contextDependentMappings[0].conditionList == listOf("Final_Sigma")) {
|
||||||
|
"The locale-agnostic conditional mappings $contextDependentMappings are not handled."
|
||||||
|
}
|
||||||
|
|
||||||
|
casedRanges.sortBy { it.first }
|
||||||
|
caseIgnorableRanges.sortBy { it.first }
|
||||||
|
|
||||||
|
val strategy = RangesWritingStrategy.of(KotlinTarget.Native)
|
||||||
|
|
||||||
|
FileWriter(outputFile).use { writer ->
|
||||||
|
writer.writeHeader(outputFile, "kotlin.text")
|
||||||
|
writer.appendLine()
|
||||||
|
writer.writeIntArray("casedStart", casedRanges.map { it.first }, strategy)
|
||||||
|
writer.writeIntArray("casedEnd", casedRanges.map { it.last }, strategy)
|
||||||
|
writer.appendLine()
|
||||||
|
writer.appendLine(isCased())
|
||||||
|
writer.appendLine()
|
||||||
|
writer.writeIntArray("caseIgnorableStart", caseIgnorableRanges.map { it.first }, strategy)
|
||||||
|
writer.writeIntArray("caseIgnorableEnd", caseIgnorableRanges.map { it.last }, strategy)
|
||||||
|
writer.appendLine()
|
||||||
|
writer.appendLine(isCaseIgnorable())
|
||||||
|
writer.appendLine()
|
||||||
|
writer.appendLine(codePointBefore())
|
||||||
|
writer.appendLine()
|
||||||
|
writer.appendLine(isFinalSigmaAt())
|
||||||
|
writer.appendLine()
|
||||||
|
writer.appendLine(lowercaseImpl())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun isCased(): String = """
|
||||||
|
// Lu + Ll + Lt + Other_Lowercase + Other_Uppercase (PropList.txt of Unicode Character Database files)
|
||||||
|
// Declared internal for testing
|
||||||
|
internal fun Int.isCased(): Boolean {
|
||||||
|
if (this <= Char.MAX_VALUE.toInt()) {
|
||||||
|
when (toChar().getCategoryValue()) {
|
||||||
|
CharCategory.UPPERCASE_LETTER.value,
|
||||||
|
CharCategory.LOWERCASE_LETTER.value,
|
||||||
|
CharCategory.TITLECASE_LETTER.value -> return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val index = binarySearchRange(casedStart, this)
|
||||||
|
return index >= 0 && this <= casedEnd[index]
|
||||||
|
}
|
||||||
|
""".trimIndent()
|
||||||
|
|
||||||
|
private fun isCaseIgnorable(): String = """
|
||||||
|
// Mn + Me + Cf + Lm + Sk + Word_Break=MidLetter + Word_Break=MidNumLet + Word_Break=Single_Quote (WordBreakProperty.txt of Unicode Character Database files)
|
||||||
|
// Declared internal for testing
|
||||||
|
internal fun Int.isCaseIgnorable(): Boolean {
|
||||||
|
if (this <= Char.MAX_VALUE.toInt()) {
|
||||||
|
when (toChar().getCategoryValue()) {
|
||||||
|
CharCategory.NON_SPACING_MARK.value,
|
||||||
|
CharCategory.ENCLOSING_MARK.value,
|
||||||
|
CharCategory.FORMAT.value,
|
||||||
|
CharCategory.MODIFIER_LETTER.value,
|
||||||
|
CharCategory.MODIFIER_SYMBOL.value -> return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val index = binarySearchRange(caseIgnorableStart, this)
|
||||||
|
return index >= 0 && this <= caseIgnorableEnd[index]
|
||||||
|
}
|
||||||
|
""".trimIndent()
|
||||||
|
|
||||||
|
private fun codePointBefore(): String = """
|
||||||
|
private fun String.codePointBefore(index: Int): Int {
|
||||||
|
val low = this[index]
|
||||||
|
if (low.isLowSurrogate() && index - 1 >= 0) {
|
||||||
|
val high = this[index - 1]
|
||||||
|
if (high.isHighSurrogate()) {
|
||||||
|
return Char.toCodePoint(high, low)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return low.toInt()
|
||||||
|
}
|
||||||
|
""".trimIndent()
|
||||||
|
|
||||||
|
private fun isFinalSigmaAt(): String = """
|
||||||
|
// \p{cased} (\p{case-ignorable})* Sigma !( (\p{case-ignorable})* \p{cased} )
|
||||||
|
// The regular-expression operator * is "possessive", consuming as many characters as possible, with no backup.
|
||||||
|
// This is significant in the case of Final_Sigma, because the sets of case-ignorable and cased characters are not disjoint.
|
||||||
|
private fun String.isFinalSigmaAt(index: Int): Boolean {
|
||||||
|
if (this[index] == '\u03A3' && index > 0) {
|
||||||
|
var i = index - 1
|
||||||
|
var codePoint: Int = 0
|
||||||
|
while (i >= 0) {
|
||||||
|
codePoint = codePointBefore(i)
|
||||||
|
if (codePoint.isCaseIgnorable()) {
|
||||||
|
i -= codePoint.charCount()
|
||||||
|
} else {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (i >= 0 && codePoint.isCased()) {
|
||||||
|
var j = index + 1
|
||||||
|
while (j < length) {
|
||||||
|
codePoint = codePointAt(j)
|
||||||
|
if (codePoint.isCaseIgnorable()) {
|
||||||
|
j += codePoint.charCount()
|
||||||
|
} else {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (j >= length || !codePoint.isCased()) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
""".trimIndent()
|
||||||
|
|
||||||
|
private fun lowercaseImpl(): String = """
|
||||||
|
internal fun String.lowercaseImpl(): String {
|
||||||
|
var unchangedIndex = 0
|
||||||
|
while (unchangedIndex < this.length) {
|
||||||
|
val codePoint = codePointAt(unchangedIndex)
|
||||||
|
if (codePoint.lowercaseCodePoint() != codePoint) { // '\u0130' and '\u03A3' have lowercase corresponding mapping in UnicodeData.txt, no need to check them separately
|
||||||
|
break
|
||||||
|
}
|
||||||
|
unchangedIndex += codePoint.charCount()
|
||||||
|
}
|
||||||
|
if (unchangedIndex == this.length) {
|
||||||
|
return this
|
||||||
|
}
|
||||||
|
|
||||||
|
val sb = StringBuilder(this.length)
|
||||||
|
sb.appendRange(this, 0, unchangedIndex)
|
||||||
|
|
||||||
|
var index = unchangedIndex
|
||||||
|
|
||||||
|
while (index < this.length) {
|
||||||
|
if (this[index] == '\u0130') {
|
||||||
|
sb.append("\u0069\u0307")
|
||||||
|
index++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (isFinalSigmaAt(index)) {
|
||||||
|
sb.append('\u03C2')
|
||||||
|
index++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
val codePoint = codePointAt(index)
|
||||||
|
val lowercaseCodePoint = codePoint.lowercaseCodePoint()
|
||||||
|
sb.appendCodePoint(lowercaseCodePoint)
|
||||||
|
index += codePoint.charCount()
|
||||||
|
}
|
||||||
|
|
||||||
|
return sb.toString()
|
||||||
|
}
|
||||||
|
""".trimIndent()
|
||||||
|
}
|
||||||
|
|
||||||
|
private class CasedRangesBuilder : RangesBuilder() {
|
||||||
|
private val id = "Cased"
|
||||||
|
|
||||||
|
override fun categoryId(categoryCode: String): String = when (categoryCode) {
|
||||||
|
CharCategory.UPPERCASE_LETTER.code,
|
||||||
|
CharCategory.LOWERCASE_LETTER.code,
|
||||||
|
CharCategory.TITLECASE_LETTER.code -> id
|
||||||
|
else -> "Else"
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun shouldSkip(categoryId: String): Boolean {
|
||||||
|
return categoryId != id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private class CaseIgnorableRangesBuilder : RangesBuilder() {
|
||||||
|
private val id = "CaseIgnorable"
|
||||||
|
|
||||||
|
override fun categoryId(categoryCode: String): String = when (categoryCode) {
|
||||||
|
CharCategory.NON_SPACING_MARK.code,
|
||||||
|
CharCategory.ENCLOSING_MARK.code,
|
||||||
|
CharCategory.FORMAT.code,
|
||||||
|
CharCategory.MODIFIER_LETTER.code,
|
||||||
|
CharCategory.MODIFIER_SYMBOL.code -> id
|
||||||
|
else -> "Else"
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun shouldSkip(categoryId: String): Boolean {
|
||||||
|
return categoryId != id
|
||||||
|
}
|
||||||
|
}
|
||||||
+104
@@ -0,0 +1,104 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||||
|
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package generators.unicode.mappings.string
|
||||||
|
|
||||||
|
import generators.unicode.SpecialCasingLine
|
||||||
|
import generators.unicode.UnicodeDataLine
|
||||||
|
import generators.unicode.writeHeader
|
||||||
|
import java.io.File
|
||||||
|
import java.io.FileWriter
|
||||||
|
|
||||||
|
internal class StringUppercaseGenerator(
|
||||||
|
private val outputFile: File,
|
||||||
|
unicodeDataLines: List<UnicodeDataLine>
|
||||||
|
) : StringCasingGenerator(unicodeDataLines) {
|
||||||
|
|
||||||
|
override fun SpecialCasingLine.mapping(): List<String> = uppercaseMapping
|
||||||
|
|
||||||
|
override fun UnicodeDataLine.mapping(): String = uppercaseMapping
|
||||||
|
|
||||||
|
fun generate() {
|
||||||
|
check(contextDependentMappings.isEmpty()) {
|
||||||
|
"The locale-agnostic conditional mappings $contextDependentMappings are not handled."
|
||||||
|
}
|
||||||
|
|
||||||
|
FileWriter(outputFile).use { writer ->
|
||||||
|
writer.writeHeader(outputFile, "kotlin.text")
|
||||||
|
writer.appendLine()
|
||||||
|
writer.appendLine(codePointAt())
|
||||||
|
writer.appendLine()
|
||||||
|
writer.appendLine(charCount())
|
||||||
|
writer.appendLine()
|
||||||
|
writer.appendLine(appendCodePoint())
|
||||||
|
writer.appendLine()
|
||||||
|
writer.appendLine(uppercaseImpl())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun charCount(): String = """
|
||||||
|
internal fun Int.charCount(): Int = if (this >= Char.MIN_SUPPLEMENTARY_CODE_POINT) 2 else 1
|
||||||
|
""".trimIndent()
|
||||||
|
|
||||||
|
private fun codePointAt(): String = """
|
||||||
|
internal fun String.codePointAt(index: Int): Int {
|
||||||
|
val high = this[index]
|
||||||
|
if (high.isHighSurrogate() && index + 1 < this.length) {
|
||||||
|
val low = this[index + 1]
|
||||||
|
if (low.isLowSurrogate()) {
|
||||||
|
return Char.toCodePoint(high, low)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return high.toInt()
|
||||||
|
}
|
||||||
|
""".trimIndent()
|
||||||
|
|
||||||
|
private fun appendCodePoint(): String = """
|
||||||
|
internal fun StringBuilder.appendCodePoint(codePoint: Int) {
|
||||||
|
if (codePoint < Char.MIN_SUPPLEMENTARY_CODE_POINT) {
|
||||||
|
append(codePoint.toChar())
|
||||||
|
} else {
|
||||||
|
append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10))
|
||||||
|
append(Char.MIN_LOW_SURROGATE + (codePoint and 0x3ff))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
""".trimIndent()
|
||||||
|
|
||||||
|
private fun uppercaseImpl(): String = """
|
||||||
|
internal fun String.uppercaseImpl(): String {
|
||||||
|
var unchangedIndex = 0
|
||||||
|
while (unchangedIndex < this.length) {
|
||||||
|
val codePoint = codePointAt(unchangedIndex)
|
||||||
|
if (this[unchangedIndex].oneToManyUppercase() != null || codePoint.uppercaseCodePoint() != codePoint) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
unchangedIndex += codePoint.charCount()
|
||||||
|
}
|
||||||
|
if (unchangedIndex == this.length) {
|
||||||
|
return this
|
||||||
|
}
|
||||||
|
|
||||||
|
val sb = StringBuilder(this.length)
|
||||||
|
sb.appendRange(this, 0, unchangedIndex)
|
||||||
|
|
||||||
|
var index = unchangedIndex
|
||||||
|
|
||||||
|
while (index < this.length) {
|
||||||
|
val specialCasing = this[index].oneToManyUppercase()
|
||||||
|
if (specialCasing != null) {
|
||||||
|
sb.append(specialCasing)
|
||||||
|
index++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
val codePoint = codePointAt(index)
|
||||||
|
val uppercaseCodePoint = codePoint.uppercaseCodePoint()
|
||||||
|
sb.appendCodePoint(uppercaseCodePoint)
|
||||||
|
index += codePoint.charCount()
|
||||||
|
}
|
||||||
|
|
||||||
|
return sb.toString()
|
||||||
|
}
|
||||||
|
""".trimIndent()
|
||||||
|
}
|
||||||
+2
-2
@@ -18,9 +18,9 @@ internal sealed class RangesWritingStrategy {
|
|||||||
abstract fun rangeRef(name: String): String
|
abstract fun rangeRef(name: String): String
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
fun of(target: KotlinTarget, wrapperName: String): RangesWritingStrategy {
|
fun of(target: KotlinTarget, wrapperName: String? = null): RangesWritingStrategy {
|
||||||
return when (target.platform) {
|
return when (target.platform) {
|
||||||
Platform.JS -> JsRangesWritingStrategy(wrapperName)
|
Platform.JS -> JsRangesWritingStrategy(wrapperName!!)
|
||||||
else -> NativeRangesWritingStrategy
|
else -> NativeRangesWritingStrategy
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user