Implement String case conversion generators #KT-45213

This commit is contained in:
Abduqodiri Qurbonzoda
2021-03-01 23:09:15 +03:00
parent 1066de1be6
commit fc5e4c20c1
7 changed files with 563 additions and 19 deletions
@@ -9,6 +9,9 @@ import generators.unicode.mappings.oneToOne.MappingsGenerator
import generators.unicode.mappings.oneToMany.OneToManyMappingsGenerator
import generators.unicode.ranges.CharCategoryTestGenerator
import generators.unicode.ranges.RangesGenerator
import generators.unicode.mappings.string.StringCasingTestGenerator
import generators.unicode.mappings.string.StringLowercaseGenerator
import generators.unicode.mappings.string.StringUppercaseGenerator
import templates.COPYRIGHT_NOTICE
import templates.KotlinTarget
import templates.readCopyrightNoticeFromProfile
@@ -21,6 +24,9 @@ import kotlin.system.exitProcess
private const val unicodeVersion = "13.0.0"
private const val unicodeDataUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/UnicodeData.txt"
private const val specialCasingUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/SpecialCasing.txt"
private const val propListUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/PropList.txt"
private const val wordBreakPropertyUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/auxiliary/WordBreakProperty.txt"
private const val derivedCorePropertiesUrl = "https://www.unicode.org/Public/$unicodeVersion/ucd/DerivedCoreProperties.txt"
/**
* This program generates sources related to UnicodeData.txt and SpecialCasing.txt.
@@ -32,14 +38,29 @@ private const val specialCasingUrl = "https://www.unicode.org/Public/$unicodeVer
* No tests are generated.
*/
fun main(args: Array<String>) {
val unicodeDataLines = URL(unicodeDataUrl).openStream().reader().readLines()
val bmpUnicodeDataLines = unicodeDataLines.mapNotNull { line ->
UnicodeDataLine(line.split(";")).takeIf { it.char.length <= 4 } // Basic Multilingual Plane (BMP)
fun readLines(url: String): List<String> {
return URL(url).openStream().reader().readLines()
}
val specialCasingLines = URL(specialCasingUrl).openStream().reader().readLines()
val nonCommentSpecialCasingLines = specialCasingLines.mapNotNull { line ->
if (line.isEmpty() || line.startsWith("#")) null else SpecialCasingLine(line.split("; "))
val unicodeDataLines = readLines(unicodeDataUrl).map { line -> UnicodeDataLine(line.split(";")) }
val bmpUnicodeDataLines = unicodeDataLines.filter { line -> line.char.length <= 4 } // Basic Multilingual Plane (BMP)
fun String.isEmptyOrComment(): Boolean = isEmpty() || startsWith("#")
val specialCasingLines = readLines(specialCasingUrl).filterNot(String::isEmptyOrComment).map { line ->
SpecialCasingLine(line.split("; "))
}
val propListLines = readLines(propListUrl).filterNot(String::isEmptyOrComment).map { line ->
PropertyLine(line.split("; ").map { it.trim() })
}
val wordBreakPropertyLines = readLines(wordBreakPropertyUrl).filterNot(String::isEmptyOrComment).map { line ->
PropertyLine(line.split("; ").map { it.trim() })
}
val derivedCorePropertiesLines = readLines(derivedCorePropertiesUrl).filterNot(String::isEmptyOrComment).map { line ->
PropertyLine(line.split("; ").map { it.trim() })
}
val categoryRangesGenerators = mutableListOf<RangesGenerator>()
@@ -79,6 +100,11 @@ fun main(args: Array<String>) {
var categoryTestGenerator: CharCategoryTestGenerator? = null
var stringUppercaseGenerator: StringUppercaseGenerator? = null
var stringLowercaseGenerator: StringLowercaseGenerator? = null
var stringCasingTestGenerator: StringCasingTestGenerator? = null
when (args.size) {
1 -> {
val baseDir = File(args.first())
@@ -93,15 +119,16 @@ fun main(args: Array<String>) {
addRangesGenerators(jsIrGeneratedDir, KotlinTarget.JS_IR)
// For debugging. To see the file content
fun writeContent(lines: List<String>, fileName: String) {
val file = baseDir.resolve("libraries/tools/kotlin-stdlib-gen/src/generators/unicode/$fileName")
file.writeText(lines.joinToString(separator = "\n"))
fun downloadFile(fromUrl: String) {
val fileName = File(fromUrl).name
val dest = baseDir.resolve("libraries/tools/kotlin-stdlib-gen/src/generators/unicode/$fileName")
dest.writeText(readLines(fromUrl).joinToString(separator = "\n"))
}
writeContent(unicodeDataLines, "UnicodeData.txt")
writeContent(specialCasingLines, "SpecialCasing.txt")
downloadFile(unicodeDataUrl)
downloadFile(specialCasingUrl)
}
2 -> {
val (targetName, targetDir) = args
3 -> {
val (targetName, targetDir, testDir) = args
val target = KotlinTarget.values.singleOrNull { it.name.equals(targetName, ignoreCase = true) }
?: error("Invalid target: $targetName")
@@ -112,13 +139,17 @@ fun main(args: Array<String>) {
if (target == KotlinTarget.Native) {
addOneToOneMappingsGenerators(generatedDir, target)
addOneToManyMappingsGenerators(generatedDir, target)
stringUppercaseGenerator = StringUppercaseGenerator(generatedDir.resolve("_StringUppercase.kt"), unicodeDataLines)
stringLowercaseGenerator = StringLowercaseGenerator(generatedDir.resolve("_StringLowercase.kt"), unicodeDataLines)
stringCasingTestGenerator = StringCasingTestGenerator(File(testDir))
}
}
else -> {
println(
"""Parameters:
<kotlin-base-dir> - generates UnicodeData.txt sources for js and js-ir targets using paths derived from specified base path
<UnicodeData.txt-path> <target> <target-dir> - generates UnicodeData.txt sources for the specified target in the specified target directory
<kotlin-base-dir> - generates sources for js and js-ir targets using paths derived from specified base path
<target> <target-dir> <test-dir> - generates sources for the specified target in the specified target directory
"""
)
exitProcess(1)
@@ -144,7 +175,23 @@ fun main(args: Array<String>) {
}
oneToManyMappingsGenerators.forEach {
nonCommentSpecialCasingLines.forEach { line -> it.appendLine(line) }
specialCasingLines.forEach { line -> it.appendLine(line) }
it.generate()
}
stringUppercaseGenerator?.let {
specialCasingLines.forEach { line -> it.appendSpecialCasingLine(line) }
it.generate()
}
stringLowercaseGenerator?.let {
specialCasingLines.forEach { line -> it.appendSpecialCasingLine(line) }
propListLines.forEach { line -> it.appendPropListLine(line) }
wordBreakPropertyLines.forEach { line -> it.appendWordBreakPropertyLine(line) }
it.generate()
}
stringCasingTestGenerator?.let {
derivedCorePropertiesLines.forEach { line -> it.appendDerivedCorePropertiesLine(line) }
it.generate()
}
}
@@ -0,0 +1,23 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package generators.unicode
internal class PropertyLine(properties: List<String>) {
init {
require(properties.size == 2)
}
val rangeStart: String = properties[0].split("..").first()
val rangeEnd: String = properties[0].split("..").last()
val property: String = properties[1].takeWhile { it != ' ' }
override fun toString(): String {
return "PropertyLine{rangeStart=$rangeStart" +
", rangeEnd=$rangeEnd" +
", property=$property" +
"}"
}
}
@@ -0,0 +1,33 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package generators.unicode.mappings.string
import generators.unicode.SpecialCasingLine
import generators.unicode.UnicodeDataLine
import generators.unicode.hexToInt
internal abstract class StringCasingGenerator(
unicodeDataLines: List<UnicodeDataLine>
) {
private val unicodeDataLines = unicodeDataLines.associateBy { it.char.hexToInt() }
protected val contextDependentMappings = mutableListOf<SpecialCasingLine>()
fun appendSpecialCasingLine(line: SpecialCasingLine) {
if (line.conditionList.isEmpty()) return
val isLocaleAgnosticCondition = line.conditionList.all { it.length > 2 }
val unicodeLine = unicodeDataLines[line.char.hexToInt()]
val unicodeDataMapping = unicodeLine?.mapping()?.takeIf { it.isNotEmpty() } ?: line.char
if (isLocaleAgnosticCondition && line.mapping() != listOf(unicodeDataMapping)) {
contextDependentMappings.add(line)
}
}
abstract fun SpecialCasingLine.mapping(): List<String>
abstract fun UnicodeDataLine.mapping(): String
}
@@ -0,0 +1,91 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package generators.unicode.mappings.string
import generators.requireExistingDir
import generators.unicode.PropertyLine
import generators.unicode.hexToInt
import generators.unicode.toHexIntLiteral
import generators.unicode.writeHeader
import java.io.File
import java.io.FileWriter
internal class StringCasingTestGenerator(private val outputDir: File) {
private val casedRanges = mutableListOf<PropertyLine>()
private val caseIgnorableRanges = mutableListOf<PropertyLine>()
init {
outputDir.requireExistingDir()
}
fun appendDerivedCorePropertiesLine(line: PropertyLine) {
when (line.property) {
"Cased" -> casedRanges.add(line)
"Case_Ignorable" -> caseIgnorableRanges.add(line)
}
}
fun generate() {
generateIsCasedTest()
generateIsCaseIgnorableTest()
}
private fun generateIsCasedTest() {
val file = outputDir.resolve("_IsCasedTest.kt")
generateRangesTest(casedRanges, file, "casedRanges", "isCased")
}
private fun generateIsCaseIgnorableTest() {
val file = outputDir.resolve("_IsCaseIgnorableTest.kt")
generateRangesTest(caseIgnorableRanges, file, "caseIgnorableRanges", "isCaseIgnorable")
}
private fun generateRangesTest(
ranges: List<PropertyLine>,
file: File,
rangesArrayName: String,
functionName: String
) {
FileWriter(file).use { writer ->
writer.writeHeader(file, "test.text")
writer.appendLine()
writer.appendLine("import kotlin.test.*")
writer.appendLine()
writer.appendLine("@SharedImmutable")
writer.appendLine("private val $rangesArrayName = arrayOf<IntRange>(")
ranges.forEach {
val start = it.rangeStart.hexToInt().toHexIntLiteral()
val end = it.rangeEnd.hexToInt().toHexIntLiteral()
writer.appendLine(" $start..$end,")
}
writer.appendLine(")")
writer.appendLine()
writer.appendLine(test(rangesArrayName, functionName))
}
}
@OptIn(ExperimentalStdlibApi::class)
private fun test(rangesArrayName: String, functionName: String): String = """
class ${functionName.replaceFirstChar { it.uppercase() }}Test {
@Test
fun $functionName() {
var lastChecked = -1
for (range in $rangesArrayName) {
for (codePoint in lastChecked + 1 until range.first) {
assertFalse(codePoint.$functionName())
}
for (codePoint in range.first..range.last) {
assertTrue(codePoint.$functionName())
}
lastChecked = range.last
}
for (codePoint in lastChecked + 1..0x10FFFF) {
assertFalse(codePoint.$functionName())
}
}
}
""".trimIndent()
}
@@ -0,0 +1,246 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package generators.unicode.mappings.string
import generators.unicode.*
import generators.unicode.PropertyLine
import generators.unicode.SpecialCasingLine
import generators.unicode.UnicodeDataLine
import generators.unicode.ranges.RangesWritingStrategy
import generators.unicode.ranges.builders.RangesBuilder
import templates.KotlinTarget
import java.io.File
import java.io.FileWriter
internal class StringLowercaseGenerator(
private val outputFile: File,
unicodeDataLines: List<UnicodeDataLine>,
) : StringCasingGenerator(unicodeDataLines) {
private val casedRanges = mutableListOf<IntRange>()
private val caseIgnorableRanges = mutableListOf<IntRange>()
init {
val casedRangesBuilder = CasedRangesBuilder()
val caseIgnorableRangesBuilder = CaseIgnorableRangesBuilder()
unicodeDataLines.forEach { line ->
if (line.char.length > 4) {
casedRangesBuilder.append(line.char, line.name, line.categoryCode)
caseIgnorableRangesBuilder.append(line.char, line.name, line.categoryCode)
}
}
casedRangesBuilder.build().let { (start, end, _) ->
start.indices.forEach { casedRanges.add(start[it]..end[it]) }
}
caseIgnorableRangesBuilder.build().let { (start, end, _) ->
start.indices.forEach { caseIgnorableRanges.add(start[it]..end[it]) }
}
}
override fun SpecialCasingLine.mapping(): List<String> = lowercaseMapping
override fun UnicodeDataLine.mapping(): String = lowercaseMapping
fun appendPropListLine(line: PropertyLine) {
when (line.property) {
"Other_Lowercase",
"Other_Uppercase" -> casedRanges.add(line.rangeStart.hexToInt()..line.rangeEnd.hexToInt())
}
}
fun appendWordBreakPropertyLine(line: PropertyLine) {
when (line.property) {
"MidLetter",
"MidNumLet",
"Single_Quote" -> caseIgnorableRanges.add(line.rangeStart.hexToInt()..line.rangeEnd.hexToInt())
}
}
fun generate() {
check(contextDependentMappings.size == 1 || contextDependentMappings[0].conditionList == listOf("Final_Sigma")) {
"The locale-agnostic conditional mappings $contextDependentMappings are not handled."
}
casedRanges.sortBy { it.first }
caseIgnorableRanges.sortBy { it.first }
val strategy = RangesWritingStrategy.of(KotlinTarget.Native)
FileWriter(outputFile).use { writer ->
writer.writeHeader(outputFile, "kotlin.text")
writer.appendLine()
writer.writeIntArray("casedStart", casedRanges.map { it.first }, strategy)
writer.writeIntArray("casedEnd", casedRanges.map { it.last }, strategy)
writer.appendLine()
writer.appendLine(isCased())
writer.appendLine()
writer.writeIntArray("caseIgnorableStart", caseIgnorableRanges.map { it.first }, strategy)
writer.writeIntArray("caseIgnorableEnd", caseIgnorableRanges.map { it.last }, strategy)
writer.appendLine()
writer.appendLine(isCaseIgnorable())
writer.appendLine()
writer.appendLine(codePointBefore())
writer.appendLine()
writer.appendLine(isFinalSigmaAt())
writer.appendLine()
writer.appendLine(lowercaseImpl())
}
}
private fun isCased(): String = """
// Lu + Ll + Lt + Other_Lowercase + Other_Uppercase (PropList.txt of Unicode Character Database files)
// Declared internal for testing
internal fun Int.isCased(): Boolean {
if (this <= Char.MAX_VALUE.toInt()) {
when (toChar().getCategoryValue()) {
CharCategory.UPPERCASE_LETTER.value,
CharCategory.LOWERCASE_LETTER.value,
CharCategory.TITLECASE_LETTER.value -> return true
}
}
val index = binarySearchRange(casedStart, this)
return index >= 0 && this <= casedEnd[index]
}
""".trimIndent()
private fun isCaseIgnorable(): String = """
// Mn + Me + Cf + Lm + Sk + Word_Break=MidLetter + Word_Break=MidNumLet + Word_Break=Single_Quote (WordBreakProperty.txt of Unicode Character Database files)
// Declared internal for testing
internal fun Int.isCaseIgnorable(): Boolean {
if (this <= Char.MAX_VALUE.toInt()) {
when (toChar().getCategoryValue()) {
CharCategory.NON_SPACING_MARK.value,
CharCategory.ENCLOSING_MARK.value,
CharCategory.FORMAT.value,
CharCategory.MODIFIER_LETTER.value,
CharCategory.MODIFIER_SYMBOL.value -> return true
}
}
val index = binarySearchRange(caseIgnorableStart, this)
return index >= 0 && this <= caseIgnorableEnd[index]
}
""".trimIndent()
private fun codePointBefore(): String = """
private fun String.codePointBefore(index: Int): Int {
val low = this[index]
if (low.isLowSurrogate() && index - 1 >= 0) {
val high = this[index - 1]
if (high.isHighSurrogate()) {
return Char.toCodePoint(high, low)
}
}
return low.toInt()
}
""".trimIndent()
private fun isFinalSigmaAt(): String = """
// \p{cased} (\p{case-ignorable})* Sigma !( (\p{case-ignorable})* \p{cased} )
// The regular-expression operator * is "possessive", consuming as many characters as possible, with no backup.
// This is significant in the case of Final_Sigma, because the sets of case-ignorable and cased characters are not disjoint.
private fun String.isFinalSigmaAt(index: Int): Boolean {
if (this[index] == '\u03A3' && index > 0) {
var i = index - 1
var codePoint: Int = 0
while (i >= 0) {
codePoint = codePointBefore(i)
if (codePoint.isCaseIgnorable()) {
i -= codePoint.charCount()
} else {
break
}
}
if (i >= 0 && codePoint.isCased()) {
var j = index + 1
while (j < length) {
codePoint = codePointAt(j)
if (codePoint.isCaseIgnorable()) {
j += codePoint.charCount()
} else {
break
}
}
if (j >= length || !codePoint.isCased()) {
return true
}
}
}
return false
}
""".trimIndent()
private fun lowercaseImpl(): String = """
internal fun String.lowercaseImpl(): String {
var unchangedIndex = 0
while (unchangedIndex < this.length) {
val codePoint = codePointAt(unchangedIndex)
if (codePoint.lowercaseCodePoint() != codePoint) { // '\u0130' and '\u03A3' have lowercase corresponding mapping in UnicodeData.txt, no need to check them separately
break
}
unchangedIndex += codePoint.charCount()
}
if (unchangedIndex == this.length) {
return this
}
val sb = StringBuilder(this.length)
sb.appendRange(this, 0, unchangedIndex)
var index = unchangedIndex
while (index < this.length) {
if (this[index] == '\u0130') {
sb.append("\u0069\u0307")
index++
continue
}
if (isFinalSigmaAt(index)) {
sb.append('\u03C2')
index++
continue
}
val codePoint = codePointAt(index)
val lowercaseCodePoint = codePoint.lowercaseCodePoint()
sb.appendCodePoint(lowercaseCodePoint)
index += codePoint.charCount()
}
return sb.toString()
}
""".trimIndent()
}
private class CasedRangesBuilder : RangesBuilder() {
private val id = "Cased"
override fun categoryId(categoryCode: String): String = when (categoryCode) {
CharCategory.UPPERCASE_LETTER.code,
CharCategory.LOWERCASE_LETTER.code,
CharCategory.TITLECASE_LETTER.code -> id
else -> "Else"
}
override fun shouldSkip(categoryId: String): Boolean {
return categoryId != id
}
}
private class CaseIgnorableRangesBuilder : RangesBuilder() {
private val id = "CaseIgnorable"
override fun categoryId(categoryCode: String): String = when (categoryCode) {
CharCategory.NON_SPACING_MARK.code,
CharCategory.ENCLOSING_MARK.code,
CharCategory.FORMAT.code,
CharCategory.MODIFIER_LETTER.code,
CharCategory.MODIFIER_SYMBOL.code -> id
else -> "Else"
}
override fun shouldSkip(categoryId: String): Boolean {
return categoryId != id
}
}
@@ -0,0 +1,104 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package generators.unicode.mappings.string
import generators.unicode.SpecialCasingLine
import generators.unicode.UnicodeDataLine
import generators.unicode.writeHeader
import java.io.File
import java.io.FileWriter
internal class StringUppercaseGenerator(
private val outputFile: File,
unicodeDataLines: List<UnicodeDataLine>
) : StringCasingGenerator(unicodeDataLines) {
override fun SpecialCasingLine.mapping(): List<String> = uppercaseMapping
override fun UnicodeDataLine.mapping(): String = uppercaseMapping
fun generate() {
check(contextDependentMappings.isEmpty()) {
"The locale-agnostic conditional mappings $contextDependentMappings are not handled."
}
FileWriter(outputFile).use { writer ->
writer.writeHeader(outputFile, "kotlin.text")
writer.appendLine()
writer.appendLine(codePointAt())
writer.appendLine()
writer.appendLine(charCount())
writer.appendLine()
writer.appendLine(appendCodePoint())
writer.appendLine()
writer.appendLine(uppercaseImpl())
}
}
private fun charCount(): String = """
internal fun Int.charCount(): Int = if (this >= Char.MIN_SUPPLEMENTARY_CODE_POINT) 2 else 1
""".trimIndent()
private fun codePointAt(): String = """
internal fun String.codePointAt(index: Int): Int {
val high = this[index]
if (high.isHighSurrogate() && index + 1 < this.length) {
val low = this[index + 1]
if (low.isLowSurrogate()) {
return Char.toCodePoint(high, low)
}
}
return high.toInt()
}
""".trimIndent()
private fun appendCodePoint(): String = """
internal fun StringBuilder.appendCodePoint(codePoint: Int) {
if (codePoint < Char.MIN_SUPPLEMENTARY_CODE_POINT) {
append(codePoint.toChar())
} else {
append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10))
append(Char.MIN_LOW_SURROGATE + (codePoint and 0x3ff))
}
}
""".trimIndent()
private fun uppercaseImpl(): String = """
internal fun String.uppercaseImpl(): String {
var unchangedIndex = 0
while (unchangedIndex < this.length) {
val codePoint = codePointAt(unchangedIndex)
if (this[unchangedIndex].oneToManyUppercase() != null || codePoint.uppercaseCodePoint() != codePoint) {
break
}
unchangedIndex += codePoint.charCount()
}
if (unchangedIndex == this.length) {
return this
}
val sb = StringBuilder(this.length)
sb.appendRange(this, 0, unchangedIndex)
var index = unchangedIndex
while (index < this.length) {
val specialCasing = this[index].oneToManyUppercase()
if (specialCasing != null) {
sb.append(specialCasing)
index++
continue
}
val codePoint = codePointAt(index)
val uppercaseCodePoint = codePoint.uppercaseCodePoint()
sb.appendCodePoint(uppercaseCodePoint)
index += codePoint.charCount()
}
return sb.toString()
}
""".trimIndent()
}
@@ -18,9 +18,9 @@ internal sealed class RangesWritingStrategy {
abstract fun rangeRef(name: String): String
companion object {
fun of(target: KotlinTarget, wrapperName: String): RangesWritingStrategy {
fun of(target: KotlinTarget, wrapperName: String? = null): RangesWritingStrategy {
return when (target.platform) {
Platform.JS -> JsRangesWritingStrategy(wrapperName)
Platform.JS -> JsRangesWritingStrategy(wrapperName!!)
else -> NativeRangesWritingStrategy
}
}