[K/N] Experimental Char code point constants and functions

Internalize WASM Char code point functions.

As a part of efforts to stabilize Native stdlib.
This commit is contained in:
Abduqodiri Qurbonzoda
2023-04-19 15:19:11 +03:00
committed by Space Team
parent c6eaadb44f
commit b25f3be81b
18 changed files with 153 additions and 49 deletions
@@ -77,6 +77,7 @@ internal fun Int.isCaseIgnorable(): Boolean {
return index >= 0 && this <= caseIgnorableEnd[index]
}
@OptIn(kotlin.experimental.ExperimentalNativeApi::class)
private fun String.codePointBefore(index: Int): Int {
val low = this[index]
if (low.isLowSurrogate() && index - 1 >= 0) {
@@ -10,6 +10,7 @@ package kotlin.text
// See: https://github.com/JetBrains/kotlin/tree/master/libraries/stdlib
//
@OptIn(kotlin.experimental.ExperimentalNativeApi::class)
internal fun String.codePointAt(index: Int): Int {
val high = this[index]
if (high.isHighSurrogate() && index + 1 < this.length) {
@@ -21,10 +22,10 @@ internal fun String.codePointAt(index: Int): Int {
return high.code
}
internal fun Int.charCount(): Int = if (this >= Char.MIN_SUPPLEMENTARY_CODE_POINT) 2 else 1
internal fun Int.charCount(): Int = if (this > Char.MAX_VALUE.code) 2 else 1
internal fun StringBuilder.appendCodePoint(codePoint: Int) {
if (codePoint < Char.MIN_SUPPLEMENTARY_CODE_POINT) {
if (codePoint <= Char.MAX_VALUE.code) {
append(codePoint.toChar())
} else {
append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10))
@@ -6,6 +6,7 @@
package kotlin
import kotlin.experimental.ExperimentalNativeApi
import kotlin.native.internal.GCUnsafeCall
import kotlin.native.internal.TypedIntrinsic
import kotlin.native.internal.IntrinsicType
@@ -151,18 +152,30 @@ public class Char private constructor() : Comparable<Char> {
public const val MAX_SURROGATE: Char = MAX_LOW_SURROGATE
/**
* The minimum value of a supplementary code point, `\u0x10000`. Kotlin/Native specific.
* The minimum value of a supplementary code point, `\u0x10000`.
*
* Note that this constant is experimental.
* In the future it could be deprecated in favour of another constant of a `CodePoint` type.
*/
@ExperimentalNativeApi
public const val MIN_SUPPLEMENTARY_CODE_POINT: Int = 0x10000
/**
* The minimum value of a Unicode code point. Kotlin/Native specific.
* The minimum value of a Unicode code point.
*
* Note that this constant is experimental.
* In the future it could be deprecated in favour of another constant of a `CodePoint` type.
*/
@ExperimentalNativeApi
public const val MIN_CODE_POINT = 0x000000
/**
* The maximum value of a Unicode code point. Kotlin/Native specific.
* The maximum value of a Unicode code point.
*
* Note that this constant is experimental.
* In the future it could be deprecated in favour of another constant of a `CodePoint` type.
*/
@ExperimentalNativeApi
public const val MAX_CODE_POINT = 0X10FFFF
/**
@@ -5,6 +5,7 @@
package kotlin.text
import kotlin.experimental.ExperimentalNativeApi
import kotlin.native.internal.GCUnsafeCall
/**
@@ -30,6 +31,58 @@ external public actual fun Char.isLowSurrogate(): Boolean
@GCUnsafeCall("Kotlin_Char_isISOControl")
external public actual fun Char.isISOControl(): Boolean
/**
* Converts a surrogate pair to a unicode code point. Doesn't validate that the characters are a valid surrogate pair.
*
* Note that this function is unstable.
* In the future it could be deprecated in favour of an overload that would return a `CodePoint` type.
*/
@ExperimentalNativeApi
// TODO: Consider removing from public API
public fun Char.Companion.toCodePoint(high: Char, low: Char): Int =
(((high - MIN_HIGH_SURROGATE) shl 10) or (low - MIN_LOW_SURROGATE)) + 0x10000
/**
* Checks if the codepoint specified is a supplementary codepoint or not.
*
* Note that this function is unstable.
* In the future it could be deprecated in favour of an overload that would accept a `CodePoint` type.
*/
@ExperimentalNativeApi
// TODO: Consider removing from public API
public fun Char.Companion.isSupplementaryCodePoint(codepoint: Int): Boolean =
codepoint in MIN_SUPPLEMENTARY_CODE_POINT..MAX_CODE_POINT
/**
* Checks if the specified [high] and [low] chars are [Char.isHighSurrogate] and [Char.isLowSurrogate] correspondingly.
*/
@ExperimentalNativeApi
// TODO: Consider removing from public API
public fun Char.Companion.isSurrogatePair(high: Char, low: Char): Boolean = high.isHighSurrogate() && low.isLowSurrogate()
/**
* Converts the codepoint specified to a char array. If the codepoint is not supplementary, the method will
* return an array with one element otherwise it will return an array A with a high surrogate in A[0] and
* a low surrogate in A[1].
*
*
* Note that this function is unstable.
* In the future it could be deprecated in favour of an overload that would accept a `CodePoint` type.
*/
@ExperimentalNativeApi
// TODO: Consider removing from public API
@Suppress("DEPRECATION")
public fun Char.Companion.toChars(codePoint: Int): CharArray =
when {
codePoint in 0 until MIN_SUPPLEMENTARY_CODE_POINT -> charArrayOf(codePoint.toChar())
codePoint in MIN_SUPPLEMENTARY_CODE_POINT..MAX_CODE_POINT -> {
val low = ((codePoint - 0x10000) and 0x3FF) + MIN_LOW_SURROGATE.toInt()
val high = (((codePoint - 0x10000) ushr 10) and 0x3FF) + MIN_HIGH_SURROGATE.toInt()
charArrayOf(high.toChar(), low.toChar())
}
else -> throw IllegalArgumentException()
}
@SharedImmutable
private val digits = intArrayOf(
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
@@ -234,7 +234,7 @@ internal actual fun checkRadix(radix: Int): Int {
// TODO: Make public when supplementary codepoints are supported.
/** Converts a unicode code point to lower case. */
internal fun Char.Companion.toLowerCase(codePoint: Int): Int =
if (codePoint < MIN_SUPPLEMENTARY_CODE_POINT) {
if (codePoint <= MAX_VALUE.code) {
@Suppress("DEPRECATION")
codePoint.toChar().lowercaseChar().toInt()
} else {
@@ -243,40 +243,9 @@ internal fun Char.Companion.toLowerCase(codePoint: Int): Int =
/** Converts a unicode code point to upper case. */
internal fun Char.Companion.toUpperCase(codePoint: Int): Int =
if (codePoint < MIN_SUPPLEMENTARY_CODE_POINT) {
if (codePoint <= MAX_VALUE.code) {
@Suppress("DEPRECATION")
codePoint.toChar().uppercaseChar().toInt()
} else {
codePoint // TODO: Implement this transformation for supplementary codepoints.
}
/** Converts a surrogate pair to a unicode code point. Doesn't validate that the characters are a valid surrogate pair. */
// TODO: Consider removing from public API
public fun Char.Companion.toCodePoint(high: Char, low: Char): Int =
(((high - MIN_HIGH_SURROGATE) shl 10) or (low - MIN_LOW_SURROGATE)) + 0x10000
/** Checks if the codepoint specified is a supplementary codepoint or not. */
// TODO: Consider removing from public API
public fun Char.Companion.isSupplementaryCodePoint(codepoint: Int): Boolean =
codepoint in MIN_SUPPLEMENTARY_CODE_POINT..MAX_CODE_POINT
// TODO: Consider removing from public API
public fun Char.Companion.isSurrogatePair(high: Char, low: Char): Boolean = high.isHighSurrogate() && low.isLowSurrogate()
/**
* Converts the codepoint specified to a char array. If the codepoint is not supplementary, the method will
* return an array with one element otherwise it will return an array A with a high surrogate in A[0] and
* a low surrogate in A[1].
*/
// TODO: Consider removing from public API
@Suppress("DEPRECATION")
public fun Char.Companion.toChars(codePoint: Int): CharArray =
when {
codePoint in 0 until MIN_SUPPLEMENTARY_CODE_POINT -> charArrayOf(codePoint.toChar())
codePoint in MIN_SUPPLEMENTARY_CODE_POINT..MAX_CODE_POINT -> {
val low = ((codePoint - 0x10000) and 0x3FF) + MIN_LOW_SURROGATE.toInt()
val high = (((codePoint - 0x10000) ushr 10) and 0x3FF) + MIN_HIGH_SURROGATE.toInt()
charArrayOf(high.toChar(), low.toChar())
}
else -> throw IllegalArgumentException()
}
@@ -23,6 +23,7 @@
@file:Suppress("DEPRECATION") // Char.toInt()
package kotlin.text.regex
import kotlin.experimental.ExperimentalNativeApi
import kotlin.collections.associate
import kotlin.native.concurrent.AtomicReference
import kotlin.native.concurrent.freeze
@@ -373,6 +374,8 @@ internal abstract class AbstractCharClass : SpecialToken() {
init {
initValues()
}
@OptIn(ExperimentalNativeApi::class)
override fun computeValue(): AbstractCharClass =
object: AbstractCharClass() {
override fun contains(ch: Int): Boolean = alt xor (ch in start..end)
@@ -22,6 +22,7 @@
@file:Suppress("DEPRECATION") // Char.toInt()
package kotlin.text.regex
import kotlin.experimental.ExperimentalNativeApi
import kotlin.native.BitSet
import kotlin.native.ObsoleteNativeApi
@@ -72,6 +73,7 @@ internal class CharClass(val ignoreCase: Boolean = false, negative: Boolean = fa
* We can use this method safely even if nonBitSet != null
* due to specific of range constructions in regular expressions.
*/
@OptIn(ExperimentalNativeApi::class)
fun add(ch: Int): CharClass {
var character = ch
if (ignoreCase) {
@@ -215,6 +217,7 @@ internal class CharClass(val ignoreCase: Boolean = false, negative: Boolean = fa
return this
}
@OptIn(ExperimentalNativeApi::class)
fun add(start: Int, end: Int): CharClass {
if (start > end)
throw IllegalArgumentException("Incorrect range of symbols (start > end)")
@@ -499,7 +502,7 @@ internal class CharClass(val ignoreCase: Boolean = false, negative: Boolean = fa
}
}
@OptIn(ExperimentalStdlibApi::class)
@OptIn(ExperimentalNativeApi::class)
override val instance: AbstractCharClass
get() {
@@ -533,7 +536,7 @@ internal class CharClass(val ignoreCase: Boolean = false, negative: Boolean = fa
}
}
@OptIn(ExperimentalStdlibApi::class)
@OptIn(ExperimentalNativeApi::class)
//for debugging purposes only
override fun toString(): String {
val temp = StringBuilder()
@@ -234,6 +234,7 @@ internal class Lexer(val patternString: String, flags: Int) {
/**
* Returns the next code point in the pattern string.
*/
@OptIn(ExperimentalNativeApi::class)
private fun nextCodePoint(): Int {
val high = pattern[nextIndex()] // nextIndex skips comments and whitespaces if comments flag is on.
if (high.isHighSurrogate()) {
@@ -789,6 +790,7 @@ internal class Lexer(val patternString: String, flags: Int) {
return ch >= 0
}
@OptIn(ExperimentalNativeApi::class)
private fun String.codePointAt(index: Int): Int {
val high = this[index]
if (high.isHighSurrogate() && index + 1 < this.length) {
@@ -841,6 +843,7 @@ internal class Lexer(val patternString: String, flags: Int) {
/**
* Normalize given string.
*/
@OptIn(ExperimentalNativeApi::class)
fun normalize(input: String): String {
val inputChars = input.toCharArray()
val inputLength = inputChars.size
@@ -23,6 +23,8 @@
@file:Suppress("DEPRECATION") // Char.toInt()
package kotlin.text.regex
import kotlin.experimental.ExperimentalNativeApi
/** Represents a compiled pattern used by [Regex] for matching, searching, or replacing strings. */
internal class Pattern(val pattern: String, flags: Int = 0) {
@@ -194,6 +196,7 @@ internal class Pattern(val pattern: String, flags: Int = 0) {
/**
* T->aaa
*/
@OptIn(ExperimentalNativeApi::class)
private fun processSequence(): AbstractSet {
val substring = StringBuilder()
while (!lexemes.isEmpty()
@@ -787,6 +790,7 @@ internal class Pattern(val pattern: String, flags: Int = 0) {
return RangeSet(charClass, hasFlag(CASE_INSENSITIVE))
}
@OptIn(ExperimentalNativeApi::class)
private fun processCharSet(ch: Int): AbstractSet {
val isSupplCodePoint = Char.isSupplementaryCodePoint(ch)
@@ -22,6 +22,8 @@
package kotlin.text.regex
import kotlin.experimental.ExperimentalNativeApi
/** Represents canonical decomposition of Unicode character. Is used when CANON_EQ flag of Pattern class is specified. */
open internal class DecomposedCharSet(
/** Decomposition of the Unicode codepoint */
@@ -34,6 +36,7 @@ open internal class DecomposedCharSet(
private var readCharsForCodePoint = 1
/** UTF-16 encoding of decomposedChar */
@OptIn(ExperimentalNativeApi::class)
private val decomposedCharUTF16: String by lazy {
val strBuff = StringBuilder()
@@ -122,6 +125,7 @@ open internal class DecomposedCharSet(
get() = "decomposed char: $decomposedChar"
/** Reads Unicode codepoint from [testString] starting from [strIndex] until [rightBound]. */
@OptIn(ExperimentalNativeApi::class)
fun codePointAt(strIndex: Int, testString: CharSequence, rightBound: Int): Int {
var index = strIndex
@@ -22,6 +22,8 @@
package kotlin.text.regex
import kotlin.experimental.ExperimentalNativeApi
/**
* Node accepting any character except line terminators.
*/
@@ -34,6 +36,7 @@ internal class DotSet(val lt: AbstractLineTerminator, val matchLineTerminator: B
override val consumesFixedLength: Boolean
get() = true
@OptIn(ExperimentalNativeApi::class)
override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int {
val rightBound = testString.length
if (startIndex >= rightBound) {
@@ -22,6 +22,8 @@
package kotlin.text.regex
import kotlin.experimental.ExperimentalNativeApi
/**
* This class represents nodes constructed with character sequences. For
* example, lets consider regular expression: ".*word.*". During regular
@@ -89,6 +91,7 @@ open internal class SequenceSet(substring: CharSequence, val ignoreCase: Boolean
return -1
}
@OptIn(ExperimentalNativeApi::class)
override fun first(set: AbstractSet): Boolean {
if (ignoreCase) {
return super.first(set)
@@ -87,6 +87,8 @@
package kotlin.text.regex
import kotlin.experimental.ExperimentalNativeApi
/**
* Represents node accepting single character from the given char class.
* This character can be supplementary (2 chars needed to represent) or from
@@ -118,6 +120,7 @@ open internal class SupplementaryRangeSet(charClass: AbstractCharClass, val igno
if (index < rightBound) {
val low = testString[index++]
@OptIn(ExperimentalNativeApi::class)
if (Char.isSurrogatePair(high, low) && contains(Char.toCodePoint(high, low))) {
return next.matches(index, testString, matchResult)
}
@@ -21,10 +21,10 @@ internal fun String.codePointAt(index: Int): Int {
return high.code
}
internal fun Int.charCount(): Int = if (this >= Char.MIN_SUPPLEMENTARY_CODE_POINT) 2 else 1
internal fun Int.charCount(): Int = if (this > Char.MAX_VALUE.code) 2 else 1
internal fun StringBuilder.appendCodePoint(codePoint: Int) {
if (codePoint < Char.MIN_SUPPLEMENTARY_CODE_POINT) {
if (codePoint <= Char.MAX_VALUE.code) {
append(codePoint.toChar())
} else {
append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10))
@@ -27,6 +27,33 @@ public actual fun Char.isHighSurrogate(): Boolean = this in Char.MIN_HIGH_SURROG
*/
public actual fun Char.isLowSurrogate(): Boolean = this in Char.MIN_LOW_SURROGATE..Char.MAX_LOW_SURROGATE
/** Converts a surrogate pair to a unicode code point. Doesn't validate that the characters are a valid surrogate pair. */
internal fun Char.Companion.toCodePoint(high: Char, low: Char): Int =
(((high - MIN_HIGH_SURROGATE) shl 10) or (low - MIN_LOW_SURROGATE)) + 0x10000
/** Checks if the codepoint specified is a supplementary codepoint or not. */
internal fun Char.Companion.isSupplementaryCodePoint(codepoint: Int): Boolean =
codepoint in MIN_SUPPLEMENTARY_CODE_POINT..MAX_CODE_POINT
internal fun Char.Companion.isSurrogatePair(high: Char, low: Char): Boolean = high.isHighSurrogate() && low.isLowSurrogate()
/**
* Converts the codepoint specified to a char array. If the codepoint is not supplementary, the method will
* return an array with one element otherwise it will return an array A with a high surrogate in A[0] and
* a low surrogate in A[1].
*/
@Suppress("DEPRECATION")
internal fun Char.Companion.toChars(codePoint: Int): CharArray =
when {
codePoint in 0 until MIN_SUPPLEMENTARY_CODE_POINT -> charArrayOf(codePoint.toChar())
codePoint in MIN_SUPPLEMENTARY_CODE_POINT..MAX_CODE_POINT -> {
val low = ((codePoint - 0x10000) and 0x3FF) + MIN_LOW_SURROGATE.toInt()
val high = (((codePoint - 0x10000) ushr 10) and 0x3FF) + MIN_HIGH_SURROGATE.toInt()
charArrayOf(high.toChar(), low.toChar())
}
else -> throw IllegalArgumentException()
}
internal actual fun digitOf(char: Char, radix: Int): Int = when {
char >= '0' && char <= '9' -> char - '0'
char >= 'A' && char <= 'Z' -> char - 'A' + 10
@@ -129,7 +129,9 @@ fun main(args: Array<String>) {
addRangesGenerators(nativeGeneratedDir, KotlinTarget.Native)
addOneToOneMappingsGenerators(nativeGeneratedDir, KotlinTarget.Native)
addOneToManyMappingsGenerators(nativeGeneratedDir, KotlinTarget.Native)
stringUppercaseGenerators.add(StringUppercaseGenerator(nativeGeneratedDir.resolve("_StringUppercase.kt"), unicodeDataLines))
stringUppercaseGenerators.add(
StringUppercaseGenerator(nativeGeneratedDir.resolve("_StringUppercase.kt"), unicodeDataLines, KotlinTarget.Native)
)
stringLowercaseGenerators.add(
StringLowercaseGenerator(nativeGeneratedDir.resolve("_StringLowercase.kt"), unicodeDataLines, KotlinTarget.Native)
)
@@ -138,7 +140,9 @@ fun main(args: Array<String>) {
addRangesGenerators(wasmGeneratedDir, KotlinTarget.WASM)
addOneToOneMappingsGenerators(wasmGeneratedDir, KotlinTarget.WASM)
addOneToManyMappingsGenerators(wasmGeneratedDir, KotlinTarget.WASM)
stringUppercaseGenerators.add(StringUppercaseGenerator(wasmGeneratedDir.resolve("_StringUppercase.kt"), unicodeDataLines))
stringUppercaseGenerators.add(
StringUppercaseGenerator(wasmGeneratedDir.resolve("_StringUppercase.kt"), unicodeDataLines, KotlinTarget.WASM)
)
stringLowercaseGenerators.add(
StringLowercaseGenerator(wasmGeneratedDir.resolve("_StringLowercase.kt"), unicodeDataLines, KotlinTarget.WASM)
)
@@ -132,7 +132,7 @@ internal class StringLowercaseGenerator(
}
return low.code
}
""".trimIndent()
""".trimIndent().prependOptInExperimentalNativeApi(target)
private fun isFinalSigmaAt(): String = """
// \p{cased} (\p{case-ignorable})* Sigma !( (\p{case-ignorable})* \p{cased} )
@@ -8,12 +8,14 @@ package generators.unicode.mappings.string
import generators.unicode.SpecialCasingLine
import generators.unicode.UnicodeDataLine
import generators.unicode.writeHeader
import templates.KotlinTarget
import java.io.File
import java.io.FileWriter
internal class StringUppercaseGenerator(
private val outputFile: File,
unicodeDataLines: List<UnicodeDataLine>
unicodeDataLines: List<UnicodeDataLine>,
private val target: KotlinTarget
) : StringCasingGenerator(unicodeDataLines) {
override fun SpecialCasingLine.mapping(): List<String> = uppercaseMapping
@@ -39,7 +41,7 @@ internal class StringUppercaseGenerator(
}
private fun charCount(): String = """
internal fun Int.charCount(): Int = if (this >= Char.MIN_SUPPLEMENTARY_CODE_POINT) 2 else 1
internal fun Int.charCount(): Int = if (this > Char.MAX_VALUE.code) 2 else 1
""".trimIndent()
private fun codePointAt(): String = """
@@ -53,11 +55,11 @@ internal class StringUppercaseGenerator(
}
return high.code
}
""".trimIndent()
""".trimIndent().prependOptInExperimentalNativeApi(target)
private fun appendCodePoint(): String = """
internal fun StringBuilder.appendCodePoint(codePoint: Int) {
if (codePoint < Char.MIN_SUPPLEMENTARY_CODE_POINT) {
if (codePoint <= Char.MAX_VALUE.code) {
append(codePoint.toChar())
} else {
append(Char.MIN_HIGH_SURROGATE + ((codePoint - 0x10000) shr 10))
@@ -101,4 +103,12 @@ internal class StringUppercaseGenerator(
return sb.toString()
}
""".trimIndent()
}
internal fun String.prependOptInExperimentalNativeApi(target: KotlinTarget): String {
return if (target == KotlinTarget.Native) {
"@OptIn(kotlin.experimental.ExperimentalNativeApi::class)\n$this"
} else {
this
}
}