New zero-terminated utf8 to String conversion API
This commit is contained in:
Abduqodiri Qurbonzoda
committed by
GitHub
GitHub
parent
eb1221f465
commit
c348873773
@@ -554,6 +554,60 @@ public fun CPointer<IntVar>.toKStringFromUtf32(): String {
|
||||
return String(chars)
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes a string from the bytes in UTF-8 encoding in this array.
|
||||
* Bytes following the first occurrence of `0` byte, if it occurs, are not decoded.
|
||||
*
|
||||
* Malformed byte sequences are replaced by the replacement char `\uFFFD`.
|
||||
*/
|
||||
@UseExperimental(ExperimentalStdlibApi::class)
|
||||
@SinceKotlin("1.3")
|
||||
public fun ByteArray.toKString() : String {
|
||||
val realEndIndex = realEndIndex(this, 0, this.size)
|
||||
return decodeToString(0, realEndIndex)
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes a string from the bytes in UTF-8 encoding in this array or its subrange.
|
||||
* Bytes following the first occurrence of `0` byte, if it occurs, are not decoded.
|
||||
*
|
||||
* @param startIndex the beginning (inclusive) of the subrange to decode, 0 by default.
|
||||
* @param endIndex the end (exclusive) of the subrange to decode, size of this array by default.
|
||||
* @param throwOnInvalidSequence specifies whether to throw an exception on malformed byte sequence or replace it by the replacement char `\uFFFD`.
|
||||
*
|
||||
* @throws IndexOutOfBoundsException if [startIndex] is less than zero or [endIndex] is greater than the size of this array.
|
||||
* @throws IllegalArgumentException if [startIndex] is greater than [endIndex].
|
||||
* @throws CharacterCodingException if the byte array contains malformed UTF-8 byte sequence and [throwOnInvalidSequence] is true.
|
||||
*/
|
||||
@UseExperimental(ExperimentalStdlibApi::class)
|
||||
@SinceKotlin("1.3")
|
||||
public fun ByteArray.toKString(
|
||||
startIndex: Int = 0,
|
||||
endIndex: Int = this.size,
|
||||
throwOnInvalidSequence: Boolean = false
|
||||
) : String {
|
||||
checkBoundsIndexes(startIndex, endIndex, this.size)
|
||||
val realEndIndex = realEndIndex(this, startIndex, endIndex)
|
||||
return decodeToString(startIndex, realEndIndex, throwOnInvalidSequence)
|
||||
}
|
||||
|
||||
private fun realEndIndex(byteArray: ByteArray, startIndex: Int, endIndex: Int): Int {
|
||||
var index = startIndex
|
||||
while (index < endIndex && byteArray[index] != 0.toByte()) {
|
||||
index++
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
private fun checkBoundsIndexes(startIndex: Int, endIndex: Int, size: Int) {
|
||||
if (startIndex < 0 || endIndex > size) {
|
||||
throw IndexOutOfBoundsException("startIndex: $startIndex, endIndex: $endIndex, size: $size")
|
||||
}
|
||||
if (startIndex > endIndex) {
|
||||
throw IllegalArgumentException("startIndex: $startIndex > endIndex: $endIndex")
|
||||
}
|
||||
}
|
||||
|
||||
public class MemScope : ArenaBase() {
|
||||
|
||||
val memScope: MemScope
|
||||
|
||||
@@ -21,8 +21,7 @@ import kotlin.native.internal.TypedIntrinsic
|
||||
import kotlin.native.internal.IntrinsicType
|
||||
|
||||
internal fun decodeFromUtf8(bytes: ByteArray): String = bytes.decodeToString()
|
||||
|
||||
fun encodeToUtf8(str: String): ByteArray = str.encodeToByteArray()
|
||||
internal fun encodeToUtf8(str: String): ByteArray = str.encodeToByteArray()
|
||||
|
||||
@TypedIntrinsic(IntrinsicType.INTEROP_BITS_TO_FLOAT)
|
||||
external fun bitsToFloat(bits: Int): Float
|
||||
|
||||
@@ -2159,7 +2159,7 @@ task indexof(type: KonanLocalTest) {
|
||||
|
||||
task utf8(type: KonanLocalTest) {
|
||||
expectedFail = (project.testTarget == 'wasm32') // Uses exceptions.
|
||||
goldValue = "Hello\nПривет\n\uD800\uDC00\n\n\uFFFD\uFFFD\n\uFFFD12\n\uFFFD12\n12\uFFFD\n\uD83D\uDE25\n"
|
||||
goldValue = "Hello\nПривет\n\uD800\uDC00\n\n\uFFFD\uFFFD\n\uFFFD12\n\uFFFD12\n12\uFFFD\n\uD83D\uDE25\n\uD83D\uDE25\n\uD83D\uDE25\n"
|
||||
source = "runtime/text/utf8.kt"
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ fun main(args: Array<String>) = memScoped {
|
||||
}.ptr
|
||||
|
||||
if (inflateInit2(z, -15) == Z_OK && inflate(z, Z_FINISH) == Z_STREAM_END && inflateEnd(z) == Z_OK)
|
||||
println(buffer.stringFromUtf8())
|
||||
println(buffer.toKString())
|
||||
}
|
||||
println(golden.toKString())
|
||||
}
|
||||
|
||||
@@ -7,11 +7,13 @@ package runtime.text.utf8
|
||||
|
||||
import kotlin.test.*
|
||||
import kotlin.reflect.KClass
|
||||
import kotlinx.cinterop.toKString
|
||||
|
||||
// region Util
|
||||
// -------------------------------------- Utils --------------------------------------
|
||||
fun assertEquals(expected: ByteArray, actual: ByteArray, message: String) =
|
||||
assertTrue(expected.contentEquals(actual), message)
|
||||
|
||||
|
||||
fun checkUtf16to8(string: String, expected: IntArray, conversion: String.() -> ByteArray) {
|
||||
expected.forEach {
|
||||
assertTrue(it in Byte.MIN_VALUE..Byte.MAX_VALUE, "Expected array contains illegal values")
|
||||
@@ -25,15 +27,35 @@ fun checkUtf16to8(string: String, expected: IntArray, conversion: String.() -> B
|
||||
""".trimIndent())
|
||||
}
|
||||
|
||||
fun checkUtf16to8Replacing(string: String, expected: IntArray) = checkUtf16to8(string, expected) { toUtf8() }
|
||||
fun checkUtf16to8Throwing(string: String, expected: IntArray) = checkUtf16to8(string, expected) { toUtf8OrThrow() }
|
||||
// Utils for checking successful UTF-16 to UTF-8 conversion.
|
||||
fun checkUtf16to8Replacing(string: String, expected: IntArray) {
|
||||
checkUtf16to8(string, expected) { toUtf8() }
|
||||
checkUtf16to8(string, expected) { encodeToByteArray() }
|
||||
}
|
||||
fun checkUtf16to8Throwing(string: String, expected: IntArray) {
|
||||
checkUtf16to8(string, expected) { toUtf8OrThrow() }
|
||||
checkUtf16to8(string, expected) { encodeToByteArray(throwOnInvalidSequence = true) }
|
||||
}
|
||||
fun checkValidUtf16to8(string: String, expected: IntArray) {
|
||||
checkUtf16to8Replacing(string, expected)
|
||||
checkUtf16to8Throwing(string, expected)
|
||||
}
|
||||
|
||||
fun checkUtf16to8Replacing(string: String, expected: IntArray, start: Int, size: Int) =
|
||||
checkUtf16to8(string, expected) { toUtf8(start, size) }
|
||||
fun checkUtf16to8Replacing(string: String, expected: IntArray, start: Int, size: Int) {
|
||||
checkUtf16to8(string, expected) { toUtf8(start, size) }
|
||||
checkUtf16to8(string, expected) { encodeToByteArray(start, start + size) }
|
||||
}
|
||||
fun checkUtf16to8Throwing(string: String, expected: IntArray, start: Int, size: Int) {
|
||||
checkUtf16to8(string, expected) { toUtf8OrThrow(start, size) }
|
||||
checkUtf16to8(string, expected) { encodeToByteArray(start, start + size, true) }
|
||||
}
|
||||
fun checkValidUtf16to8(string: String, expected: IntArray, start: Int, size: Int) {
|
||||
checkUtf16to8Replacing(string, expected, start, size)
|
||||
checkUtf16to8Throwing(string, expected, start, size)
|
||||
}
|
||||
|
||||
fun checkUtf16to8Throwing(string: String, expected: IntArray, start: Int, size: Int) =
|
||||
checkUtf16to8(string, expected) { toUtf8OrThrow(start, size) }
|
||||
|
||||
// Utils for checking successful UTF-8 to UTF-16 conversion.
|
||||
fun checkUtf8to16(expected: String, array: IntArray, conversion: ByteArray.() -> String) {
|
||||
array.forEach {
|
||||
assertTrue(it in Byte.MIN_VALUE..Byte.MAX_VALUE, "Expected array contains illegal values")
|
||||
@@ -47,14 +69,62 @@ fun checkUtf8to16(expected: String, array: IntArray, conversion: ByteArray.() ->
|
||||
""".trimIndent())
|
||||
}
|
||||
|
||||
fun checkUtf8to16Replacing(expected: String, array: IntArray) = checkUtf8to16(expected, array) { stringFromUtf8() }
|
||||
fun checkUtf8to16Throwing(expected: String, array: IntArray) = checkUtf8to16(expected, array) { stringFromUtf8OrThrow() }
|
||||
fun checkZeroTerminatedUtf8to16Replacing(expected: String, array: IntArray) {
|
||||
checkUtf8to16(expected, array) { stringFromUtf8() }
|
||||
checkUtf8to16(expected, array) { toKString() }
|
||||
}
|
||||
fun checkUtf8to16Replacing(expected: String, array: IntArray) {
|
||||
checkUtf8to16(expected, array) { stringFromUtf8() }
|
||||
checkZeroTerminatedUtf8to16Replacing(expected, array)
|
||||
checkZeroTerminatedUtf8to16Replacing(expected, array.copyOf(array.size + 1))
|
||||
}
|
||||
fun checkZeroTerminatedUtf8to16Throwing(expected: String, array: IntArray) {
|
||||
checkUtf8to16(expected, array) { stringFromUtf8OrThrow() }
|
||||
checkUtf8to16(expected, array) { toKString(throwOnInvalidSequence = true) }
|
||||
}
|
||||
fun checkUtf8to16Throwing(expected: String, array: IntArray) {
|
||||
checkUtf8to16(expected, array) { decodeToString(throwOnInvalidSequence = true) }
|
||||
checkZeroTerminatedUtf8to16Throwing(expected, array)
|
||||
checkZeroTerminatedUtf8to16Throwing(expected, array.copyOf(array.size + 1))
|
||||
}
|
||||
fun checkValidUtf8to16(expected: String, array: IntArray) {
|
||||
checkUtf8to16Replacing(expected, array)
|
||||
checkUtf8to16Throwing(expected, array)
|
||||
}
|
||||
fun checkValidZeroTerminatedUtf8to16(expected: String, array: IntArray) {
|
||||
checkZeroTerminatedUtf8to16Replacing(expected, array)
|
||||
checkZeroTerminatedUtf8to16Throwing(expected, array)
|
||||
}
|
||||
|
||||
fun checkUtf8to16Replacing(expected: String, array: IntArray, start: Int, size: Int) =
|
||||
checkUtf8to16(expected, array) { stringFromUtf8(start, size) }
|
||||
fun checkUtf8to16Throwing(expected: String, array: IntArray, start: Int, size: Int) =
|
||||
checkUtf8to16(expected, array) { stringFromUtf8OrThrow(start, size) }
|
||||
fun checkZeroTerminatedUtf8to16Replacing(expected: String, array: IntArray, start: Int, size: Int) {
|
||||
checkUtf8to16(expected, array) { stringFromUtf8(start, size) }
|
||||
checkUtf8to16(expected, array) { toKString(start, start + size) }
|
||||
}
|
||||
fun checkUtf8to16Replacing(expected: String, array: IntArray, start: Int, size: Int) {
|
||||
checkUtf8to16(expected, array) { decodeToString(start, start + size) }
|
||||
checkZeroTerminatedUtf8to16Replacing(expected, array, start, size)
|
||||
checkZeroTerminatedUtf8to16Replacing(expected, array.copyOf(array.size + 1), start, size)
|
||||
}
|
||||
fun checkZeroTerminatedUtf8to16Throwing(expected: String, array: IntArray, start: Int, size: Int) {
|
||||
checkUtf8to16(expected, array) { stringFromUtf8OrThrow(start, size) }
|
||||
checkUtf8to16(expected, array) { toKString(start, start + size, true) }
|
||||
}
|
||||
fun checkUtf8to16Throwing(expected: String, array: IntArray, start: Int, size: Int) {
|
||||
checkUtf8to16(expected, array) { decodeToString(start, start + size, true) }
|
||||
checkZeroTerminatedUtf8to16Throwing(expected, array, start, size)
|
||||
checkZeroTerminatedUtf8to16Throwing(expected, array.copyOf(array.size + 1), start, size)
|
||||
}
|
||||
fun checkValidUtf8to16(expected: String, array: IntArray, start: Int, size: Int) {
|
||||
checkUtf8to16Replacing(expected, array, start, size)
|
||||
checkUtf8to16Throwing(expected, array, start, size)
|
||||
}
|
||||
fun checkValidZeroTerminatedUtf8to16(expected: String, array: IntArray, start: Int, size: Int) {
|
||||
checkZeroTerminatedUtf8to16Replacing(expected, array, start, size)
|
||||
checkZeroTerminatedUtf8to16Throwing(expected, array, start, size)
|
||||
}
|
||||
|
||||
|
||||
// Utils for checking malformed UTF-16 to UTF-8 conversion.
|
||||
fun <T: Any> checkThrows(e: KClass<T>, string: String, action: () -> Unit) {
|
||||
var exception: Throwable? = null
|
||||
try {
|
||||
@@ -62,7 +132,7 @@ fun <T: Any> checkThrows(e: KClass<T>, string: String, action: () -> Unit) {
|
||||
} catch (e: Throwable) {
|
||||
exception = e
|
||||
}
|
||||
assertNotNull(exception, "No excpetion was thrown for string: $string")
|
||||
assertNotNull(exception, "No exception was thrown for string: $string")
|
||||
assertTrue(e.isInstance(exception),"""
|
||||
Wrong exception was thrown for string: $string
|
||||
Expected: ${e.qualifiedName}
|
||||
@@ -70,51 +140,114 @@ fun <T: Any> checkThrows(e: KClass<T>, string: String, action: () -> Unit) {
|
||||
""".trimIndent())
|
||||
}
|
||||
|
||||
fun checkUtf16to8Throws(string: String) = checkThrows(IllegalCharacterConversionException::class, string) {
|
||||
string.toUtf8OrThrow()
|
||||
fun checkUtf16to8Throws(string: String) {
|
||||
checkThrows(IllegalCharacterConversionException::class, string) { string.toUtf8OrThrow() }
|
||||
checkThrows(CharacterCodingException::class, string) { string.encodeToByteArray(throwOnInvalidSequence = true) }
|
||||
}
|
||||
fun checkUtf16to8Throws(string: String, start: Int, size: Int) {
|
||||
checkThrows(IllegalCharacterConversionException::class, string) { string.toUtf8OrThrow(start, size) }
|
||||
checkThrows(CharacterCodingException::class, string) { string.encodeToByteArray(start, start + size, true) }
|
||||
}
|
||||
|
||||
fun checkUtf16to8Throws(string: String, start: Int, size: Int) =
|
||||
checkThrows(IllegalCharacterConversionException::class, string) {
|
||||
string.toUtf8OrThrow(start, size)
|
||||
}
|
||||
|
||||
fun checkUtf8to16Throws(string: String, array: IntArray)
|
||||
= checkThrows(IllegalCharacterConversionException::class, string) {
|
||||
// Utils for checking malformed UTF-8 to UTF-16 conversion.
|
||||
fun <T: Any> checkUtf8to16Throws(e: KClass<T>, string: String, array: IntArray, conversion: ByteArray.() -> String)
|
||||
= checkThrows(e, string) {
|
||||
array.forEach {
|
||||
assertTrue(it in Byte.MIN_VALUE..Byte.MAX_VALUE, "Expected array contains illegal values")
|
||||
}
|
||||
val arrayBytes = ByteArray(array.size) { i -> array[i].toByte() }
|
||||
arrayBytes.stringFromUtf8OrThrow()
|
||||
arrayBytes.conversion()
|
||||
}
|
||||
|
||||
fun checkUtf8to16Throws(string: String, array: IntArray, start: Int, size: Int)
|
||||
= checkThrows(IllegalCharacterConversionException::class, string) {
|
||||
array.forEach {
|
||||
assertTrue(it in Byte.MIN_VALUE..Byte.MAX_VALUE, "Expected array contains illegal values")
|
||||
}
|
||||
val arrayBytes = ByteArray(array.size) { i -> array[i].toByte() }
|
||||
arrayBytes.stringFromUtf8OrThrow(start, size)
|
||||
fun checkZeroTerminatedUtf8to16Throws(string: String, array: IntArray) {
|
||||
checkUtf8to16Throws(IllegalCharacterConversionException::class, string, array) { stringFromUtf8OrThrow() }
|
||||
checkUtf8to16Throws(CharacterCodingException::class, string, array) { toKString(throwOnInvalidSequence = true) }
|
||||
}
|
||||
fun checkUtf8to16Throws(string: String, array: IntArray) {
|
||||
checkUtf8to16Throws(CharacterCodingException::class, string, array) { decodeToString(throwOnInvalidSequence = true) }
|
||||
checkZeroTerminatedUtf8to16Throws(string, array)
|
||||
checkZeroTerminatedUtf8to16Throws(string, array.copyOf(array.size + 1))
|
||||
}
|
||||
fun checkZeroTerminatedUtf8to16Throws(string: String, array: IntArray, start: Int, size: Int) {
|
||||
checkUtf8to16Throws(IllegalCharacterConversionException::class, string, array) { stringFromUtf8OrThrow(start, size) }
|
||||
checkUtf8to16Throws(CharacterCodingException::class, string, array) { toKString(start, start + size, true) }
|
||||
}
|
||||
fun checkUtf8to16Throws(string: String, array: IntArray, start: Int, size: Int) {
|
||||
checkUtf8to16Throws(CharacterCodingException::class, string, array) { decodeToString(start, start + size, true) }
|
||||
checkZeroTerminatedUtf8to16Throws(string, array, start, size)
|
||||
checkZeroTerminatedUtf8to16Throws(string, array.copyOf(array.size + 1), start, size)
|
||||
}
|
||||
|
||||
|
||||
fun checkDobuleConversionThrows(string: String) = checkThrows(NumberFormatException::class, string) {
|
||||
// Utils for checking String to floating-point number conversion.
|
||||
fun checkDoubleConversionThrows(string: String) = checkThrows(NumberFormatException::class, string) {
|
||||
string.toDouble()
|
||||
}
|
||||
|
||||
fun checkFloatConversionThrows(string: String) = checkThrows(NumberFormatException::class, string) {
|
||||
string.toFloat()
|
||||
}
|
||||
// endregion
|
||||
|
||||
// region UTF-16 -> UTF-8 conversion
|
||||
|
||||
// Utils for checking invalid-bounds-exception thrown by UTF-16 to UTF-8 conversion.
|
||||
fun <T: Any> checkOutOfBoundsUtf16to8Replacing(e: KClass<T>, string: String, start: Int, size: Int) {
|
||||
checkThrows(e, string) { string.toUtf8(start, size) }
|
||||
checkThrows(e, string) { string.encodeToByteArray(start, start + size) }
|
||||
}
|
||||
fun <T: Any> checkOutOfBoundsUtf16to8Throwing(e: KClass<T>, string: String, start: Int, size: Int) {
|
||||
checkThrows(e, string) { string.toUtf8OrThrow(start, size) }
|
||||
checkThrows(e, string) { string.encodeToByteArray(start, start + size, true) }
|
||||
}
|
||||
fun <T: Any> checkOutOfBoundsUtf16to8(e: KClass<T>, string: String, start: Int, size: Int) {
|
||||
checkOutOfBoundsUtf16to8Replacing(e, string, start, size)
|
||||
checkOutOfBoundsUtf16to8Throwing(e, string, start, size)
|
||||
}
|
||||
|
||||
|
||||
// Utils for checking invalid-bounds-exception thrown by UTF-8 to UTF-16 conversion.
|
||||
fun <T: Any> checkOutOfBoundsZeroTerminatedUtf8to16Replacing(e: KClass<T>, string: String, byteArray: ByteArray, start: Int, size: Int) {
|
||||
checkThrows(e, string) { byteArray.stringFromUtf8(start, size) }
|
||||
checkThrows(e, string) { byteArray.toKString(start, start + size) }
|
||||
}
|
||||
fun <T: Any> checkOutOfBoundsUtf8to16Replacing(e: KClass<T>, string: String, byteArray: ByteArray, start: Int, size: Int) {
|
||||
checkThrows(e, string) { byteArray.decodeToString(start, start + size) }
|
||||
checkOutOfBoundsZeroTerminatedUtf8to16Replacing(e, string, byteArray, start, size)
|
||||
}
|
||||
fun <T: Any> checkOutOfBoundsZeroTerminatedUtf8to16Throwing(e: KClass<T>, string: String, byteArray: ByteArray, start: Int, size: Int) {
|
||||
checkThrows(e, string) { byteArray.stringFromUtf8OrThrow(start, size) }
|
||||
checkThrows(e, string) { byteArray.toKString(start, start + size, true) }
|
||||
}
|
||||
fun <T: Any> checkOutOfBoundsUtf8to16Throwing(e: KClass<T>, string: String, byteArray: ByteArray, start: Int, size: Int) {
|
||||
checkThrows(e, string) { byteArray.decodeToString(start, start + size, true) }
|
||||
checkOutOfBoundsZeroTerminatedUtf8to16Throwing(e, string, byteArray, start, size)
|
||||
}
|
||||
fun <T: Any> checkOutOfBoundsUtf8to16(e: KClass<T>, string: String, byteArray: ByteArray, start: Int, size: Int) {
|
||||
checkOutOfBoundsUtf8to16Replacing(e, string, byteArray, start, size)
|
||||
checkOutOfBoundsUtf8to16Throwing(e, string, byteArray, start, size)
|
||||
}
|
||||
fun <T: Any> checkOutOfBoundsZeroTerminatedUtf8to16(e: KClass<T>, string: String, byteArray: ByteArray, start: Int, size: Int) {
|
||||
checkOutOfBoundsZeroTerminatedUtf8to16Replacing(e, string, byteArray, start, size)
|
||||
checkOutOfBoundsZeroTerminatedUtf8to16Throwing(e, string, byteArray, start, size)
|
||||
}
|
||||
|
||||
|
||||
// Util for performing action on result of UTF-8 to UTF-16 conversion.
|
||||
fun convertUtf8to16(byteArray: ByteArray, action: (String) -> Unit) {
|
||||
byteArray.stringFromUtf8().let { action(it) }
|
||||
byteArray.decodeToString().let { action(it) }
|
||||
byteArray.toKString().let { action(it) }
|
||||
}
|
||||
|
||||
// ------------------------- Test UTF-16 to UTF-8 conversion -------------------------
|
||||
fun test16to8() {
|
||||
// Test manual conversion with replacement.
|
||||
// Valid strings.
|
||||
checkUtf16to8Replacing("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()))
|
||||
checkUtf16to8Replacing("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126))
|
||||
checkUtf16to8Replacing("\uD800\uDC00", intArrayOf(-16, -112, -128, -128))
|
||||
checkUtf16to8Replacing("", intArrayOf())
|
||||
checkValidUtf16to8("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()))
|
||||
checkValidUtf16to8("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126))
|
||||
checkValidUtf16to8("\uD800\uDC00", intArrayOf(-16, -112, -128, -128))
|
||||
checkValidUtf16to8("", intArrayOf())
|
||||
|
||||
// Test manual conversion with replacement.
|
||||
// Illegal surrogate pair -> replace with default
|
||||
checkUtf16to8Replacing("\uDC00\uD800", intArrayOf(-17, -65, -67, -17, -65, -67))
|
||||
// Different kinds of input
|
||||
@@ -125,14 +258,7 @@ fun test16to8() {
|
||||
checkUtf16to8Replacing("\uDC0012", intArrayOf(-17, -65, -67, '1'.toInt(), '2'.toInt()))
|
||||
checkUtf16to8Replacing("12\uD800", intArrayOf('1'.toInt(), '2'.toInt(), -17, -65, -67))
|
||||
|
||||
|
||||
// Test manual conversion with an exception if an input is invalid.
|
||||
// Valid strings.
|
||||
checkUtf16to8Throwing("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()))
|
||||
checkUtf16to8Throwing("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126))
|
||||
checkUtf16to8Throwing("\uD800\uDC00", intArrayOf(-16, -112, -128, -128))
|
||||
checkUtf16to8Replacing("", intArrayOf())
|
||||
|
||||
// Illegal surrogate pair -> throw
|
||||
checkUtf16to8Throws("\uDC00\uD800")
|
||||
// Different kinds of input (including illegal one) -> throw
|
||||
@@ -145,13 +271,13 @@ fun test16to8() {
|
||||
// Test double parsing.
|
||||
assertEquals(4.2, "4.2".toDouble())
|
||||
// Illegal surrogate pair -> throw
|
||||
checkDobuleConversionThrows("\uDC00\uD800")
|
||||
checkDoubleConversionThrows("\uDC00\uD800")
|
||||
// Different kinds of input (including illegal one) -> throw
|
||||
checkDobuleConversionThrows("\uD800\uDC001\uDC00\uD800")
|
||||
checkDoubleConversionThrows("\uD800\uDC001\uDC00\uD800")
|
||||
// Lone surrogate - throw
|
||||
checkDobuleConversionThrows("\uD80012")
|
||||
checkDobuleConversionThrows("\uDC0012")
|
||||
checkDobuleConversionThrows("12\uD800")
|
||||
checkDoubleConversionThrows("\uD80012")
|
||||
checkDoubleConversionThrows("\uDC0012")
|
||||
checkDoubleConversionThrows("12\uD800")
|
||||
|
||||
// Test float parsing.
|
||||
assertEquals(4.2F, "4.2".toFloat())
|
||||
@@ -166,21 +292,27 @@ fun test16to8() {
|
||||
}
|
||||
|
||||
fun test16to8CustomBorders() {
|
||||
// Test manual conversion with replacement and custom borders.
|
||||
// Valid strings.
|
||||
checkUtf16to8Replacing("Hello!", intArrayOf('H'.toInt(), 'e'.toInt()), 0, 2)
|
||||
checkUtf16to8Replacing("Hello!", intArrayOf('e'.toInt(), 'l'.toInt()), 1, 2)
|
||||
checkUtf16to8Replacing("Hello!", intArrayOf('o'.toInt(), '!'.toInt()), 4, 2)
|
||||
checkUtf16to8Replacing("Hello!", intArrayOf(), 0, 0)
|
||||
checkUtf16to8Replacing("Hello!", intArrayOf(), 6, 0)
|
||||
checkValidUtf16to8("Hello!", intArrayOf('H'.toInt(), 'e'.toInt()), 0, 2)
|
||||
checkValidUtf16to8("Hello!", intArrayOf('e'.toInt(), 'l'.toInt()), 1, 2)
|
||||
checkValidUtf16to8("Hello!", intArrayOf('o'.toInt(), '!'.toInt()), 4, 2)
|
||||
checkValidUtf16to8("Hello!", intArrayOf(), 0, 0)
|
||||
checkValidUtf16to8("Hello!", intArrayOf(), 6, 0)
|
||||
|
||||
checkUtf16to8Replacing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00",
|
||||
checkValidUtf16to8("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00",
|
||||
intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 0, 4)
|
||||
checkUtf16to8Replacing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00",
|
||||
checkValidUtf16to8("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00",
|
||||
intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 2, 4)
|
||||
checkUtf16to8Replacing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00",
|
||||
checkValidUtf16to8("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00",
|
||||
intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 4, 4)
|
||||
|
||||
// Index out of bound
|
||||
checkOutOfBoundsUtf16to8(IndexOutOfBoundsException::class, "Hello", -1, 4)
|
||||
checkOutOfBoundsUtf16to8(IndexOutOfBoundsException::class, "Hello", 5, 10)
|
||||
checkOutOfBoundsUtf16to8(IndexOutOfBoundsException::class, "Hello", 2, 10)
|
||||
checkOutOfBoundsUtf16to8(IllegalArgumentException::class, "Hello", 3, -2)
|
||||
|
||||
// Test manual conversion with replacement and custom borders.
|
||||
// Illegal surrogate pair -> replace with default
|
||||
checkUtf16to8Replacing("\uDC00\uD80012",
|
||||
intArrayOf(-17, -65, -67, -17, -65, -67, '1'.toInt()), 0, 3)
|
||||
@@ -193,28 +325,7 @@ fun test16to8CustomBorders() {
|
||||
checkUtf16to8Replacing("1\uD800\uDC002", intArrayOf('1'.toInt(), -17, -65, -67), 0, 2)
|
||||
checkUtf16to8Replacing("1\uD800\uDC002", intArrayOf(-17, -65, -67, '2'.toInt()), 2, 2)
|
||||
|
||||
// Index out of bound
|
||||
checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8(-1, 4)}
|
||||
checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8(5, 10)}
|
||||
checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8(2, 10)}
|
||||
checkThrows(IllegalArgumentException::class, "Hello") { "Hello".toUtf8(3, -2)}
|
||||
|
||||
|
||||
// Test manual conversion with an exception if an input is invalid and custom borders.
|
||||
// Valid strings.
|
||||
checkUtf16to8Throwing("Hello!", intArrayOf('H'.toInt(), 'e'.toInt()), 0, 2)
|
||||
checkUtf16to8Throwing("Hello!", intArrayOf('e'.toInt(), 'l'.toInt()), 1, 2)
|
||||
checkUtf16to8Throwing("Hello!", intArrayOf('o'.toInt(), '!'.toInt()), 4, 2)
|
||||
checkUtf16to8Throwing("Hello!", intArrayOf(), 0, 0)
|
||||
checkUtf16to8Throwing("Hello!", intArrayOf(), 6, 0)
|
||||
|
||||
checkUtf16to8Throwing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00",
|
||||
intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 0, 4)
|
||||
checkUtf16to8Throwing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00",
|
||||
intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 2, 4)
|
||||
checkUtf16to8Throwing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00",
|
||||
intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 4, 4)
|
||||
|
||||
// Illegal surrogate pair -> throw
|
||||
checkUtf16to8Throws("\uDC00\uD80012", 0, 3)
|
||||
checkUtf16to8Throws("1\uDC00\uD8002", 1, 3)
|
||||
@@ -223,12 +334,6 @@ fun test16to8CustomBorders() {
|
||||
// Lone surrogate -> throw
|
||||
checkUtf16to8Throws("1\uD800\uDC002", 0, 2)
|
||||
checkUtf16to8Throws("1\uD800\uDC002", 2, 2)
|
||||
|
||||
// Index out of bound
|
||||
checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8OrThrow(-1, 4)}
|
||||
checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8OrThrow(5, 10)}
|
||||
checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8OrThrow(2, 10)}
|
||||
checkThrows(IllegalArgumentException::class, "Hello") { "Hello".toUtf8OrThrow(3, -2)}
|
||||
}
|
||||
|
||||
fun testPrint() {
|
||||
@@ -247,21 +352,21 @@ fun testPrint() {
|
||||
|
||||
// https://github.com/JetBrains/kotlin-native/issues/1091
|
||||
val array = byteArrayOf(0xF0.toByte(), 0x9F.toByte(), 0x98.toByte(), 0xA5.toByte())
|
||||
val badStr = array.stringFromUtf8()
|
||||
assertEquals(2, badStr.length)
|
||||
println(badStr)
|
||||
convertUtf8to16(array) { badString ->
|
||||
assertEquals(2, badString.length)
|
||||
println(badString)
|
||||
}
|
||||
}
|
||||
// endregion
|
||||
|
||||
// region UTF-8 -> UTF-16 conversion
|
||||
// ------------------------- Test UTF-8 to UTF-16 conversion -------------------------
|
||||
fun test8to16() {
|
||||
// Test manual conversion with replacement.
|
||||
// Valid strings.
|
||||
checkUtf8to16Replacing("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()))
|
||||
checkUtf8to16Replacing("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126))
|
||||
checkUtf8to16Replacing("\uD800\uDC00", intArrayOf(-16, -112, -128, -128))
|
||||
checkUtf8to16Replacing("", intArrayOf())
|
||||
checkValidUtf8to16("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()))
|
||||
checkValidUtf8to16("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126))
|
||||
checkValidUtf8to16("\uD800\uDC00", intArrayOf(-16, -112, -128, -128))
|
||||
checkValidUtf8to16("", intArrayOf())
|
||||
|
||||
// Test manual conversion with replacement.
|
||||
// Incorrect UTF-8 lead character.
|
||||
checkUtf8to16Replacing("\uFFFD1", intArrayOf(-1, '1'.toInt()))
|
||||
|
||||
@@ -270,12 +375,6 @@ fun test8to16() {
|
||||
checkUtf8to16Replacing("\uFFFD1\uFFFD", intArrayOf(-16, -97, -104, '1'.toInt(), -16, -97, -104))
|
||||
|
||||
// Test manual conversion with exception throwing
|
||||
// Valid strings.
|
||||
checkUtf8to16Throwing("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()))
|
||||
checkUtf8to16Throwing("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126))
|
||||
checkUtf8to16Throwing("\uD800\uDC00", intArrayOf(-16, -112, -128, -128))
|
||||
checkUtf8to16Throwing("", intArrayOf())
|
||||
|
||||
// Incorrect UTF-8 lead character -> throw.
|
||||
checkUtf8to16Throws("\uFFFD1", intArrayOf(-1, '1'.toInt()))
|
||||
|
||||
@@ -285,26 +384,35 @@ fun test8to16() {
|
||||
}
|
||||
|
||||
fun test8to16CustomBorders() {
|
||||
// Conversion with replacement.
|
||||
checkUtf8to16Replacing("He",
|
||||
// Valid strings.
|
||||
checkValidUtf8to16("He",
|
||||
intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()),0, 2)
|
||||
checkUtf8to16Replacing("ll",
|
||||
checkValidUtf8to16("ll",
|
||||
intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 2, 2)
|
||||
checkUtf8to16Replacing("lo",
|
||||
checkValidUtf8to16("lo",
|
||||
intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 3, 2)
|
||||
checkUtf8to16Replacing("",
|
||||
checkValidUtf8to16("",
|
||||
intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 0, 0)
|
||||
|
||||
checkUtf8to16Replacing("\uD800\uDC00\uD800\uDC00",
|
||||
checkValidUtf8to16("\uD800\uDC00\uD800\uDC00",
|
||||
intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128),
|
||||
0, 8)
|
||||
checkUtf8to16Replacing("\uD800\uDC00\uD800\uDC00",
|
||||
checkValidUtf8to16("\uD800\uDC00\uD800\uDC00",
|
||||
intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128),
|
||||
4, 8)
|
||||
checkUtf8to16Replacing("\uD800\uDC00\uD800\uDC00",
|
||||
checkValidUtf8to16("\uD800\uDC00\uD800\uDC00",
|
||||
intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128),
|
||||
8, 8)
|
||||
|
||||
// Index out of bound
|
||||
val helloArray = byteArrayOf('H'.toByte(), 'e'.toByte(), 'l'.toByte(), 'l'.toByte(), 'o'.toByte())
|
||||
checkOutOfBoundsUtf8to16(IndexOutOfBoundsException::class, "Hello", helloArray, -1, 4)
|
||||
checkOutOfBoundsUtf8to16(IndexOutOfBoundsException::class, "Hello", helloArray, 5, 10)
|
||||
checkOutOfBoundsUtf8to16(IndexOutOfBoundsException::class, "Hello", helloArray, 2, 10)
|
||||
checkOutOfBoundsUtf8to16(IndexOutOfBoundsException::class, "Hello", helloArray, 10, 0)
|
||||
checkOutOfBoundsUtf8to16(IllegalArgumentException::class, "Hello", helloArray, 3, -2)
|
||||
|
||||
// Test manual conversion with replacement and custom borders.
|
||||
// Incorrect UTF-8 lead character.
|
||||
checkUtf8to16Replacing("\uFFFD1", intArrayOf(-1, '1'.toInt(), '2'.toInt()), 0, 2)
|
||||
checkUtf8to16Replacing("\uFFFD2", intArrayOf('1'.toInt(), -1, '2'.toInt(), '3'.toInt()), 1, 2)
|
||||
@@ -315,35 +423,7 @@ fun test8to16CustomBorders() {
|
||||
checkUtf8to16Replacing("\uFFFD2", intArrayOf('1'.toInt(), -16, -97, -104, '2'.toInt(), '3'.toInt()), 1, 4)
|
||||
checkUtf8to16Replacing("2\uFFFD", intArrayOf('1'.toInt(), '2'.toInt(), -16, -97, -104), 1, 4)
|
||||
|
||||
|
||||
// Index out of bound
|
||||
val helloArray = byteArrayOf('H'.toByte(), 'e'.toByte(), 'l'.toByte(), 'l'.toByte(), 'o'.toByte())
|
||||
checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8(-1, 4)}
|
||||
checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8(5, 10)}
|
||||
checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8(2, 10)}
|
||||
checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8(10, 0)}
|
||||
checkThrows(IllegalArgumentException::class, "Hello") { helloArray.stringFromUtf8(3, -2)}
|
||||
|
||||
// Conversion with throwing
|
||||
checkUtf8to16Throwing("He",
|
||||
intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()),0, 2)
|
||||
checkUtf8to16Throwing("ll",
|
||||
intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 2, 2)
|
||||
checkUtf8to16Throwing("lo",
|
||||
intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 3, 2)
|
||||
checkUtf8to16Throwing("",
|
||||
intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 0, 0)
|
||||
|
||||
checkUtf8to16Throwing("\uD800\uDC00\uD800\uDC00",
|
||||
intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128),
|
||||
0, 8)
|
||||
checkUtf8to16Throwing("\uD800\uDC00\uD800\uDC00",
|
||||
intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128),
|
||||
4, 8)
|
||||
checkUtf8to16Throwing("\uD800\uDC00\uD800\uDC00",
|
||||
intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128),
|
||||
8, 8)
|
||||
|
||||
// Test manual conversion with an exception if an input is invalid and custom borders.
|
||||
// Incorrect UTF-8 lead character.
|
||||
checkUtf8to16Throws("\uFFFD1", intArrayOf(-1, '1'.toInt(), '2'.toInt()), 0, 2)
|
||||
checkUtf8to16Throws("\uFFFD2", intArrayOf('1'.toInt(), -1, '2'.toInt(), '3'.toInt()), 1, 2)
|
||||
@@ -353,20 +433,61 @@ fun test8to16CustomBorders() {
|
||||
checkUtf8to16Throws("\uFFFD1", intArrayOf(-16, -97, -104, '1'.toInt(), '2'.toInt()), 0, 4)
|
||||
checkUtf8to16Throws("\uFFFD2", intArrayOf('1'.toInt(), -16, -97, -104, '2'.toInt(), '3'.toInt()), 1, 4)
|
||||
checkUtf8to16Throws("2\uFFFD", intArrayOf('1'.toInt(), '2'.toInt(), -16, -97, -104), 1, 4)
|
||||
|
||||
// Index out of bound
|
||||
checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8OrThrow(-1, 4)}
|
||||
checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8OrThrow(5, 10)}
|
||||
checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8OrThrow(2, 10)}
|
||||
checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8OrThrow(10, 0)}
|
||||
checkThrows(IllegalArgumentException::class, "Hello") { helloArray.stringFromUtf8OrThrow(3, -2)}
|
||||
}
|
||||
// endregion
|
||||
|
||||
// ----------------- Test zero-terminated UTF-8 to UTF-16 conversion -----------------
|
||||
fun testZeroTerminated8To16() {
|
||||
// Valid strings.
|
||||
checkValidZeroTerminatedUtf8to16("Hell", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 0, 'o'.toInt()))
|
||||
checkValidZeroTerminatedUtf8to16("При", intArrayOf(-48, -97, -47, -128, -48, -72, 0, -48, -78, 0, -48, -75, -47, -126))
|
||||
checkValidZeroTerminatedUtf8to16("\uD800\uDC00", intArrayOf(-16, -112, -128, -128, 0, -16, -112, -128, -128))
|
||||
checkValidZeroTerminatedUtf8to16("", intArrayOf(0, 'H'.toInt()))
|
||||
|
||||
// Test manual conversion with replacement.
|
||||
// Incorrect UTF-8 lead character.
|
||||
checkZeroTerminatedUtf8to16Replacing("\uFFFD", intArrayOf(-1, 0, '1'.toInt()))
|
||||
|
||||
// Incomplete codepoint.
|
||||
checkZeroTerminatedUtf8to16Replacing("\uFFFD", intArrayOf(-16, -97, -104, 0, '1'.toInt()))
|
||||
checkZeroTerminatedUtf8to16Replacing("\uFFFD1", intArrayOf(-16, -97, -104, '1'.toInt(), 0, -16, -97, -104))
|
||||
|
||||
// Test manual conversion with exception throwing
|
||||
// Incorrect UTF-8 lead character -> throw.
|
||||
checkZeroTerminatedUtf8to16Throws("\uFFFD", intArrayOf(-1, 0, '1'.toInt()))
|
||||
|
||||
// Incomplete codepoint -> throw.
|
||||
checkZeroTerminatedUtf8to16Throws("\uFFFD", intArrayOf(-16, -97, -104, 0, '1'.toInt()))
|
||||
checkZeroTerminatedUtf8to16Throws("\uFFFD1", intArrayOf(-16, -97, -104, '1'.toInt(), 0, -16, -97, -104))
|
||||
}
|
||||
|
||||
fun testZeroTerminated8To16CustomBorders() {
|
||||
val array = intArrayOf('a'.toInt(), 'a'.toInt(), 'a'.toInt(), 0, 'b'.toInt(), 'b'.toInt(), 'b'.toInt(), 0)
|
||||
checkValidZeroTerminatedUtf8to16("aaa", array, 0, 5)
|
||||
checkValidZeroTerminatedUtf8to16("a", array, 2, 2)
|
||||
checkValidZeroTerminatedUtf8to16("", array, 3, 2)
|
||||
checkValidZeroTerminatedUtf8to16("bb", array, 4, 2)
|
||||
checkValidZeroTerminatedUtf8to16("bbb", array, 4, 3)
|
||||
checkValidZeroTerminatedUtf8to16("bbb", array, 4, 4)
|
||||
checkValidZeroTerminatedUtf8to16("bb", array, 5, 3)
|
||||
checkValidZeroTerminatedUtf8to16("b", array, 6, 2)
|
||||
checkValidZeroTerminatedUtf8to16("", array, 7, 1)
|
||||
checkValidZeroTerminatedUtf8to16("", array, 8, 0)
|
||||
|
||||
val byteArray = ByteArray(array.size) { array[it].toByte() }
|
||||
checkOutOfBoundsZeroTerminatedUtf8to16(IndexOutOfBoundsException::class, "aaa0bbb0", byteArray, -1, 4)
|
||||
checkOutOfBoundsZeroTerminatedUtf8to16(IndexOutOfBoundsException::class, "aaa0bbb0", byteArray, 8, 10)
|
||||
checkOutOfBoundsZeroTerminatedUtf8to16(IndexOutOfBoundsException::class, "aaa0bbb0", byteArray, 2, 10)
|
||||
checkOutOfBoundsZeroTerminatedUtf8to16(IndexOutOfBoundsException::class, "aaa0bbb0", byteArray, 10, 0)
|
||||
checkOutOfBoundsZeroTerminatedUtf8to16(IllegalArgumentException::class, "aaa0bbb0", byteArray, 3, -2)
|
||||
}
|
||||
|
||||
// ------------------------------------ Run tests ------------------------------------
|
||||
@Test fun runTest() {
|
||||
test16to8()
|
||||
test16to8CustomBorders()
|
||||
test8to16()
|
||||
test8to16CustomBorders()
|
||||
testZeroTerminated8To16()
|
||||
testZeroTerminated8To16CustomBorders()
|
||||
testPrint()
|
||||
}
|
||||
|
||||
@@ -49,11 +49,8 @@ OBJ_GETTER(utf8ToUtf16Impl, const char* rawString, const char* end, uint32_t cha
|
||||
}
|
||||
|
||||
template<utf16to8 conversion>
|
||||
OBJ_GETTER(utf16ToUtf8Impl, KString thiz, KInt start, KInt size) {
|
||||
OBJ_GETTER(unsafeUtf16ToUtf8Impl, KString thiz, KInt start, KInt size) {
|
||||
RuntimeAssert(thiz->type_info() == theStringTypeInfo, "Must use String");
|
||||
if (start < 0 || size < 0 || (size > static_cast<KInt>(thiz->count_) - start)) {
|
||||
ThrowArrayIndexOutOfBoundsException();
|
||||
}
|
||||
const KChar* utf16 = CharArrayAddressOfElementAt(thiz, start);
|
||||
KStdString utf8;
|
||||
conversion(utf16, utf16 + size, back_inserter(utf8));
|
||||
@@ -791,12 +788,9 @@ OBJ_GETTER(Kotlin_String_toLowerCase, KString thiz) {
|
||||
RETURN_OBJ(result->obj());
|
||||
}
|
||||
|
||||
OBJ_GETTER(Kotlin_String_fromCharArray, KConstRef thiz, KInt start, KInt size) {
|
||||
OBJ_GETTER(Kotlin_String_unsafeStringFromCharArray, KConstRef thiz, KInt start, KInt size) {
|
||||
const ArrayHeader* array = thiz->array();
|
||||
RuntimeAssert(array->type_info() == theCharArrayTypeInfo, "Must use a char array");
|
||||
if (start < 0 || size < 0 || size > array->count_ - start) {
|
||||
ThrowArrayIndexOutOfBoundsException();
|
||||
}
|
||||
|
||||
if (size == 0) {
|
||||
RETURN_RESULT_OF0(TheEmptyString);
|
||||
@@ -878,37 +872,34 @@ KInt Kotlin_String_getStringLength(KString thiz) {
|
||||
return thiz->count_;
|
||||
}
|
||||
|
||||
const char* byteArrayAsCString(KConstRef thiz, KInt start, KInt size) {
|
||||
const char* unsafeByteArrayAsCString(KConstRef thiz, KInt start, KInt size) {
|
||||
const ArrayHeader* array = thiz->array();
|
||||
RuntimeAssert(array->type_info() == theByteArrayTypeInfo, "Must use a byte array");
|
||||
if (start < 0 || size < 0 || size > array->count_ - start) {
|
||||
ThrowArrayIndexOutOfBoundsException();
|
||||
}
|
||||
return reinterpret_cast<const char*>(ByteArrayAddressOfElementAt(array, start));
|
||||
}
|
||||
|
||||
OBJ_GETTER(Kotlin_ByteArray_stringFromUtf8OrThrow, KConstRef thiz, KInt start, KInt size) {
|
||||
OBJ_GETTER(Kotlin_ByteArray_unsafeStringFromUtf8OrThrow, KConstRef thiz, KInt start, KInt size) {
|
||||
if (size == 0) {
|
||||
RETURN_RESULT_OF0(TheEmptyString);
|
||||
}
|
||||
const char* rawString = byteArrayAsCString(thiz, start, size);
|
||||
const char* rawString = unsafeByteArrayAsCString(thiz, start, size);
|
||||
RETURN_RESULT_OF(utf8ToUtf16OrThrow, rawString, size);
|
||||
}
|
||||
|
||||
OBJ_GETTER(Kotlin_ByteArray_stringFromUtf8, KConstRef thiz, KInt start, KInt size) {
|
||||
OBJ_GETTER(Kotlin_ByteArray_unsafeStringFromUtf8, KConstRef thiz, KInt start, KInt size) {
|
||||
if (size == 0) {
|
||||
RETURN_RESULT_OF0(TheEmptyString);
|
||||
}
|
||||
const char* rawString = byteArrayAsCString(thiz, start, size);
|
||||
const char* rawString = unsafeByteArrayAsCString(thiz, start, size);
|
||||
RETURN_RESULT_OF(utf8ToUtf16, rawString, size);
|
||||
}
|
||||
|
||||
OBJ_GETTER(Kotlin_String_toUtf8, KString thiz, KInt start, KInt size) {
|
||||
RETURN_RESULT_OF(utf16ToUtf8Impl<utf8::with_replacement::utf16to8>, thiz, start, size);
|
||||
OBJ_GETTER(Kotlin_String_unsafeStringToUtf8, KString thiz, KInt start, KInt size) {
|
||||
RETURN_RESULT_OF(unsafeUtf16ToUtf8Impl<utf8::with_replacement::utf16to8>, thiz, start, size);
|
||||
}
|
||||
|
||||
OBJ_GETTER(Kotlin_String_toUtf8OrThrow, KString thiz, KInt start, KInt size) {
|
||||
RETURN_RESULT_OF(utf16ToUtf8Impl<utf16toUtf8OrThrow>, thiz, start, size);
|
||||
OBJ_GETTER(Kotlin_String_unsafeStringToUtf8OrThrow, KString thiz, KInt start, KInt size) {
|
||||
RETURN_RESULT_OF(unsafeUtf16ToUtf8Impl<utf16toUtf8OrThrow>, thiz, start, size);
|
||||
}
|
||||
|
||||
KInt Kotlin_StringBuilder_insertString(KRef builder, KInt position, KString fromString) {
|
||||
|
||||
@@ -166,6 +166,7 @@ public actual open class NumberFormatException : IllegalArgumentException {
|
||||
actual constructor(message: String?) : super(message)
|
||||
}
|
||||
|
||||
@Deprecated("Use CharacterCodingException instead", ReplaceWith("CharacterCodingException"))
|
||||
public open class IllegalCharacterConversionException : IllegalArgumentException {
|
||||
|
||||
constructor(): super()
|
||||
|
||||
@@ -5,38 +5,41 @@
|
||||
|
||||
package kotlin.native
|
||||
|
||||
import kotlinx.cinterop.toKString
|
||||
|
||||
/**
|
||||
* Converts an UTF-8 array into a [String]. Replaces invalid input sequences with a default character.
|
||||
*/
|
||||
@Deprecated(
|
||||
"Use toKString or decodeToString instead",
|
||||
ReplaceWith("toKString()", "kotlinx.cinterop.toKString")
|
||||
)
|
||||
public fun ByteArray.stringFromUtf8() : String {
|
||||
@Suppress("DEPRECATION")
|
||||
return this.stringFromUtf8(0, this.size)
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts an UTF-8 array into a [String]. Replaces invalid input sequences with a default character.
|
||||
*/
|
||||
@Deprecated(
|
||||
"Use toKString or decodeToString instead",
|
||||
ReplaceWith("toKString(start, start + size)", "kotlinx.cinterop.toKString")
|
||||
)
|
||||
public fun ByteArray.stringFromUtf8(start: Int = 0, size: Int = this.size) : String {
|
||||
checkBoundsIndexes(start, start + size, this.size)
|
||||
return stringFromUtf8Impl(start, realSize(this, size))
|
||||
}
|
||||
|
||||
@SymbolName("Kotlin_ByteArray_stringFromUtf8")
|
||||
internal external fun ByteArray.stringFromUtf8Impl(start: Int, size: Int) : String
|
||||
|
||||
private fun realSize(byteArray: ByteArray, size: Int): Int {
|
||||
var realSize = 0
|
||||
while (realSize < size && byteArray[realSize] != 0.toByte()) {
|
||||
realSize++
|
||||
}
|
||||
return realSize
|
||||
return toKString(start, start + size)
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts an UTF-8 array into a [String].
|
||||
* @throws [IllegalCharacterConversionException] if the input is invalid.
|
||||
*/
|
||||
@Deprecated(
|
||||
"Use toKString or decodeToString instead",
|
||||
ReplaceWith("toKString(throwOnInvalidSequence = true)", "kotlinx.cinterop.toKString")
|
||||
)
|
||||
public fun ByteArray.stringFromUtf8OrThrow() : String {
|
||||
@Suppress("DEPRECATION")
|
||||
return this.stringFromUtf8OrThrow(0, this.size)
|
||||
}
|
||||
|
||||
@@ -44,18 +47,19 @@ public fun ByteArray.stringFromUtf8OrThrow() : String {
|
||||
* Converts an UTF-8 array into a [String].
|
||||
* @throws [IllegalCharacterConversionException] if the input is invalid.
|
||||
*/
|
||||
@Deprecated(
|
||||
"Use toKString or decodeToString instead",
|
||||
ReplaceWith("toKString(start, start + size, throwOnInvalidSequence = true)", "kotlinx.cinterop.toKString")
|
||||
)
|
||||
public fun ByteArray.stringFromUtf8OrThrow(start: Int = 0, size: Int = this.size) : String {
|
||||
checkBoundsIndexes(start, start + size, this.size)
|
||||
try {
|
||||
return stringFromUtf8OrThrowImpl(start, realSize(this, size))
|
||||
return toKString(start, start + size, throwOnInvalidSequence = true)
|
||||
} catch (e: CharacterCodingException) {
|
||||
@Suppress("DEPRECATION")
|
||||
throw IllegalCharacterConversionException()
|
||||
}
|
||||
}
|
||||
|
||||
@SymbolName("Kotlin_ByteArray_stringFromUtf8OrThrow")
|
||||
internal external fun ByteArray.stringFromUtf8OrThrowImpl(start: Int, size: Int) : String
|
||||
|
||||
/**
|
||||
* Converts a [String] into an UTF-8 array. Replaces invalid input sequences with a default character.
|
||||
*/
|
||||
@@ -71,12 +75,9 @@ public fun String.toUtf8() : ByteArray {
|
||||
@Deprecated("Use encodeToByteArray instead", ReplaceWith("encodeToByteArray(start, start + size)"))
|
||||
public fun String.toUtf8(start: Int = 0, size: Int = this.length) : ByteArray {
|
||||
checkBoundsIndexes(start, start + size, this.length)
|
||||
return toUtf8Impl(start, size)
|
||||
return unsafeStringToUtf8(start, size)
|
||||
}
|
||||
|
||||
@SymbolName("Kotlin_String_toUtf8")
|
||||
internal external fun String.toUtf8Impl(start: Int, size: Int) : ByteArray
|
||||
|
||||
/**
|
||||
* Converts a [String] into an UTF-8 array.
|
||||
* @throws [IllegalCharacterConversionException] if the input is invalid.
|
||||
@@ -95,8 +96,9 @@ public fun String.toUtf8OrThrow() : ByteArray {
|
||||
public fun String.toUtf8OrThrow(start: Int = 0, size: Int = this.length) : ByteArray {
|
||||
checkBoundsIndexes(start, start + size, this.length)
|
||||
try {
|
||||
return toUtf8OrThrowImpl(start, size)
|
||||
return unsafeStringToUtf8OrThrow(start, size)
|
||||
} catch (e: CharacterCodingException) {
|
||||
@Suppress("DEPRECATION")
|
||||
throw IllegalCharacterConversionException()
|
||||
}
|
||||
}
|
||||
@@ -110,11 +112,20 @@ internal fun checkBoundsIndexes(startIndex: Int, endIndex: Int, size: Int) {
|
||||
}
|
||||
}
|
||||
|
||||
@SymbolName("Kotlin_String_toUtf8OrThrow")
|
||||
internal external fun String.toUtf8OrThrowImpl(start: Int, size: Int) : ByteArray
|
||||
@SymbolName("Kotlin_ByteArray_unsafeStringFromUtf8")
|
||||
internal external fun ByteArray.unsafeStringFromUtf8(start: Int, size: Int) : String
|
||||
|
||||
@SymbolName("Kotlin_String_fromCharArray")
|
||||
internal external fun fromCharArray(array: CharArray, start: Int, size: Int) : String
|
||||
@SymbolName("Kotlin_ByteArray_unsafeStringFromUtf8OrThrow")
|
||||
internal external fun ByteArray.unsafeStringFromUtf8OrThrow(start: Int, size: Int) : String
|
||||
|
||||
@SymbolName("Kotlin_String_unsafeStringToUtf8")
|
||||
internal external fun String.unsafeStringToUtf8(start: Int, size: Int) : ByteArray
|
||||
|
||||
@SymbolName("Kotlin_String_unsafeStringToUtf8OrThrow")
|
||||
internal external fun String.unsafeStringToUtf8OrThrow(start: Int, size: Int) : ByteArray
|
||||
|
||||
@SymbolName("Kotlin_String_unsafeStringFromCharArray")
|
||||
internal external fun unsafeStringFromCharArray(array: CharArray, start: Int, size: Int) : String
|
||||
|
||||
@SymbolName("Kotlin_StringBuilder_insertString")
|
||||
internal external fun insertString(array: CharArray, start: Int, value: String): Int
|
||||
|
||||
@@ -162,7 +162,7 @@ class NumberConverter {
|
||||
if (k == expt - 1)
|
||||
formattedDecimal[charPos++] = '0'
|
||||
formattedDecimal[charPos++] = 'E'
|
||||
return fromCharArray(formattedDecimal, 0, charPos) + expt.toString()
|
||||
return unsafeStringFromCharArray(formattedDecimal, 0, charPos) + expt.toString()
|
||||
}
|
||||
|
||||
private fun freeFormat(): String {
|
||||
@@ -192,7 +192,7 @@ class NumberConverter {
|
||||
k--
|
||||
u = if (getCount < setCount) uArray[getCount++] else -1
|
||||
} while (u != -1 || k >= -1)
|
||||
return fromCharArray(formattedDecimal, 0, charPos)
|
||||
return unsafeStringFromCharArray(formattedDecimal, 0, charPos)
|
||||
}
|
||||
|
||||
private fun bigIntDigitGeneratorInstImpl(f: Long, e: Int,
|
||||
|
||||
@@ -53,12 +53,12 @@ actual class StringBuilder private constructor (
|
||||
|
||||
actual override fun subSequence(startIndex: Int, endIndex: Int): CharSequence = substring(startIndex, endIndex)
|
||||
|
||||
override fun toString(): String = fromCharArray(array, 0, _length)
|
||||
override fun toString(): String = unsafeStringFromCharArray(array, 0, _length)
|
||||
|
||||
fun substring(startIndex: Int, endIndex: Int): String {
|
||||
checkInsertIndex(startIndex)
|
||||
checkInsertIndexFrom(endIndex, startIndex)
|
||||
return fromCharArray(array, startIndex, endIndex - startIndex)
|
||||
return unsafeStringFromCharArray(array, startIndex, endIndex - startIndex)
|
||||
}
|
||||
|
||||
fun trimToSize() {
|
||||
|
||||
@@ -205,7 +205,7 @@ public actual fun CharSequence.repeat(n: Int): String {
|
||||
else -> {
|
||||
when (length) {
|
||||
0 -> ""
|
||||
1 -> this[0].let { char -> fromCharArray(CharArray(n) { char }, 0, n) }
|
||||
1 -> this[0].let { char -> CharArray(n) { char }.concatToString() }
|
||||
else -> {
|
||||
val sb = StringBuilder(n * length)
|
||||
for (i in 1..n) {
|
||||
@@ -221,7 +221,7 @@ public actual fun CharSequence.repeat(n: Int): String {
|
||||
/**
|
||||
* Converts the characters in the specified array to a string.
|
||||
*/
|
||||
public actual fun String(chars: CharArray): String = fromCharArray(chars, 0, chars.size)
|
||||
public actual fun String(chars: CharArray): String = chars.concatToString()
|
||||
|
||||
/**
|
||||
* Converts the characters from a portion of the specified array to a string.
|
||||
@@ -233,14 +233,14 @@ public actual fun String(chars: CharArray, offset: Int, length: Int): String {
|
||||
if (offset < 0 || length < 0 || offset + length > chars.size)
|
||||
throw ArrayIndexOutOfBoundsException()
|
||||
|
||||
return fromCharArray(chars, offset, length)
|
||||
return unsafeStringFromCharArray(chars, offset, length)
|
||||
}
|
||||
|
||||
/**
|
||||
* Concatenates characters in this [CharArray] into a String.
|
||||
*/
|
||||
@SinceKotlin("1.3")
|
||||
public actual fun CharArray.concatToString(): String = fromCharArray(this, 0, size)
|
||||
public actual fun CharArray.concatToString(): String = unsafeStringFromCharArray(this, 0, size)
|
||||
|
||||
/**
|
||||
* Concatenates characters in this [CharArray] or its subrange into a String.
|
||||
@@ -254,7 +254,7 @@ public actual fun CharArray.concatToString(): String = fromCharArray(this, 0, si
|
||||
@SinceKotlin("1.3")
|
||||
public actual fun CharArray.concatToString(startIndex: Int, endIndex: Int): String {
|
||||
checkBoundsIndexes(startIndex, endIndex, size)
|
||||
return fromCharArray(this, startIndex, endIndex - startIndex)
|
||||
return unsafeStringFromCharArray(this, startIndex, endIndex - startIndex)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -278,7 +278,7 @@ public actual fun String.toCharArray(startIndex: Int, endIndex: Int): CharArray
|
||||
* Malformed byte sequences are replaced by the replacement char `\uFFFD`.
|
||||
*/
|
||||
@SinceKotlin("1.3")
|
||||
public actual fun ByteArray.decodeToString(): String = stringFromUtf8Impl(0, size)
|
||||
public actual fun ByteArray.decodeToString(): String = unsafeStringFromUtf8(0, size)
|
||||
|
||||
/**
|
||||
* Decodes a string from the bytes in UTF-8 encoding in this array or its subrange.
|
||||
@@ -295,9 +295,9 @@ public actual fun ByteArray.decodeToString(): String = stringFromUtf8Impl(0, siz
|
||||
public actual fun ByteArray.decodeToString(startIndex: Int, endIndex: Int, throwOnInvalidSequence: Boolean): String {
|
||||
checkBoundsIndexes(startIndex, endIndex, size)
|
||||
return if (throwOnInvalidSequence)
|
||||
stringFromUtf8OrThrowImpl(startIndex, endIndex - startIndex)
|
||||
unsafeStringFromUtf8OrThrow(startIndex, endIndex - startIndex)
|
||||
else
|
||||
stringFromUtf8Impl(startIndex, endIndex - startIndex)
|
||||
unsafeStringFromUtf8(startIndex, endIndex - startIndex)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -306,7 +306,7 @@ public actual fun ByteArray.decodeToString(startIndex: Int, endIndex: Int, throw
|
||||
* Any malformed char sequence is replaced by the replacement byte sequence.
|
||||
*/
|
||||
@SinceKotlin("1.3")
|
||||
public actual fun String.encodeToByteArray(): ByteArray = toUtf8Impl(0, length)
|
||||
public actual fun String.encodeToByteArray(): ByteArray = unsafeStringToUtf8(0, length)
|
||||
|
||||
/**
|
||||
* Encodes this string or its substring to an array of bytes in UTF-8 encoding.
|
||||
@@ -323,9 +323,9 @@ public actual fun String.encodeToByteArray(): ByteArray = toUtf8Impl(0, length)
|
||||
public actual fun String.encodeToByteArray(startIndex: Int, endIndex: Int, throwOnInvalidSequence: Boolean): ByteArray {
|
||||
checkBoundsIndexes(startIndex, endIndex, length)
|
||||
return if (throwOnInvalidSequence)
|
||||
toUtf8OrThrowImpl(startIndex, endIndex - startIndex)
|
||||
unsafeStringToUtf8OrThrow(startIndex, endIndex - startIndex)
|
||||
else
|
||||
toUtf8Impl(startIndex, endIndex - startIndex)
|
||||
unsafeStringToUtf8(startIndex, endIndex - startIndex)
|
||||
}
|
||||
|
||||
@SymbolName("Kotlin_String_compareToIgnoreCase")
|
||||
|
||||
@@ -39,7 +39,7 @@ actual fun readFile(fileName: String): String {
|
||||
actual fun Double.format(decimalNumber: Int): String {
|
||||
var buffer = ByteArray(1024)
|
||||
snprintf(buffer.refTo(0), buffer.size.toULong(), "%.${decimalNumber}f", this)
|
||||
return buffer.stringFromUtf8()
|
||||
return buffer.toKString()
|
||||
}
|
||||
|
||||
actual fun writeToFile(fileName: String, text: String) {
|
||||
@@ -91,7 +91,7 @@ class CUrl(url: String, user: String? = null, password: String? = null, followLo
|
||||
|
||||
fun CPointer<ByteVar>.toKString(length: Int): String {
|
||||
val bytes = this.readBytes(length)
|
||||
return bytes.stringFromUtf8()
|
||||
return bytes.toKString()
|
||||
}
|
||||
|
||||
fun collectResponse(buffer: CPointer<ByteVar>?, size: size_t, nitems: size_t, userdata: COpaquePointer?): size_t {
|
||||
|
||||
Reference in New Issue
Block a user