diff --git a/Interop/Runtime/src/main/kotlin/kotlinx/cinterop/Utils.kt b/Interop/Runtime/src/main/kotlin/kotlinx/cinterop/Utils.kt index ec2acd311f9..0da04b0478d 100644 --- a/Interop/Runtime/src/main/kotlin/kotlinx/cinterop/Utils.kt +++ b/Interop/Runtime/src/main/kotlin/kotlinx/cinterop/Utils.kt @@ -554,6 +554,60 @@ public fun CPointer.toKStringFromUtf32(): String { return String(chars) } +/** + * Decodes a string from the bytes in UTF-8 encoding in this array. + * Bytes following the first occurrence of `0` byte, if it occurs, are not decoded. + * + * Malformed byte sequences are replaced by the replacement char `\uFFFD`. + */ +@UseExperimental(ExperimentalStdlibApi::class) +@SinceKotlin("1.3") +public fun ByteArray.toKString() : String { + val realEndIndex = realEndIndex(this, 0, this.size) + return decodeToString(0, realEndIndex) +} + +/** + * Decodes a string from the bytes in UTF-8 encoding in this array or its subrange. + * Bytes following the first occurrence of `0` byte, if it occurs, are not decoded. + * + * @param startIndex the beginning (inclusive) of the subrange to decode, 0 by default. + * @param endIndex the end (exclusive) of the subrange to decode, size of this array by default. + * @param throwOnInvalidSequence specifies whether to throw an exception on malformed byte sequence or replace it by the replacement char `\uFFFD`. + * + * @throws IndexOutOfBoundsException if [startIndex] is less than zero or [endIndex] is greater than the size of this array. + * @throws IllegalArgumentException if [startIndex] is greater than [endIndex]. + * @throws CharacterCodingException if the byte array contains malformed UTF-8 byte sequence and [throwOnInvalidSequence] is true. + */ +@UseExperimental(ExperimentalStdlibApi::class) +@SinceKotlin("1.3") +public fun ByteArray.toKString( + startIndex: Int = 0, + endIndex: Int = this.size, + throwOnInvalidSequence: Boolean = false +) : String { + checkBoundsIndexes(startIndex, endIndex, this.size) + val realEndIndex = realEndIndex(this, startIndex, endIndex) + return decodeToString(startIndex, realEndIndex, throwOnInvalidSequence) +} + +private fun realEndIndex(byteArray: ByteArray, startIndex: Int, endIndex: Int): Int { + var index = startIndex + while (index < endIndex && byteArray[index] != 0.toByte()) { + index++ + } + return index +} + +private fun checkBoundsIndexes(startIndex: Int, endIndex: Int, size: Int) { + if (startIndex < 0 || endIndex > size) { + throw IndexOutOfBoundsException("startIndex: $startIndex, endIndex: $endIndex, size: $size") + } + if (startIndex > endIndex) { + throw IllegalArgumentException("startIndex: $startIndex > endIndex: $endIndex") + } +} + public class MemScope : ArenaBase() { val memScope: MemScope diff --git a/Interop/Runtime/src/native/kotlin/kotlinx/cinterop/NativeUtils.kt b/Interop/Runtime/src/native/kotlin/kotlinx/cinterop/NativeUtils.kt index 1e845840070..10d39259606 100644 --- a/Interop/Runtime/src/native/kotlin/kotlinx/cinterop/NativeUtils.kt +++ b/Interop/Runtime/src/native/kotlin/kotlinx/cinterop/NativeUtils.kt @@ -21,8 +21,7 @@ import kotlin.native.internal.TypedIntrinsic import kotlin.native.internal.IntrinsicType internal fun decodeFromUtf8(bytes: ByteArray): String = bytes.decodeToString() - -fun encodeToUtf8(str: String): ByteArray = str.encodeToByteArray() +internal fun encodeToUtf8(str: String): ByteArray = str.encodeToByteArray() @TypedIntrinsic(IntrinsicType.INTEROP_BITS_TO_FLOAT) external fun bitsToFloat(bits: Int): Float diff --git a/backend.native/tests/build.gradle b/backend.native/tests/build.gradle index effd656e354..d81b608671a 100644 --- a/backend.native/tests/build.gradle +++ b/backend.native/tests/build.gradle @@ -2159,7 +2159,7 @@ task indexof(type: KonanLocalTest) { task utf8(type: KonanLocalTest) { expectedFail = (project.testTarget == 'wasm32') // Uses exceptions. - goldValue = "Hello\nПривет\n\uD800\uDC00\n\n\uFFFD\uFFFD\n\uFFFD12\n\uFFFD12\n12\uFFFD\n\uD83D\uDE25\n" + goldValue = "Hello\nПривет\n\uD800\uDC00\n\n\uFFFD\uFFFD\n\uFFFD12\n\uFFFD12\n12\uFFFD\n\uD83D\uDE25\n\uD83D\uDE25\n\uD83D\uDE25\n" source = "runtime/text/utf8.kt" } diff --git a/backend.native/tests/interop/platform_zlib.kt b/backend.native/tests/interop/platform_zlib.kt index 87d337d9771..0de4f44fea8 100644 --- a/backend.native/tests/interop/platform_zlib.kt +++ b/backend.native/tests/interop/platform_zlib.kt @@ -21,7 +21,7 @@ fun main(args: Array) = memScoped { }.ptr if (inflateInit2(z, -15) == Z_OK && inflate(z, Z_FINISH) == Z_STREAM_END && inflateEnd(z) == Z_OK) - println(buffer.stringFromUtf8()) + println(buffer.toKString()) } println(golden.toKString()) } diff --git a/backend.native/tests/runtime/text/utf8.kt b/backend.native/tests/runtime/text/utf8.kt index 7a349e25628..531293c2e85 100644 --- a/backend.native/tests/runtime/text/utf8.kt +++ b/backend.native/tests/runtime/text/utf8.kt @@ -7,11 +7,13 @@ package runtime.text.utf8 import kotlin.test.* import kotlin.reflect.KClass +import kotlinx.cinterop.toKString -// region Util +// -------------------------------------- Utils -------------------------------------- fun assertEquals(expected: ByteArray, actual: ByteArray, message: String) = assertTrue(expected.contentEquals(actual), message) + fun checkUtf16to8(string: String, expected: IntArray, conversion: String.() -> ByteArray) { expected.forEach { assertTrue(it in Byte.MIN_VALUE..Byte.MAX_VALUE, "Expected array contains illegal values") @@ -25,15 +27,35 @@ fun checkUtf16to8(string: String, expected: IntArray, conversion: String.() -> B """.trimIndent()) } -fun checkUtf16to8Replacing(string: String, expected: IntArray) = checkUtf16to8(string, expected) { toUtf8() } -fun checkUtf16to8Throwing(string: String, expected: IntArray) = checkUtf16to8(string, expected) { toUtf8OrThrow() } +// Utils for checking successful UTF-16 to UTF-8 conversion. +fun checkUtf16to8Replacing(string: String, expected: IntArray) { + checkUtf16to8(string, expected) { toUtf8() } + checkUtf16to8(string, expected) { encodeToByteArray() } +} +fun checkUtf16to8Throwing(string: String, expected: IntArray) { + checkUtf16to8(string, expected) { toUtf8OrThrow() } + checkUtf16to8(string, expected) { encodeToByteArray(throwOnInvalidSequence = true) } +} +fun checkValidUtf16to8(string: String, expected: IntArray) { + checkUtf16to8Replacing(string, expected) + checkUtf16to8Throwing(string, expected) +} -fun checkUtf16to8Replacing(string: String, expected: IntArray, start: Int, size: Int) = - checkUtf16to8(string, expected) { toUtf8(start, size) } +fun checkUtf16to8Replacing(string: String, expected: IntArray, start: Int, size: Int) { + checkUtf16to8(string, expected) { toUtf8(start, size) } + checkUtf16to8(string, expected) { encodeToByteArray(start, start + size) } +} +fun checkUtf16to8Throwing(string: String, expected: IntArray, start: Int, size: Int) { + checkUtf16to8(string, expected) { toUtf8OrThrow(start, size) } + checkUtf16to8(string, expected) { encodeToByteArray(start, start + size, true) } +} +fun checkValidUtf16to8(string: String, expected: IntArray, start: Int, size: Int) { + checkUtf16to8Replacing(string, expected, start, size) + checkUtf16to8Throwing(string, expected, start, size) +} -fun checkUtf16to8Throwing(string: String, expected: IntArray, start: Int, size: Int) = - checkUtf16to8(string, expected) { toUtf8OrThrow(start, size) } +// Utils for checking successful UTF-8 to UTF-16 conversion. fun checkUtf8to16(expected: String, array: IntArray, conversion: ByteArray.() -> String) { array.forEach { assertTrue(it in Byte.MIN_VALUE..Byte.MAX_VALUE, "Expected array contains illegal values") @@ -47,14 +69,62 @@ fun checkUtf8to16(expected: String, array: IntArray, conversion: ByteArray.() -> """.trimIndent()) } -fun checkUtf8to16Replacing(expected: String, array: IntArray) = checkUtf8to16(expected, array) { stringFromUtf8() } -fun checkUtf8to16Throwing(expected: String, array: IntArray) = checkUtf8to16(expected, array) { stringFromUtf8OrThrow() } +fun checkZeroTerminatedUtf8to16Replacing(expected: String, array: IntArray) { + checkUtf8to16(expected, array) { stringFromUtf8() } + checkUtf8to16(expected, array) { toKString() } +} +fun checkUtf8to16Replacing(expected: String, array: IntArray) { + checkUtf8to16(expected, array) { stringFromUtf8() } + checkZeroTerminatedUtf8to16Replacing(expected, array) + checkZeroTerminatedUtf8to16Replacing(expected, array.copyOf(array.size + 1)) +} +fun checkZeroTerminatedUtf8to16Throwing(expected: String, array: IntArray) { + checkUtf8to16(expected, array) { stringFromUtf8OrThrow() } + checkUtf8to16(expected, array) { toKString(throwOnInvalidSequence = true) } +} +fun checkUtf8to16Throwing(expected: String, array: IntArray) { + checkUtf8to16(expected, array) { decodeToString(throwOnInvalidSequence = true) } + checkZeroTerminatedUtf8to16Throwing(expected, array) + checkZeroTerminatedUtf8to16Throwing(expected, array.copyOf(array.size + 1)) +} +fun checkValidUtf8to16(expected: String, array: IntArray) { + checkUtf8to16Replacing(expected, array) + checkUtf8to16Throwing(expected, array) +} +fun checkValidZeroTerminatedUtf8to16(expected: String, array: IntArray) { + checkZeroTerminatedUtf8to16Replacing(expected, array) + checkZeroTerminatedUtf8to16Throwing(expected, array) +} -fun checkUtf8to16Replacing(expected: String, array: IntArray, start: Int, size: Int) = - checkUtf8to16(expected, array) { stringFromUtf8(start, size) } -fun checkUtf8to16Throwing(expected: String, array: IntArray, start: Int, size: Int) = - checkUtf8to16(expected, array) { stringFromUtf8OrThrow(start, size) } +fun checkZeroTerminatedUtf8to16Replacing(expected: String, array: IntArray, start: Int, size: Int) { + checkUtf8to16(expected, array) { stringFromUtf8(start, size) } + checkUtf8to16(expected, array) { toKString(start, start + size) } +} +fun checkUtf8to16Replacing(expected: String, array: IntArray, start: Int, size: Int) { + checkUtf8to16(expected, array) { decodeToString(start, start + size) } + checkZeroTerminatedUtf8to16Replacing(expected, array, start, size) + checkZeroTerminatedUtf8to16Replacing(expected, array.copyOf(array.size + 1), start, size) +} +fun checkZeroTerminatedUtf8to16Throwing(expected: String, array: IntArray, start: Int, size: Int) { + checkUtf8to16(expected, array) { stringFromUtf8OrThrow(start, size) } + checkUtf8to16(expected, array) { toKString(start, start + size, true) } +} +fun checkUtf8to16Throwing(expected: String, array: IntArray, start: Int, size: Int) { + checkUtf8to16(expected, array) { decodeToString(start, start + size, true) } + checkZeroTerminatedUtf8to16Throwing(expected, array, start, size) + checkZeroTerminatedUtf8to16Throwing(expected, array.copyOf(array.size + 1), start, size) +} +fun checkValidUtf8to16(expected: String, array: IntArray, start: Int, size: Int) { + checkUtf8to16Replacing(expected, array, start, size) + checkUtf8to16Throwing(expected, array, start, size) +} +fun checkValidZeroTerminatedUtf8to16(expected: String, array: IntArray, start: Int, size: Int) { + checkZeroTerminatedUtf8to16Replacing(expected, array, start, size) + checkZeroTerminatedUtf8to16Throwing(expected, array, start, size) +} + +// Utils for checking malformed UTF-16 to UTF-8 conversion. fun checkThrows(e: KClass, string: String, action: () -> Unit) { var exception: Throwable? = null try { @@ -62,7 +132,7 @@ fun checkThrows(e: KClass, string: String, action: () -> Unit) { } catch (e: Throwable) { exception = e } - assertNotNull(exception, "No excpetion was thrown for string: $string") + assertNotNull(exception, "No exception was thrown for string: $string") assertTrue(e.isInstance(exception),""" Wrong exception was thrown for string: $string Expected: ${e.qualifiedName} @@ -70,51 +140,114 @@ fun checkThrows(e: KClass, string: String, action: () -> Unit) { """.trimIndent()) } -fun checkUtf16to8Throws(string: String) = checkThrows(IllegalCharacterConversionException::class, string) { - string.toUtf8OrThrow() +fun checkUtf16to8Throws(string: String) { + checkThrows(IllegalCharacterConversionException::class, string) { string.toUtf8OrThrow() } + checkThrows(CharacterCodingException::class, string) { string.encodeToByteArray(throwOnInvalidSequence = true) } +} +fun checkUtf16to8Throws(string: String, start: Int, size: Int) { + checkThrows(IllegalCharacterConversionException::class, string) { string.toUtf8OrThrow(start, size) } + checkThrows(CharacterCodingException::class, string) { string.encodeToByteArray(start, start + size, true) } } -fun checkUtf16to8Throws(string: String, start: Int, size: Int) = - checkThrows(IllegalCharacterConversionException::class, string) { - string.toUtf8OrThrow(start, size) - } -fun checkUtf8to16Throws(string: String, array: IntArray) - = checkThrows(IllegalCharacterConversionException::class, string) { +// Utils for checking malformed UTF-8 to UTF-16 conversion. +fun checkUtf8to16Throws(e: KClass, string: String, array: IntArray, conversion: ByteArray.() -> String) + = checkThrows(e, string) { array.forEach { assertTrue(it in Byte.MIN_VALUE..Byte.MAX_VALUE, "Expected array contains illegal values") } val arrayBytes = ByteArray(array.size) { i -> array[i].toByte() } - arrayBytes.stringFromUtf8OrThrow() + arrayBytes.conversion() } -fun checkUtf8to16Throws(string: String, array: IntArray, start: Int, size: Int) - = checkThrows(IllegalCharacterConversionException::class, string) { - array.forEach { - assertTrue(it in Byte.MIN_VALUE..Byte.MAX_VALUE, "Expected array contains illegal values") - } - val arrayBytes = ByteArray(array.size) { i -> array[i].toByte() } - arrayBytes.stringFromUtf8OrThrow(start, size) +fun checkZeroTerminatedUtf8to16Throws(string: String, array: IntArray) { + checkUtf8to16Throws(IllegalCharacterConversionException::class, string, array) { stringFromUtf8OrThrow() } + checkUtf8to16Throws(CharacterCodingException::class, string, array) { toKString(throwOnInvalidSequence = true) } +} +fun checkUtf8to16Throws(string: String, array: IntArray) { + checkUtf8to16Throws(CharacterCodingException::class, string, array) { decodeToString(throwOnInvalidSequence = true) } + checkZeroTerminatedUtf8to16Throws(string, array) + checkZeroTerminatedUtf8to16Throws(string, array.copyOf(array.size + 1)) +} +fun checkZeroTerminatedUtf8to16Throws(string: String, array: IntArray, start: Int, size: Int) { + checkUtf8to16Throws(IllegalCharacterConversionException::class, string, array) { stringFromUtf8OrThrow(start, size) } + checkUtf8to16Throws(CharacterCodingException::class, string, array) { toKString(start, start + size, true) } +} +fun checkUtf8to16Throws(string: String, array: IntArray, start: Int, size: Int) { + checkUtf8to16Throws(CharacterCodingException::class, string, array) { decodeToString(start, start + size, true) } + checkZeroTerminatedUtf8to16Throws(string, array, start, size) + checkZeroTerminatedUtf8to16Throws(string, array.copyOf(array.size + 1), start, size) } -fun checkDobuleConversionThrows(string: String) = checkThrows(NumberFormatException::class, string) { +// Utils for checking String to floating-point number conversion. +fun checkDoubleConversionThrows(string: String) = checkThrows(NumberFormatException::class, string) { string.toDouble() } fun checkFloatConversionThrows(string: String) = checkThrows(NumberFormatException::class, string) { string.toFloat() } -// endregion -// region UTF-16 -> UTF-8 conversion + +// Utils for checking invalid-bounds-exception thrown by UTF-16 to UTF-8 conversion. +fun checkOutOfBoundsUtf16to8Replacing(e: KClass, string: String, start: Int, size: Int) { + checkThrows(e, string) { string.toUtf8(start, size) } + checkThrows(e, string) { string.encodeToByteArray(start, start + size) } +} +fun checkOutOfBoundsUtf16to8Throwing(e: KClass, string: String, start: Int, size: Int) { + checkThrows(e, string) { string.toUtf8OrThrow(start, size) } + checkThrows(e, string) { string.encodeToByteArray(start, start + size, true) } +} +fun checkOutOfBoundsUtf16to8(e: KClass, string: String, start: Int, size: Int) { + checkOutOfBoundsUtf16to8Replacing(e, string, start, size) + checkOutOfBoundsUtf16to8Throwing(e, string, start, size) +} + + +// Utils for checking invalid-bounds-exception thrown by UTF-8 to UTF-16 conversion. +fun checkOutOfBoundsZeroTerminatedUtf8to16Replacing(e: KClass, string: String, byteArray: ByteArray, start: Int, size: Int) { + checkThrows(e, string) { byteArray.stringFromUtf8(start, size) } + checkThrows(e, string) { byteArray.toKString(start, start + size) } +} +fun checkOutOfBoundsUtf8to16Replacing(e: KClass, string: String, byteArray: ByteArray, start: Int, size: Int) { + checkThrows(e, string) { byteArray.decodeToString(start, start + size) } + checkOutOfBoundsZeroTerminatedUtf8to16Replacing(e, string, byteArray, start, size) +} +fun checkOutOfBoundsZeroTerminatedUtf8to16Throwing(e: KClass, string: String, byteArray: ByteArray, start: Int, size: Int) { + checkThrows(e, string) { byteArray.stringFromUtf8OrThrow(start, size) } + checkThrows(e, string) { byteArray.toKString(start, start + size, true) } +} +fun checkOutOfBoundsUtf8to16Throwing(e: KClass, string: String, byteArray: ByteArray, start: Int, size: Int) { + checkThrows(e, string) { byteArray.decodeToString(start, start + size, true) } + checkOutOfBoundsZeroTerminatedUtf8to16Throwing(e, string, byteArray, start, size) +} +fun checkOutOfBoundsUtf8to16(e: KClass, string: String, byteArray: ByteArray, start: Int, size: Int) { + checkOutOfBoundsUtf8to16Replacing(e, string, byteArray, start, size) + checkOutOfBoundsUtf8to16Throwing(e, string, byteArray, start, size) +} +fun checkOutOfBoundsZeroTerminatedUtf8to16(e: KClass, string: String, byteArray: ByteArray, start: Int, size: Int) { + checkOutOfBoundsZeroTerminatedUtf8to16Replacing(e, string, byteArray, start, size) + checkOutOfBoundsZeroTerminatedUtf8to16Throwing(e, string, byteArray, start, size) +} + + +// Util for performing action on result of UTF-8 to UTF-16 conversion. +fun convertUtf8to16(byteArray: ByteArray, action: (String) -> Unit) { + byteArray.stringFromUtf8().let { action(it) } + byteArray.decodeToString().let { action(it) } + byteArray.toKString().let { action(it) } +} + +// ------------------------- Test UTF-16 to UTF-8 conversion ------------------------- fun test16to8() { - // Test manual conversion with replacement. // Valid strings. - checkUtf16to8Replacing("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt())) - checkUtf16to8Replacing("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126)) - checkUtf16to8Replacing("\uD800\uDC00", intArrayOf(-16, -112, -128, -128)) - checkUtf16to8Replacing("", intArrayOf()) + checkValidUtf16to8("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt())) + checkValidUtf16to8("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126)) + checkValidUtf16to8("\uD800\uDC00", intArrayOf(-16, -112, -128, -128)) + checkValidUtf16to8("", intArrayOf()) + + // Test manual conversion with replacement. // Illegal surrogate pair -> replace with default checkUtf16to8Replacing("\uDC00\uD800", intArrayOf(-17, -65, -67, -17, -65, -67)) // Different kinds of input @@ -125,14 +258,7 @@ fun test16to8() { checkUtf16to8Replacing("\uDC0012", intArrayOf(-17, -65, -67, '1'.toInt(), '2'.toInt())) checkUtf16to8Replacing("12\uD800", intArrayOf('1'.toInt(), '2'.toInt(), -17, -65, -67)) - // Test manual conversion with an exception if an input is invalid. - // Valid strings. - checkUtf16to8Throwing("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt())) - checkUtf16to8Throwing("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126)) - checkUtf16to8Throwing("\uD800\uDC00", intArrayOf(-16, -112, -128, -128)) - checkUtf16to8Replacing("", intArrayOf()) - // Illegal surrogate pair -> throw checkUtf16to8Throws("\uDC00\uD800") // Different kinds of input (including illegal one) -> throw @@ -145,13 +271,13 @@ fun test16to8() { // Test double parsing. assertEquals(4.2, "4.2".toDouble()) // Illegal surrogate pair -> throw - checkDobuleConversionThrows("\uDC00\uD800") + checkDoubleConversionThrows("\uDC00\uD800") // Different kinds of input (including illegal one) -> throw - checkDobuleConversionThrows("\uD800\uDC001\uDC00\uD800") + checkDoubleConversionThrows("\uD800\uDC001\uDC00\uD800") // Lone surrogate - throw - checkDobuleConversionThrows("\uD80012") - checkDobuleConversionThrows("\uDC0012") - checkDobuleConversionThrows("12\uD800") + checkDoubleConversionThrows("\uD80012") + checkDoubleConversionThrows("\uDC0012") + checkDoubleConversionThrows("12\uD800") // Test float parsing. assertEquals(4.2F, "4.2".toFloat()) @@ -166,21 +292,27 @@ fun test16to8() { } fun test16to8CustomBorders() { - // Test manual conversion with replacement and custom borders. // Valid strings. - checkUtf16to8Replacing("Hello!", intArrayOf('H'.toInt(), 'e'.toInt()), 0, 2) - checkUtf16to8Replacing("Hello!", intArrayOf('e'.toInt(), 'l'.toInt()), 1, 2) - checkUtf16to8Replacing("Hello!", intArrayOf('o'.toInt(), '!'.toInt()), 4, 2) - checkUtf16to8Replacing("Hello!", intArrayOf(), 0, 0) - checkUtf16to8Replacing("Hello!", intArrayOf(), 6, 0) + checkValidUtf16to8("Hello!", intArrayOf('H'.toInt(), 'e'.toInt()), 0, 2) + checkValidUtf16to8("Hello!", intArrayOf('e'.toInt(), 'l'.toInt()), 1, 2) + checkValidUtf16to8("Hello!", intArrayOf('o'.toInt(), '!'.toInt()), 4, 2) + checkValidUtf16to8("Hello!", intArrayOf(), 0, 0) + checkValidUtf16to8("Hello!", intArrayOf(), 6, 0) - checkUtf16to8Replacing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00", + checkValidUtf16to8("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00", intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 0, 4) - checkUtf16to8Replacing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00", + checkValidUtf16to8("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00", intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 2, 4) - checkUtf16to8Replacing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00", + checkValidUtf16to8("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00", intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 4, 4) + // Index out of bound + checkOutOfBoundsUtf16to8(IndexOutOfBoundsException::class, "Hello", -1, 4) + checkOutOfBoundsUtf16to8(IndexOutOfBoundsException::class, "Hello", 5, 10) + checkOutOfBoundsUtf16to8(IndexOutOfBoundsException::class, "Hello", 2, 10) + checkOutOfBoundsUtf16to8(IllegalArgumentException::class, "Hello", 3, -2) + + // Test manual conversion with replacement and custom borders. // Illegal surrogate pair -> replace with default checkUtf16to8Replacing("\uDC00\uD80012", intArrayOf(-17, -65, -67, -17, -65, -67, '1'.toInt()), 0, 3) @@ -193,28 +325,7 @@ fun test16to8CustomBorders() { checkUtf16to8Replacing("1\uD800\uDC002", intArrayOf('1'.toInt(), -17, -65, -67), 0, 2) checkUtf16to8Replacing("1\uD800\uDC002", intArrayOf(-17, -65, -67, '2'.toInt()), 2, 2) - // Index out of bound - checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8(-1, 4)} - checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8(5, 10)} - checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8(2, 10)} - checkThrows(IllegalArgumentException::class, "Hello") { "Hello".toUtf8(3, -2)} - - // Test manual conversion with an exception if an input is invalid and custom borders. - // Valid strings. - checkUtf16to8Throwing("Hello!", intArrayOf('H'.toInt(), 'e'.toInt()), 0, 2) - checkUtf16to8Throwing("Hello!", intArrayOf('e'.toInt(), 'l'.toInt()), 1, 2) - checkUtf16to8Throwing("Hello!", intArrayOf('o'.toInt(), '!'.toInt()), 4, 2) - checkUtf16to8Throwing("Hello!", intArrayOf(), 0, 0) - checkUtf16to8Throwing("Hello!", intArrayOf(), 6, 0) - - checkUtf16to8Throwing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00", - intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 0, 4) - checkUtf16to8Throwing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00", - intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 2, 4) - checkUtf16to8Throwing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00", - intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 4, 4) - // Illegal surrogate pair -> throw checkUtf16to8Throws("\uDC00\uD80012", 0, 3) checkUtf16to8Throws("1\uDC00\uD8002", 1, 3) @@ -223,12 +334,6 @@ fun test16to8CustomBorders() { // Lone surrogate -> throw checkUtf16to8Throws("1\uD800\uDC002", 0, 2) checkUtf16to8Throws("1\uD800\uDC002", 2, 2) - - // Index out of bound - checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8OrThrow(-1, 4)} - checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8OrThrow(5, 10)} - checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8OrThrow(2, 10)} - checkThrows(IllegalArgumentException::class, "Hello") { "Hello".toUtf8OrThrow(3, -2)} } fun testPrint() { @@ -247,21 +352,21 @@ fun testPrint() { // https://github.com/JetBrains/kotlin-native/issues/1091 val array = byteArrayOf(0xF0.toByte(), 0x9F.toByte(), 0x98.toByte(), 0xA5.toByte()) - val badStr = array.stringFromUtf8() - assertEquals(2, badStr.length) - println(badStr) + convertUtf8to16(array) { badString -> + assertEquals(2, badString.length) + println(badString) + } } -// endregion -// region UTF-8 -> UTF-16 conversion +// ------------------------- Test UTF-8 to UTF-16 conversion ------------------------- fun test8to16() { - // Test manual conversion with replacement. // Valid strings. - checkUtf8to16Replacing("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt())) - checkUtf8to16Replacing("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126)) - checkUtf8to16Replacing("\uD800\uDC00", intArrayOf(-16, -112, -128, -128)) - checkUtf8to16Replacing("", intArrayOf()) + checkValidUtf8to16("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt())) + checkValidUtf8to16("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126)) + checkValidUtf8to16("\uD800\uDC00", intArrayOf(-16, -112, -128, -128)) + checkValidUtf8to16("", intArrayOf()) + // Test manual conversion with replacement. // Incorrect UTF-8 lead character. checkUtf8to16Replacing("\uFFFD1", intArrayOf(-1, '1'.toInt())) @@ -270,12 +375,6 @@ fun test8to16() { checkUtf8to16Replacing("\uFFFD1\uFFFD", intArrayOf(-16, -97, -104, '1'.toInt(), -16, -97, -104)) // Test manual conversion with exception throwing - // Valid strings. - checkUtf8to16Throwing("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt())) - checkUtf8to16Throwing("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126)) - checkUtf8to16Throwing("\uD800\uDC00", intArrayOf(-16, -112, -128, -128)) - checkUtf8to16Throwing("", intArrayOf()) - // Incorrect UTF-8 lead character -> throw. checkUtf8to16Throws("\uFFFD1", intArrayOf(-1, '1'.toInt())) @@ -285,26 +384,35 @@ fun test8to16() { } fun test8to16CustomBorders() { - // Conversion with replacement. - checkUtf8to16Replacing("He", + // Valid strings. + checkValidUtf8to16("He", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()),0, 2) - checkUtf8to16Replacing("ll", + checkValidUtf8to16("ll", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 2, 2) - checkUtf8to16Replacing("lo", + checkValidUtf8to16("lo", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 3, 2) - checkUtf8to16Replacing("", + checkValidUtf8to16("", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 0, 0) - checkUtf8to16Replacing("\uD800\uDC00\uD800\uDC00", + checkValidUtf8to16("\uD800\uDC00\uD800\uDC00", intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128), 0, 8) - checkUtf8to16Replacing("\uD800\uDC00\uD800\uDC00", + checkValidUtf8to16("\uD800\uDC00\uD800\uDC00", intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128), 4, 8) - checkUtf8to16Replacing("\uD800\uDC00\uD800\uDC00", + checkValidUtf8to16("\uD800\uDC00\uD800\uDC00", intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128), 8, 8) + // Index out of bound + val helloArray = byteArrayOf('H'.toByte(), 'e'.toByte(), 'l'.toByte(), 'l'.toByte(), 'o'.toByte()) + checkOutOfBoundsUtf8to16(IndexOutOfBoundsException::class, "Hello", helloArray, -1, 4) + checkOutOfBoundsUtf8to16(IndexOutOfBoundsException::class, "Hello", helloArray, 5, 10) + checkOutOfBoundsUtf8to16(IndexOutOfBoundsException::class, "Hello", helloArray, 2, 10) + checkOutOfBoundsUtf8to16(IndexOutOfBoundsException::class, "Hello", helloArray, 10, 0) + checkOutOfBoundsUtf8to16(IllegalArgumentException::class, "Hello", helloArray, 3, -2) + + // Test manual conversion with replacement and custom borders. // Incorrect UTF-8 lead character. checkUtf8to16Replacing("\uFFFD1", intArrayOf(-1, '1'.toInt(), '2'.toInt()), 0, 2) checkUtf8to16Replacing("\uFFFD2", intArrayOf('1'.toInt(), -1, '2'.toInt(), '3'.toInt()), 1, 2) @@ -315,35 +423,7 @@ fun test8to16CustomBorders() { checkUtf8to16Replacing("\uFFFD2", intArrayOf('1'.toInt(), -16, -97, -104, '2'.toInt(), '3'.toInt()), 1, 4) checkUtf8to16Replacing("2\uFFFD", intArrayOf('1'.toInt(), '2'.toInt(), -16, -97, -104), 1, 4) - - // Index out of bound - val helloArray = byteArrayOf('H'.toByte(), 'e'.toByte(), 'l'.toByte(), 'l'.toByte(), 'o'.toByte()) - checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8(-1, 4)} - checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8(5, 10)} - checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8(2, 10)} - checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8(10, 0)} - checkThrows(IllegalArgumentException::class, "Hello") { helloArray.stringFromUtf8(3, -2)} - - // Conversion with throwing - checkUtf8to16Throwing("He", - intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()),0, 2) - checkUtf8to16Throwing("ll", - intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 2, 2) - checkUtf8to16Throwing("lo", - intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 3, 2) - checkUtf8to16Throwing("", - intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 0, 0) - - checkUtf8to16Throwing("\uD800\uDC00\uD800\uDC00", - intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128), - 0, 8) - checkUtf8to16Throwing("\uD800\uDC00\uD800\uDC00", - intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128), - 4, 8) - checkUtf8to16Throwing("\uD800\uDC00\uD800\uDC00", - intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128), - 8, 8) - + // Test manual conversion with an exception if an input is invalid and custom borders. // Incorrect UTF-8 lead character. checkUtf8to16Throws("\uFFFD1", intArrayOf(-1, '1'.toInt(), '2'.toInt()), 0, 2) checkUtf8to16Throws("\uFFFD2", intArrayOf('1'.toInt(), -1, '2'.toInt(), '3'.toInt()), 1, 2) @@ -353,20 +433,61 @@ fun test8to16CustomBorders() { checkUtf8to16Throws("\uFFFD1", intArrayOf(-16, -97, -104, '1'.toInt(), '2'.toInt()), 0, 4) checkUtf8to16Throws("\uFFFD2", intArrayOf('1'.toInt(), -16, -97, -104, '2'.toInt(), '3'.toInt()), 1, 4) checkUtf8to16Throws("2\uFFFD", intArrayOf('1'.toInt(), '2'.toInt(), -16, -97, -104), 1, 4) - - // Index out of bound - checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8OrThrow(-1, 4)} - checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8OrThrow(5, 10)} - checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8OrThrow(2, 10)} - checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8OrThrow(10, 0)} - checkThrows(IllegalArgumentException::class, "Hello") { helloArray.stringFromUtf8OrThrow(3, -2)} } -// endregion +// ----------------- Test zero-terminated UTF-8 to UTF-16 conversion ----------------- +fun testZeroTerminated8To16() { + // Valid strings. + checkValidZeroTerminatedUtf8to16("Hell", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 0, 'o'.toInt())) + checkValidZeroTerminatedUtf8to16("При", intArrayOf(-48, -97, -47, -128, -48, -72, 0, -48, -78, 0, -48, -75, -47, -126)) + checkValidZeroTerminatedUtf8to16("\uD800\uDC00", intArrayOf(-16, -112, -128, -128, 0, -16, -112, -128, -128)) + checkValidZeroTerminatedUtf8to16("", intArrayOf(0, 'H'.toInt())) + + // Test manual conversion with replacement. + // Incorrect UTF-8 lead character. + checkZeroTerminatedUtf8to16Replacing("\uFFFD", intArrayOf(-1, 0, '1'.toInt())) + + // Incomplete codepoint. + checkZeroTerminatedUtf8to16Replacing("\uFFFD", intArrayOf(-16, -97, -104, 0, '1'.toInt())) + checkZeroTerminatedUtf8to16Replacing("\uFFFD1", intArrayOf(-16, -97, -104, '1'.toInt(), 0, -16, -97, -104)) + + // Test manual conversion with exception throwing + // Incorrect UTF-8 lead character -> throw. + checkZeroTerminatedUtf8to16Throws("\uFFFD", intArrayOf(-1, 0, '1'.toInt())) + + // Incomplete codepoint -> throw. + checkZeroTerminatedUtf8to16Throws("\uFFFD", intArrayOf(-16, -97, -104, 0, '1'.toInt())) + checkZeroTerminatedUtf8to16Throws("\uFFFD1", intArrayOf(-16, -97, -104, '1'.toInt(), 0, -16, -97, -104)) +} + +fun testZeroTerminated8To16CustomBorders() { + val array = intArrayOf('a'.toInt(), 'a'.toInt(), 'a'.toInt(), 0, 'b'.toInt(), 'b'.toInt(), 'b'.toInt(), 0) + checkValidZeroTerminatedUtf8to16("aaa", array, 0, 5) + checkValidZeroTerminatedUtf8to16("a", array, 2, 2) + checkValidZeroTerminatedUtf8to16("", array, 3, 2) + checkValidZeroTerminatedUtf8to16("bb", array, 4, 2) + checkValidZeroTerminatedUtf8to16("bbb", array, 4, 3) + checkValidZeroTerminatedUtf8to16("bbb", array, 4, 4) + checkValidZeroTerminatedUtf8to16("bb", array, 5, 3) + checkValidZeroTerminatedUtf8to16("b", array, 6, 2) + checkValidZeroTerminatedUtf8to16("", array, 7, 1) + checkValidZeroTerminatedUtf8to16("", array, 8, 0) + + val byteArray = ByteArray(array.size) { array[it].toByte() } + checkOutOfBoundsZeroTerminatedUtf8to16(IndexOutOfBoundsException::class, "aaa0bbb0", byteArray, -1, 4) + checkOutOfBoundsZeroTerminatedUtf8to16(IndexOutOfBoundsException::class, "aaa0bbb0", byteArray, 8, 10) + checkOutOfBoundsZeroTerminatedUtf8to16(IndexOutOfBoundsException::class, "aaa0bbb0", byteArray, 2, 10) + checkOutOfBoundsZeroTerminatedUtf8to16(IndexOutOfBoundsException::class, "aaa0bbb0", byteArray, 10, 0) + checkOutOfBoundsZeroTerminatedUtf8to16(IllegalArgumentException::class, "aaa0bbb0", byteArray, 3, -2) +} + +// ------------------------------------ Run tests ------------------------------------ @Test fun runTest() { test16to8() test16to8CustomBorders() test8to16() test8to16CustomBorders() + testZeroTerminated8To16() + testZeroTerminated8To16CustomBorders() testPrint() } diff --git a/runtime/src/main/cpp/KString.cpp b/runtime/src/main/cpp/KString.cpp index 3cc413a0cab..7df2c535190 100644 --- a/runtime/src/main/cpp/KString.cpp +++ b/runtime/src/main/cpp/KString.cpp @@ -49,11 +49,8 @@ OBJ_GETTER(utf8ToUtf16Impl, const char* rawString, const char* end, uint32_t cha } template -OBJ_GETTER(utf16ToUtf8Impl, KString thiz, KInt start, KInt size) { +OBJ_GETTER(unsafeUtf16ToUtf8Impl, KString thiz, KInt start, KInt size) { RuntimeAssert(thiz->type_info() == theStringTypeInfo, "Must use String"); - if (start < 0 || size < 0 || (size > static_cast(thiz->count_) - start)) { - ThrowArrayIndexOutOfBoundsException(); - } const KChar* utf16 = CharArrayAddressOfElementAt(thiz, start); KStdString utf8; conversion(utf16, utf16 + size, back_inserter(utf8)); @@ -791,12 +788,9 @@ OBJ_GETTER(Kotlin_String_toLowerCase, KString thiz) { RETURN_OBJ(result->obj()); } -OBJ_GETTER(Kotlin_String_fromCharArray, KConstRef thiz, KInt start, KInt size) { +OBJ_GETTER(Kotlin_String_unsafeStringFromCharArray, KConstRef thiz, KInt start, KInt size) { const ArrayHeader* array = thiz->array(); RuntimeAssert(array->type_info() == theCharArrayTypeInfo, "Must use a char array"); - if (start < 0 || size < 0 || size > array->count_ - start) { - ThrowArrayIndexOutOfBoundsException(); - } if (size == 0) { RETURN_RESULT_OF0(TheEmptyString); @@ -878,37 +872,34 @@ KInt Kotlin_String_getStringLength(KString thiz) { return thiz->count_; } -const char* byteArrayAsCString(KConstRef thiz, KInt start, KInt size) { +const char* unsafeByteArrayAsCString(KConstRef thiz, KInt start, KInt size) { const ArrayHeader* array = thiz->array(); RuntimeAssert(array->type_info() == theByteArrayTypeInfo, "Must use a byte array"); - if (start < 0 || size < 0 || size > array->count_ - start) { - ThrowArrayIndexOutOfBoundsException(); - } return reinterpret_cast(ByteArrayAddressOfElementAt(array, start)); } -OBJ_GETTER(Kotlin_ByteArray_stringFromUtf8OrThrow, KConstRef thiz, KInt start, KInt size) { +OBJ_GETTER(Kotlin_ByteArray_unsafeStringFromUtf8OrThrow, KConstRef thiz, KInt start, KInt size) { if (size == 0) { RETURN_RESULT_OF0(TheEmptyString); } - const char* rawString = byteArrayAsCString(thiz, start, size); + const char* rawString = unsafeByteArrayAsCString(thiz, start, size); RETURN_RESULT_OF(utf8ToUtf16OrThrow, rawString, size); } -OBJ_GETTER(Kotlin_ByteArray_stringFromUtf8, KConstRef thiz, KInt start, KInt size) { +OBJ_GETTER(Kotlin_ByteArray_unsafeStringFromUtf8, KConstRef thiz, KInt start, KInt size) { if (size == 0) { RETURN_RESULT_OF0(TheEmptyString); } - const char* rawString = byteArrayAsCString(thiz, start, size); + const char* rawString = unsafeByteArrayAsCString(thiz, start, size); RETURN_RESULT_OF(utf8ToUtf16, rawString, size); } -OBJ_GETTER(Kotlin_String_toUtf8, KString thiz, KInt start, KInt size) { - RETURN_RESULT_OF(utf16ToUtf8Impl, thiz, start, size); +OBJ_GETTER(Kotlin_String_unsafeStringToUtf8, KString thiz, KInt start, KInt size) { + RETURN_RESULT_OF(unsafeUtf16ToUtf8Impl, thiz, start, size); } -OBJ_GETTER(Kotlin_String_toUtf8OrThrow, KString thiz, KInt start, KInt size) { - RETURN_RESULT_OF(utf16ToUtf8Impl, thiz, start, size); +OBJ_GETTER(Kotlin_String_unsafeStringToUtf8OrThrow, KString thiz, KInt start, KInt size) { + RETURN_RESULT_OF(unsafeUtf16ToUtf8Impl, thiz, start, size); } KInt Kotlin_StringBuilder_insertString(KRef builder, KInt position, KString fromString) { diff --git a/runtime/src/main/kotlin/kotlin/Exceptions.kt b/runtime/src/main/kotlin/kotlin/Exceptions.kt index 84a2fef0d01..6393c12ca1a 100644 --- a/runtime/src/main/kotlin/kotlin/Exceptions.kt +++ b/runtime/src/main/kotlin/kotlin/Exceptions.kt @@ -166,6 +166,7 @@ public actual open class NumberFormatException : IllegalArgumentException { actual constructor(message: String?) : super(message) } +@Deprecated("Use CharacterCodingException instead", ReplaceWith("CharacterCodingException")) public open class IllegalCharacterConversionException : IllegalArgumentException { constructor(): super() diff --git a/runtime/src/main/kotlin/kotlin/native/Text.kt b/runtime/src/main/kotlin/kotlin/native/Text.kt index 740c91a48cc..37370432a9b 100644 --- a/runtime/src/main/kotlin/kotlin/native/Text.kt +++ b/runtime/src/main/kotlin/kotlin/native/Text.kt @@ -5,38 +5,41 @@ package kotlin.native +import kotlinx.cinterop.toKString /** * Converts an UTF-8 array into a [String]. Replaces invalid input sequences with a default character. */ +@Deprecated( + "Use toKString or decodeToString instead", + ReplaceWith("toKString()", "kotlinx.cinterop.toKString") +) public fun ByteArray.stringFromUtf8() : String { + @Suppress("DEPRECATION") return this.stringFromUtf8(0, this.size) } /** * Converts an UTF-8 array into a [String]. Replaces invalid input sequences with a default character. */ +@Deprecated( + "Use toKString or decodeToString instead", + ReplaceWith("toKString(start, start + size)", "kotlinx.cinterop.toKString") +) public fun ByteArray.stringFromUtf8(start: Int = 0, size: Int = this.size) : String { - checkBoundsIndexes(start, start + size, this.size) - return stringFromUtf8Impl(start, realSize(this, size)) -} - -@SymbolName("Kotlin_ByteArray_stringFromUtf8") -internal external fun ByteArray.stringFromUtf8Impl(start: Int, size: Int) : String - -private fun realSize(byteArray: ByteArray, size: Int): Int { - var realSize = 0 - while (realSize < size && byteArray[realSize] != 0.toByte()) { - realSize++ - } - return realSize + return toKString(start, start + size) } /** * Converts an UTF-8 array into a [String]. * @throws [IllegalCharacterConversionException] if the input is invalid. */ +@Deprecated( + "Use toKString or decodeToString instead", + ReplaceWith("toKString(throwOnInvalidSequence = true)", "kotlinx.cinterop.toKString") +) public fun ByteArray.stringFromUtf8OrThrow() : String { + @Suppress("DEPRECATION") return this.stringFromUtf8OrThrow(0, this.size) } @@ -44,18 +47,19 @@ public fun ByteArray.stringFromUtf8OrThrow() : String { * Converts an UTF-8 array into a [String]. * @throws [IllegalCharacterConversionException] if the input is invalid. */ +@Deprecated( + "Use toKString or decodeToString instead", + ReplaceWith("toKString(start, start + size, throwOnInvalidSequence = true)", "kotlinx.cinterop.toKString") +) public fun ByteArray.stringFromUtf8OrThrow(start: Int = 0, size: Int = this.size) : String { - checkBoundsIndexes(start, start + size, this.size) try { - return stringFromUtf8OrThrowImpl(start, realSize(this, size)) + return toKString(start, start + size, throwOnInvalidSequence = true) } catch (e: CharacterCodingException) { + @Suppress("DEPRECATION") throw IllegalCharacterConversionException() } } -@SymbolName("Kotlin_ByteArray_stringFromUtf8OrThrow") -internal external fun ByteArray.stringFromUtf8OrThrowImpl(start: Int, size: Int) : String - /** * Converts a [String] into an UTF-8 array. Replaces invalid input sequences with a default character. */ @@ -71,12 +75,9 @@ public fun String.toUtf8() : ByteArray { @Deprecated("Use encodeToByteArray instead", ReplaceWith("encodeToByteArray(start, start + size)")) public fun String.toUtf8(start: Int = 0, size: Int = this.length) : ByteArray { checkBoundsIndexes(start, start + size, this.length) - return toUtf8Impl(start, size) + return unsafeStringToUtf8(start, size) } -@SymbolName("Kotlin_String_toUtf8") -internal external fun String.toUtf8Impl(start: Int, size: Int) : ByteArray - /** * Converts a [String] into an UTF-8 array. * @throws [IllegalCharacterConversionException] if the input is invalid. @@ -95,8 +96,9 @@ public fun String.toUtf8OrThrow() : ByteArray { public fun String.toUtf8OrThrow(start: Int = 0, size: Int = this.length) : ByteArray { checkBoundsIndexes(start, start + size, this.length) try { - return toUtf8OrThrowImpl(start, size) + return unsafeStringToUtf8OrThrow(start, size) } catch (e: CharacterCodingException) { + @Suppress("DEPRECATION") throw IllegalCharacterConversionException() } } @@ -110,11 +112,20 @@ internal fun checkBoundsIndexes(startIndex: Int, endIndex: Int, size: Int) { } } -@SymbolName("Kotlin_String_toUtf8OrThrow") -internal external fun String.toUtf8OrThrowImpl(start: Int, size: Int) : ByteArray +@SymbolName("Kotlin_ByteArray_unsafeStringFromUtf8") +internal external fun ByteArray.unsafeStringFromUtf8(start: Int, size: Int) : String -@SymbolName("Kotlin_String_fromCharArray") -internal external fun fromCharArray(array: CharArray, start: Int, size: Int) : String +@SymbolName("Kotlin_ByteArray_unsafeStringFromUtf8OrThrow") +internal external fun ByteArray.unsafeStringFromUtf8OrThrow(start: Int, size: Int) : String + +@SymbolName("Kotlin_String_unsafeStringToUtf8") +internal external fun String.unsafeStringToUtf8(start: Int, size: Int) : ByteArray + +@SymbolName("Kotlin_String_unsafeStringToUtf8OrThrow") +internal external fun String.unsafeStringToUtf8OrThrow(start: Int, size: Int) : ByteArray + +@SymbolName("Kotlin_String_unsafeStringFromCharArray") +internal external fun unsafeStringFromCharArray(array: CharArray, start: Int, size: Int) : String @SymbolName("Kotlin_StringBuilder_insertString") internal external fun insertString(array: CharArray, start: Int, value: String): Int diff --git a/runtime/src/main/kotlin/kotlin/native/internal/NumberConverter.kt b/runtime/src/main/kotlin/kotlin/native/internal/NumberConverter.kt index 3de5b967aba..cd53a538095 100644 --- a/runtime/src/main/kotlin/kotlin/native/internal/NumberConverter.kt +++ b/runtime/src/main/kotlin/kotlin/native/internal/NumberConverter.kt @@ -162,7 +162,7 @@ class NumberConverter { if (k == expt - 1) formattedDecimal[charPos++] = '0' formattedDecimal[charPos++] = 'E' - return fromCharArray(formattedDecimal, 0, charPos) + expt.toString() + return unsafeStringFromCharArray(formattedDecimal, 0, charPos) + expt.toString() } private fun freeFormat(): String { @@ -192,7 +192,7 @@ class NumberConverter { k-- u = if (getCount < setCount) uArray[getCount++] else -1 } while (u != -1 || k >= -1) - return fromCharArray(formattedDecimal, 0, charPos) + return unsafeStringFromCharArray(formattedDecimal, 0, charPos) } private fun bigIntDigitGeneratorInstImpl(f: Long, e: Int, diff --git a/runtime/src/main/kotlin/kotlin/text/StringBuilder.kt b/runtime/src/main/kotlin/kotlin/text/StringBuilder.kt index 87ba06fcb3d..941ee6e21b2 100644 --- a/runtime/src/main/kotlin/kotlin/text/StringBuilder.kt +++ b/runtime/src/main/kotlin/kotlin/text/StringBuilder.kt @@ -53,12 +53,12 @@ actual class StringBuilder private constructor ( actual override fun subSequence(startIndex: Int, endIndex: Int): CharSequence = substring(startIndex, endIndex) - override fun toString(): String = fromCharArray(array, 0, _length) + override fun toString(): String = unsafeStringFromCharArray(array, 0, _length) fun substring(startIndex: Int, endIndex: Int): String { checkInsertIndex(startIndex) checkInsertIndexFrom(endIndex, startIndex) - return fromCharArray(array, startIndex, endIndex - startIndex) + return unsafeStringFromCharArray(array, startIndex, endIndex - startIndex) } fun trimToSize() { diff --git a/runtime/src/main/kotlin/kotlin/text/Strings.kt b/runtime/src/main/kotlin/kotlin/text/Strings.kt index 9db8a26fcbd..e8df0b0a3cf 100644 --- a/runtime/src/main/kotlin/kotlin/text/Strings.kt +++ b/runtime/src/main/kotlin/kotlin/text/Strings.kt @@ -205,7 +205,7 @@ public actual fun CharSequence.repeat(n: Int): String { else -> { when (length) { 0 -> "" - 1 -> this[0].let { char -> fromCharArray(CharArray(n) { char }, 0, n) } + 1 -> this[0].let { char -> CharArray(n) { char }.concatToString() } else -> { val sb = StringBuilder(n * length) for (i in 1..n) { @@ -221,7 +221,7 @@ public actual fun CharSequence.repeat(n: Int): String { /** * Converts the characters in the specified array to a string. */ -public actual fun String(chars: CharArray): String = fromCharArray(chars, 0, chars.size) +public actual fun String(chars: CharArray): String = chars.concatToString() /** * Converts the characters from a portion of the specified array to a string. @@ -233,14 +233,14 @@ public actual fun String(chars: CharArray, offset: Int, length: Int): String { if (offset < 0 || length < 0 || offset + length > chars.size) throw ArrayIndexOutOfBoundsException() - return fromCharArray(chars, offset, length) + return unsafeStringFromCharArray(chars, offset, length) } /** * Concatenates characters in this [CharArray] into a String. */ @SinceKotlin("1.3") -public actual fun CharArray.concatToString(): String = fromCharArray(this, 0, size) +public actual fun CharArray.concatToString(): String = unsafeStringFromCharArray(this, 0, size) /** * Concatenates characters in this [CharArray] or its subrange into a String. @@ -254,7 +254,7 @@ public actual fun CharArray.concatToString(): String = fromCharArray(this, 0, si @SinceKotlin("1.3") public actual fun CharArray.concatToString(startIndex: Int, endIndex: Int): String { checkBoundsIndexes(startIndex, endIndex, size) - return fromCharArray(this, startIndex, endIndex - startIndex) + return unsafeStringFromCharArray(this, startIndex, endIndex - startIndex) } /** @@ -278,7 +278,7 @@ public actual fun String.toCharArray(startIndex: Int, endIndex: Int): CharArray * Malformed byte sequences are replaced by the replacement char `\uFFFD`. */ @SinceKotlin("1.3") -public actual fun ByteArray.decodeToString(): String = stringFromUtf8Impl(0, size) +public actual fun ByteArray.decodeToString(): String = unsafeStringFromUtf8(0, size) /** * Decodes a string from the bytes in UTF-8 encoding in this array or its subrange. @@ -295,9 +295,9 @@ public actual fun ByteArray.decodeToString(): String = stringFromUtf8Impl(0, siz public actual fun ByteArray.decodeToString(startIndex: Int, endIndex: Int, throwOnInvalidSequence: Boolean): String { checkBoundsIndexes(startIndex, endIndex, size) return if (throwOnInvalidSequence) - stringFromUtf8OrThrowImpl(startIndex, endIndex - startIndex) + unsafeStringFromUtf8OrThrow(startIndex, endIndex - startIndex) else - stringFromUtf8Impl(startIndex, endIndex - startIndex) + unsafeStringFromUtf8(startIndex, endIndex - startIndex) } /** @@ -306,7 +306,7 @@ public actual fun ByteArray.decodeToString(startIndex: Int, endIndex: Int, throw * Any malformed char sequence is replaced by the replacement byte sequence. */ @SinceKotlin("1.3") -public actual fun String.encodeToByteArray(): ByteArray = toUtf8Impl(0, length) +public actual fun String.encodeToByteArray(): ByteArray = unsafeStringToUtf8(0, length) /** * Encodes this string or its substring to an array of bytes in UTF-8 encoding. @@ -323,9 +323,9 @@ public actual fun String.encodeToByteArray(): ByteArray = toUtf8Impl(0, length) public actual fun String.encodeToByteArray(startIndex: Int, endIndex: Int, throwOnInvalidSequence: Boolean): ByteArray { checkBoundsIndexes(startIndex, endIndex, length) return if (throwOnInvalidSequence) - toUtf8OrThrowImpl(startIndex, endIndex - startIndex) + unsafeStringToUtf8OrThrow(startIndex, endIndex - startIndex) else - toUtf8Impl(startIndex, endIndex - startIndex) + unsafeStringToUtf8(startIndex, endIndex - startIndex) } @SymbolName("Kotlin_String_compareToIgnoreCase") diff --git a/tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt b/tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt index f0e7e41f97e..7a80fafd151 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt @@ -39,7 +39,7 @@ actual fun readFile(fileName: String): String { actual fun Double.format(decimalNumber: Int): String { var buffer = ByteArray(1024) snprintf(buffer.refTo(0), buffer.size.toULong(), "%.${decimalNumber}f", this) - return buffer.stringFromUtf8() + return buffer.toKString() } actual fun writeToFile(fileName: String, text: String) { @@ -91,7 +91,7 @@ class CUrl(url: String, user: String? = null, password: String? = null, followLo fun CPointer.toKString(length: Int): String { val bytes = this.readBytes(length) - return bytes.stringFromUtf8() + return bytes.toKString() } fun collectResponse(buffer: CPointer?, size: size_t, nitems: size_t, userdata: COpaquePointer?): size_t {