From cba7319e982ed9ba2dceb517a481cb54ed1b9352 Mon Sep 17 00:00:00 2001 From: ilmat192 Date: Tue, 12 Dec 2017 17:30:50 +0700 Subject: [PATCH] Fix utf8 conversion (#1121) --- .../kotlin/kotlinx/cinterop/NativeUtils.kt | 4 +- backend.native/tests/build.gradle | 7 + backend.native/tests/runtime/text/utf8.kt | 369 ++++++++++++++++++ runtime/src/main/cpp/Console.cpp | 3 +- runtime/src/main/cpp/Exceptions.h | 2 + runtime/src/main/cpp/KString.cpp | 86 +++- runtime/src/main/cpp/KString.h | 2 +- runtime/src/main/cpp/Porting.h | 10 + runtime/src/main/cpp/dtoa/dblparse.cpp | 4 +- runtime/src/main/cpp/dtoa/fltparse.cpp | 4 +- runtime/src/main/cpp/utf8.h | 5 + runtime/src/main/cpp/utf8/checked.h | 14 + runtime/src/main/cpp/utf8/core.h | 10 +- runtime/src/main/cpp/utf8/unchecked.h | 18 +- runtime/src/main/cpp/utf8/with_replacement.h | 148 +++++++ .../kotlin/konan/internal/RuntimeUtils.kt | 8 +- runtime/src/main/kotlin/kotlin/Exceptions.kt | 5 + .../main/kotlin/kotlin/text/StringBuilder.kt | 47 ++- .../src/main/kotlin/org/konan/libcurl/CUrl.kt | 2 +- samples/socket/src/main/kotlin/EchoServer.kt | 2 +- .../src/main/kotlin/DecoderWorker.kt | 2 +- 21 files changed, 709 insertions(+), 43 deletions(-) create mode 100644 backend.native/tests/runtime/text/utf8.kt create mode 100644 runtime/src/main/cpp/utf8/with_replacement.h diff --git a/Interop/Runtime/src/native/kotlin/kotlinx/cinterop/NativeUtils.kt b/Interop/Runtime/src/native/kotlin/kotlinx/cinterop/NativeUtils.kt index c42392d24bd..8637f459dc0 100644 --- a/Interop/Runtime/src/native/kotlin/kotlinx/cinterop/NativeUtils.kt +++ b/Interop/Runtime/src/native/kotlin/kotlinx/cinterop/NativeUtils.kt @@ -18,9 +18,9 @@ package kotlinx.cinterop import konan.internal.Intrinsic -internal fun decodeFromUtf8(bytes: ByteArray): String = kotlin.text.fromUtf8Array(bytes, 0, bytes.size) +internal fun decodeFromUtf8(bytes: ByteArray): String = bytes.stringFromUtf8() -fun encodeToUtf8(str: String): ByteArray = kotlin.text.toUtf8Array(str, 0, str.length) +fun encodeToUtf8(str: String): ByteArray = str.toUtf8() @Intrinsic external fun bitsToFloat(bits: Int): Float diff --git a/backend.native/tests/build.gradle b/backend.native/tests/build.gradle index addbd6ba2c2..0bbbb74fa10 100644 --- a/backend.native/tests/build.gradle +++ b/backend.native/tests/build.gradle @@ -1520,6 +1520,13 @@ task chars0(type: RunKonanTest) { expectedExitStatus = 0 } + +task utf8(type: RunKonanTest) { + expectedFail = (project.testTarget == 'wasm32') // Uses exceptions. + goldValue = "Hello\nПривет\n\uD800\uDC00\n\n\uFFFD\uFFFD\n\uFFFD12\n\uFFFD12\n12\uFFFD\n\uD83D\uDE25\n" + source = "runtime/text/utf8.kt" +} + task catch1(type: RunKonanTest) { expectedFail = (project.testTarget == 'wasm32') // Uses exceptions. goldValue = "Before\nCaught Throwable\nDone\n" diff --git a/backend.native/tests/runtime/text/utf8.kt b/backend.native/tests/runtime/text/utf8.kt new file mode 100644 index 00000000000..b874872ecc8 --- /dev/null +++ b/backend.native/tests/runtime/text/utf8.kt @@ -0,0 +1,369 @@ +package runtime.text.utf8 + +import kotlin.test.* +import kotlin.reflect.KClass + +// region Util +fun assertEquals(expected: ByteArray, actual: ByteArray, message: String) = + assertTrue(expected.contentEquals(actual), message) + +fun checkUtf16to8(string: String, expected: IntArray, conversion: String.() -> ByteArray) { + expected.forEach { + assertTrue(it in Byte.MIN_VALUE..Byte.MAX_VALUE, "Expected array contains illegal values") + } + val expectedBytes = ByteArray(expected.size) { i -> expected[i].toByte() } + val actual = string.conversion() + assertEquals(expectedBytes, actual, """ + Assert failed for string: $string + Expected: ${expected.joinToString()} + Actual: ${actual.joinToString()} + """.trimIndent()) +} + +fun checkUtf16to8Replacing(string: String, expected: IntArray) = checkUtf16to8(string, expected) { toUtf8() } +fun checkUtf16to8Throwing(string: String, expected: IntArray) = checkUtf16to8(string, expected) { toUtf8OrThrow() } + +fun checkUtf16to8Replacing(string: String, expected: IntArray, start: Int, size: Int) = + checkUtf16to8(string, expected) { toUtf8(start, size) } + +fun checkUtf16to8Throwing(string: String, expected: IntArray, start: Int, size: Int) = + checkUtf16to8(string, expected) { toUtf8OrThrow(start, size) } + +fun checkUtf8to16(expected: String, array: IntArray, conversion: ByteArray.() -> String) { + array.forEach { + assertTrue(it in Byte.MIN_VALUE..Byte.MAX_VALUE, "Expected array contains illegal values") + } + val arrayBytes = ByteArray(array.size) { i -> array[i].toByte() } + val actual = arrayBytes.conversion() + assertEquals(expected, actual, """ + Assert failed for string: $expected + Expected: $expected + Actual: $actual + """.trimIndent()) +} + +fun checkUtf8to16Replacing(expected: String, array: IntArray) = checkUtf8to16(expected, array) { stringFromUtf8() } +fun checkUtf8to16Throwing(expected: String, array: IntArray) = checkUtf8to16(expected, array) { stringFromUtf8OrThrow() } + +fun checkUtf8to16Replacing(expected: String, array: IntArray, start: Int, size: Int) = + checkUtf8to16(expected, array) { stringFromUtf8(start, size) } +fun checkUtf8to16Throwing(expected: String, array: IntArray, start: Int, size: Int) = + checkUtf8to16(expected, array) { stringFromUtf8OrThrow(start, size) } + +fun checkThrows(e: KClass, string: String, action: () -> Unit) { + var exception: Throwable? = null + try { + action() + } catch (e: Throwable) { + exception = e + } + assertNotNull(exception, "No excpetion was thrown for string: $string") + assertTrue(e.isInstance(exception),""" + Wrong exception was thrown for string: $string + Expected: ${e.qualifiedName} + Actual: ${exception!!::class.qualifiedName} + """.trimIndent()) +} + +fun checkUtf16to8Throws(string: String) = checkThrows(IllegalCharacterConversionException::class, string) { + string.toUtf8OrThrow() +} + +fun checkUtf16to8Throws(string: String, start: Int, size: Int) = + checkThrows(IllegalCharacterConversionException::class, string) { + string.toUtf8OrThrow(start, size) + } + +fun checkUtf8to16Throws(string: String, array: IntArray) + = checkThrows(IllegalCharacterConversionException::class, string) { + array.forEach { + assertTrue(it in Byte.MIN_VALUE..Byte.MAX_VALUE, "Expected array contains illegal values") + } + val arrayBytes = ByteArray(array.size) { i -> array[i].toByte() } + arrayBytes.stringFromUtf8OrThrow() +} + +fun checkUtf8to16Throws(string: String, array: IntArray, start: Int, size: Int) + = checkThrows(IllegalCharacterConversionException::class, string) { + array.forEach { + assertTrue(it in Byte.MIN_VALUE..Byte.MAX_VALUE, "Expected array contains illegal values") + } + val arrayBytes = ByteArray(array.size) { i -> array[i].toByte() } + arrayBytes.stringFromUtf8OrThrow(start, size) +} + + +fun checkDobuleConversionThrows(string: String) = checkThrows(NumberFormatException::class, string) { + string.toDouble() +} + +fun checkFloatConversionThrows(string: String) = checkThrows(NumberFormatException::class, string) { + string.toFloat() +} +// endregion + +// region UTF-16 -> UTF-8 conversion +fun test16to8() { + // Test manual conversion with replacement. + // Valid strings. + checkUtf16to8Replacing("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt())) + checkUtf16to8Replacing("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126)) + checkUtf16to8Replacing("\uD800\uDC00", intArrayOf(-16, -112, -128, -128)) + checkUtf16to8Replacing("", intArrayOf()) + // Illegal surrogate pair -> replace with default + checkUtf16to8Replacing("\uDC00\uD800", intArrayOf(-17, -65, -67, -17, -65, -67)) + // Different kinds of input + checkUtf16to8Replacing("\uD800\uDC001\uDC00\uD800", + intArrayOf(-16, -112, -128, -128, '1'.toInt(), -17, -65, -67, -17, -65, -67)) + // Lone surrogate - replace with default + checkUtf16to8Replacing("\uD80012", intArrayOf(-17, -65, -67, '1'.toInt(), '2'.toInt())) + checkUtf16to8Replacing("\uDC0012", intArrayOf(-17, -65, -67, '1'.toInt(), '2'.toInt())) + checkUtf16to8Replacing("12\uD800", intArrayOf('1'.toInt(), '2'.toInt(), -17, -65, -67)) + + + // Test manual conversion with an exception if an input is invalid. + // Valid strings. + checkUtf16to8Throwing("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt())) + checkUtf16to8Throwing("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126)) + checkUtf16to8Throwing("\uD800\uDC00", intArrayOf(-16, -112, -128, -128)) + checkUtf16to8Replacing("", intArrayOf()) + + // Illegal surrogate pair -> throw + checkUtf16to8Throws("\uDC00\uD800") + // Different kinds of input (including illegal one) -> throw + checkUtf16to8Throws("\uD800\uDC001\uDC00\uD800") + // Lone surrogate - throw + checkUtf16to8Throws("\uD80012") + checkUtf16to8Throws("\uDC0012") + checkUtf16to8Throws("12\uD800") + + // Test double parsing. + assertEquals(4.2, "4.2".toDouble()) + // Illegal surrogate pair -> throw + checkDobuleConversionThrows("\uDC00\uD800") + // Different kinds of input (including illegal one) -> throw + checkDobuleConversionThrows("\uD800\uDC001\uDC00\uD800") + // Lone surrogate - throw + checkDobuleConversionThrows("\uD80012") + checkDobuleConversionThrows("\uDC0012") + checkDobuleConversionThrows("12\uD800") + + // Test float parsing. + assertEquals(4.2F, "4.2".toFloat()) + // Illegal surrogate pair -> throw + checkFloatConversionThrows("\uDC00\uD800") + // Different kinds of input (including illegal one) -> throw + checkFloatConversionThrows("\uD800\uDC001\uDC00\uD800") + // Lone surrogate - throw + checkFloatConversionThrows("\uD80012") + checkFloatConversionThrows("\uDC0012") + checkFloatConversionThrows("12\uD800") +} + +fun test16to8CustomBorders() { + // Test manual conversion with replacement and custom borders. + // Valid strings. + checkUtf16to8Replacing("Hello!", intArrayOf('H'.toInt(), 'e'.toInt()), 0, 2) + checkUtf16to8Replacing("Hello!", intArrayOf('e'.toInt(), 'l'.toInt()), 1, 2) + checkUtf16to8Replacing("Hello!", intArrayOf('o'.toInt(), '!'.toInt()), 4, 2) + checkUtf16to8Replacing("Hello!", intArrayOf(), 0, 0) + checkUtf16to8Replacing("Hello!", intArrayOf(), 10, 0) + + checkUtf16to8Replacing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00", + intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 0, 4) + checkUtf16to8Replacing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00", + intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 2, 4) + checkUtf16to8Replacing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00", + intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 4, 4) + + // Illegal surrogate pair -> replace with default + checkUtf16to8Replacing("\uDC00\uD80012", + intArrayOf(-17, -65, -67, -17, -65, -67, '1'.toInt()), 0, 3) + checkUtf16to8Replacing("1\uDC00\uD8002", + intArrayOf(-17, -65, -67, -17, -65, -67, '2'.toInt()), 1, 3) + checkUtf16to8Replacing("12\uDC00\uD800", + intArrayOf('2'.toInt(), -17, -65, -67, -17, -65, -67), 1, 3) + + // Lone surrogate - replace with default + checkUtf16to8Replacing("1\uD800\uDC002", intArrayOf('1'.toInt(), -17, -65, -67), 0, 2) + checkUtf16to8Replacing("1\uD800\uDC002", intArrayOf(-17, -65, -67, '2'.toInt()), 2, 2) + + // Index out of bound + checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8(-1, 4)} + checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8(5, 10)} + checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8(2, 10)} + checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8(3, -2)} + + + // Test manual conversion with an exception if an input is invalid and custom borders. + // Valid strings. + checkUtf16to8Throwing("Hello!", intArrayOf('H'.toInt(), 'e'.toInt()), 0, 2) + checkUtf16to8Throwing("Hello!", intArrayOf('e'.toInt(), 'l'.toInt()), 1, 2) + checkUtf16to8Throwing("Hello!", intArrayOf('o'.toInt(), '!'.toInt()), 4, 2) + checkUtf16to8Throwing("Hello!", intArrayOf(), 0, 0) + checkUtf16to8Throwing("Hello!", intArrayOf(), 10, 0) + + checkUtf16to8Throwing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00", + intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 0, 4) + checkUtf16to8Throwing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00", + intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 2, 4) + checkUtf16to8Throwing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00", + intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 4, 4) + + // Illegal surrogate pair -> throw + checkUtf16to8Throws("\uDC00\uD80012", 0, 3) + checkUtf16to8Throws("1\uDC00\uD8002", 1, 3) + checkUtf16to8Throws("12\uDC00\uD800", 1, 3) + + // Lone surrogate -> throw + checkUtf16to8Throws("1\uD800\uDC002", 0, 2) + checkUtf16to8Throws("1\uD800\uDC002", 2, 2) + + // Index out of bound + checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8OrThrow(-1, 4)} + checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8OrThrow(5, 10)} + checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8OrThrow(2, 10)} + checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8OrThrow(3, -2)} +} + +fun testPrint() { + // Valid strings. + println("Hello") + println("Привет") + println("\uD800\uDC00") + println("") + + // Illegal surrogate pair -> default output + println("\uDC00\uD800") + // Lone surrogate -> default output + println("\uD80012") + println("\uDC0012") + println("12\uD800") + + // https://github.com/JetBrains/kotlin-native/issues/1091 + val array = byteArrayOf(0xF0.toByte(), 0x9F.toByte(), 0x98.toByte(), 0xA5.toByte()) + val badStr = array.stringFromUtf8() + assertEquals(2, badStr.length) + println(badStr) +} +// endregion + +// region UTF-8 -> UTF-16 conversion +fun test8to16() { + // Test manual conversion with replacement. + // Valid strings. + checkUtf8to16Replacing("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt())) + checkUtf8to16Replacing("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126)) + checkUtf8to16Replacing("\uD800\uDC00", intArrayOf(-16, -112, -128, -128)) + checkUtf8to16Replacing("", intArrayOf()) + + // Incorrect UTF-8 lead character. + checkUtf8to16Replacing("\uFFFD1", intArrayOf(-1, '1'.toInt())) + + // Incomplete codepoint. + checkUtf8to16Replacing("\uFFFD1", intArrayOf(-16, -97, -104, '1'.toInt())) + checkUtf8to16Replacing("\uFFFD1\uFFFD", intArrayOf(-16, -97, -104, '1'.toInt(), -16, -97, -104)) + + // Test manual conversion with exception throwing + // Valid strings. + checkUtf8to16Throwing("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt())) + checkUtf8to16Throwing("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126)) + checkUtf8to16Throwing("\uD800\uDC00", intArrayOf(-16, -112, -128, -128)) + checkUtf8to16Throwing("", intArrayOf()) + + // Incorrect UTF-8 lead character -> throw. + checkUtf8to16Throws("\uFFFD1", intArrayOf(-1, '1'.toInt())) + + // Incomplete codepoint -> throw. + checkUtf8to16Throws("\uFFFD1", intArrayOf(-16, -97, -104, '1'.toInt())) + checkUtf8to16Throws("\uFFFD1\uFFFD", intArrayOf(-16, -97, -104, '1'.toInt(), -16, -97, -104)) +} + +fun test8to16CustomBorders() { + // Conversion with replacement. + checkUtf8to16Replacing("He", + intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()),0, 2) + checkUtf8to16Replacing("ll", + intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 2, 2) + checkUtf8to16Replacing("lo", + intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 3, 2) + checkUtf8to16Replacing("", + intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 0, 0) + checkUtf8to16Replacing("", + intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 10, 0) + + checkUtf8to16Replacing("\uD800\uDC00\uD800\uDC00", + intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128), + 0, 8) + checkUtf8to16Replacing("\uD800\uDC00\uD800\uDC00", + intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128), + 4, 8) + checkUtf8to16Replacing("\uD800\uDC00\uD800\uDC00", + intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128), + 8, 8) + + // Incorrect UTF-8 lead character. + checkUtf8to16Replacing("\uFFFD1", intArrayOf(-1, '1'.toInt(), '2'.toInt()), 0, 2) + checkUtf8to16Replacing("\uFFFD2", intArrayOf('1'.toInt(), -1, '2'.toInt(), '3'.toInt()), 1, 2) + checkUtf8to16Replacing("2\uFFFD", intArrayOf('1'.toInt(), '2'.toInt(), -1), 1, 2) + + // Incomplete codepoint. + checkUtf8to16Replacing("\uFFFD1", intArrayOf(-16, -97, -104, '1'.toInt(), '2'.toInt()), 0, 4) + checkUtf8to16Replacing("\uFFFD2", intArrayOf('1'.toInt(), -16, -97, -104, '2'.toInt(), '3'.toInt()), 1, 4) + checkUtf8to16Replacing("2\uFFFD", intArrayOf('1'.toInt(), '2'.toInt(), -16, -97, -104), 1, 4) + + + // Index out of bound + val helloArray = byteArrayOf('H'.toByte(), 'e'.toByte(), 'l'.toByte(), 'l'.toByte(), 'o'.toByte()) + checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8(-1, 4)} + checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8(5, 10)} + checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8(2, 10)} + checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8(3, -2)} + + // Conversion with throwing + checkUtf8to16Throwing("He", + intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()),0, 2) + checkUtf8to16Throwing("ll", + intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 2, 2) + checkUtf8to16Throwing("lo", + intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 3, 2) + checkUtf8to16Throwing("", + intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 0, 0) + checkUtf8to16Throwing("", + intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 10, 0) + + checkUtf8to16Throwing("\uD800\uDC00\uD800\uDC00", + intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128), + 0, 8) + checkUtf8to16Throwing("\uD800\uDC00\uD800\uDC00", + intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128), + 4, 8) + checkUtf8to16Throwing("\uD800\uDC00\uD800\uDC00", + intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128), + 8, 8) + + // Incorrect UTF-8 lead character. + checkUtf8to16Throws("\uFFFD1", intArrayOf(-1, '1'.toInt(), '2'.toInt()), 0, 2) + checkUtf8to16Throws("\uFFFD2", intArrayOf('1'.toInt(), -1, '2'.toInt(), '3'.toInt()), 1, 2) + checkUtf8to16Throws("2\uFFFD", intArrayOf('1'.toInt(), '2'.toInt(), -1), 1, 2) + + // Incomplete codepoint. + checkUtf8to16Throws("\uFFFD1", intArrayOf(-16, -97, -104, '1'.toInt(), '2'.toInt()), 0, 4) + checkUtf8to16Throws("\uFFFD2", intArrayOf('1'.toInt(), -16, -97, -104, '2'.toInt(), '3'.toInt()), 1, 4) + checkUtf8to16Throws("2\uFFFD", intArrayOf('1'.toInt(), '2'.toInt(), -16, -97, -104), 1, 4) + + // Index out of bound + checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8OrThrow(-1, 4)} + checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8OrThrow(5, 10)} + checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8OrThrow(2, 10)} + checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8OrThrow(3, -2)} +} +// endregion + +@Test fun runTest() { + test16to8() + test16to8CustomBorders() + test8to16() + test8to16CustomBorders() + testPrint() +} diff --git a/runtime/src/main/cpp/Console.cpp b/runtime/src/main/cpp/Console.cpp index a79298e9d5b..a80fa0b9717 100644 --- a/runtime/src/main/cpp/Console.cpp +++ b/runtime/src/main/cpp/Console.cpp @@ -30,7 +30,8 @@ void Kotlin_io_Console_print(KString message) { // TODO: system stdout must be aware about UTF-8. const KChar* utf16 = CharArrayAddressOfElementAt(message, 0); KStdString utf8; - utf8::unchecked::utf16to8(utf16, utf16 + message->count_, back_inserter(utf8)); + // Replace incorrect sequences with a default codepoint (see utf8::with_replacement::default_replacement) + utf8::with_replacement::utf16to8(utf16, utf16 + message->count_, back_inserter(utf8)); konan::consoleWriteUtf8(utf8.c_str(), utf8.size()); } diff --git a/runtime/src/main/cpp/Exceptions.h b/runtime/src/main/cpp/Exceptions.h index 9f3b106b4ff..28cf84c6985 100644 --- a/runtime/src/main/cpp/Exceptions.h +++ b/runtime/src/main/cpp/Exceptions.h @@ -48,6 +48,8 @@ void ThrowNumberFormatException(); void ThrowOutOfMemoryError(); // Throws not implemented error. void ThrowNotImplementedError(); +// Throws illegal character conversion exception (used in UTF8/UTF16 conversions). +void ThrowIllegalCharacterConversionException(); // Prints out mesage of Throwable. void PrintThrowable(KRef); diff --git a/runtime/src/main/cpp/KString.cpp b/runtime/src/main/cpp/KString.cpp index 6af0d2cf41e..5dcd9be4920 100644 --- a/runtime/src/main/cpp/KString.cpp +++ b/runtime/src/main/cpp/KString.cpp @@ -37,16 +37,55 @@ namespace { -OBJ_GETTER(utf8ToUtf16, const char* rawString, size_t rawStringLength) { - uint32_t charCount = utf8::unchecked::distance(rawString, rawString + rawStringLength); - ArrayHeader* result = AllocArrayInstance( - theStringTypeInfo, charCount, OBJ_RESULT)->array(); +typedef std::back_insert_iterator KStdStringInserter; +typedef KChar* utf8to16(const char*, const char*, KChar*); +typedef KStdStringInserter utf16to8(const KChar*,const KChar*, KStdStringInserter); + +KStdStringInserter utf16toUtf8OrThrow(const KChar* start, const KChar* end, KStdStringInserter result) { + TRY_CATCH(result = utf8::utf16to8(start, end, result), + result = utf8::unchecked::utf16to8(start, end, result), + ThrowIllegalCharacterConversionException()); + return result; +} + +template +OBJ_GETTER(utf8ToUtf16Impl, const char* rawString, const char* end, uint32_t charCount) { + ArrayHeader* result = AllocArrayInstance(theStringTypeInfo, charCount, OBJ_RESULT)->array(); KChar* rawResult = CharArrayAddressOfElementAt(result, 0); - auto convertResult = - utf8::unchecked::utf8to16(rawString, rawString + rawStringLength, rawResult); + auto convertResult = conversion(rawString, end, rawResult); RETURN_OBJ(result->obj()); } +template +OBJ_GETTER(utf16ToUtf8Impl, KString thiz, KInt start, KInt size) { + RuntimeAssert(thiz->type_info() == theStringTypeInfo, "Must use String"); + if (start < 0 || size < 0 || size > thiz->count_ - start) { + ThrowArrayIndexOutOfBoundsException(); + } + const KChar* utf16 = CharArrayAddressOfElementAt(thiz, start); + KStdString utf8; + conversion(utf16, utf16 + size, back_inserter(utf8)); + ArrayHeader* result = AllocArrayInstance(theByteArrayTypeInfo, utf8.size(), OBJ_RESULT)->array(); + ::memcpy(ByteArrayAddressOfElementAt(result, 0), utf8.c_str(), utf8.size()); + RETURN_OBJ(result->obj()); +} + +OBJ_GETTER(utf8ToUtf16OrThrow, const char* rawString, size_t rawStringLength) { + const char* end = rawString + rawStringLength; + uint32_t charCount; + TRY_CATCH(charCount = utf8::utf16_length(rawString, end), + charCount = utf8::unchecked::utf16_length(rawString, end), + ThrowIllegalCharacterConversionException()); + RETURN_RESULT_OF(utf8ToUtf16Impl, rawString, end, charCount); +} + +OBJ_GETTER(utf8ToUtf16, const char* rawString, size_t rawStringLength) { + const char* end = rawString + rawStringLength; + uint32_t charCount = utf8::with_replacement::utf16_length(rawString, end); + RETURN_RESULT_OF(utf8ToUtf16Impl, rawString, end, charCount); +} + + // Case conversion is derived work from Apache Harmony. // Unicode 3.0.1 (same as Unicode 3.0.0) enum CharacterClass { @@ -731,32 +770,37 @@ KInt Kotlin_String_getStringLength(KString thiz) { return thiz->count_; } -OBJ_GETTER(Kotlin_String_fromUtf8Array, KConstRef thiz, KInt start, KInt size) { +const char* byteArrayAsCString(KConstRef thiz, KInt start, KInt size) { const ArrayHeader* array = thiz->array(); RuntimeAssert(array->type_info() == theByteArrayTypeInfo, "Must use a byte array"); if (start < 0 || size < 0 || size > array->count_ - start) { ThrowArrayIndexOutOfBoundsException(); } + return reinterpret_cast(ByteArrayAddressOfElementAt(array, start)); +} + +OBJ_GETTER(Kotlin_ByteArray_stringFromUtf8OrThrow, KConstRef thiz, KInt start, KInt size) { + const char* rawString = byteArrayAsCString(thiz, start, size); + if (size == 0) { + RETURN_RESULT_OF0(TheEmptyString); + } + RETURN_RESULT_OF(utf8ToUtf16OrThrow, rawString, size); +} + +OBJ_GETTER(Kotlin_ByteArray_stringFromUtf8, KConstRef thiz, KInt start, KInt size) { + const char* rawString = byteArrayAsCString(thiz, start, size); if (size == 0) { RETURN_RESULT_OF0(TheEmptyString); } - const char* rawString = - reinterpret_cast(ByteArrayAddressOfElementAt(array, start)); RETURN_RESULT_OF(utf8ToUtf16, rawString, size); } -OBJ_GETTER(Kotlin_String_toUtf8Array, KString thiz, KInt start, KInt size) { - RuntimeAssert(thiz->type_info() == theStringTypeInfo, "Must use String"); - if (start < 0 || size < 0 || size > thiz->count_ - start) { - ThrowArrayIndexOutOfBoundsException(); - } - const KChar* utf16 = CharArrayAddressOfElementAt(thiz, start); - KStdString utf8; - utf8::unchecked::utf16to8(utf16, utf16 + size, back_inserter(utf8)); - ArrayHeader* result = AllocArrayInstance( - theByteArrayTypeInfo, utf8.size(), OBJ_RESULT)->array(); - ::memcpy(ByteArrayAddressOfElementAt(result, 0), utf8.c_str(), utf8.size()); - RETURN_OBJ(result->obj()); +OBJ_GETTER(Kotlin_String_toUtf8, KString thiz, KInt start, KInt size) { + RETURN_RESULT_OF(utf16ToUtf8Impl, thiz, start, size); +} + +OBJ_GETTER(Kotlin_String_toUtf8OrThrow, KString thiz, KInt start, KInt size) { + RETURN_RESULT_OF(utf16ToUtf8Impl, thiz, start, size); } OBJ_GETTER(Kotlin_String_fromCharArray, KConstRef thiz, KInt start, KInt size) { diff --git a/runtime/src/main/cpp/KString.h b/runtime/src/main/cpp/KString.h index 75b69aea599..eaf8139ae6f 100644 --- a/runtime/src/main/cpp/KString.h +++ b/runtime/src/main/cpp/KString.h @@ -27,7 +27,7 @@ extern "C" { #endif OBJ_GETTER(CreateStringFromCString, const char* cstring); -OBJ_GETTER(CreateStringFromUtf8, const char* utf8, uint32_t size); +OBJ_GETTER(CreateStringFromUtf8, const char* utf8, uint32_t lengthBytes); char* CreateCStringFromString(KConstRef kstring); void DisposeCString(char* cstring); diff --git a/runtime/src/main/cpp/Porting.h b/runtime/src/main/cpp/Porting.h index 9c6664409f3..f027e689d09 100644 --- a/runtime/src/main/cpp/Porting.h +++ b/runtime/src/main/cpp/Porting.h @@ -52,6 +52,16 @@ uint64_t getTimeMillis(); uint64_t getTimeMicros(); uint64_t getTimeNanos(); +#if KONAN_NO_EXCEPTIONS +#define TRY_CATCH(tryAction, actionWithoutExceptions, catchAction) actionWithoutExceptions; +#else +#define TRY_CATCH(tryAction, actionWithoutExceptions, catchAction) \ +do { \ + try { tryAction; } \ + catch(...) { catchAction; } \ +} while(0) +#endif + } // namespace konan #endif // RUNTIME_PORTING_H diff --git a/runtime/src/main/cpp/dtoa/dblparse.cpp b/runtime/src/main/cpp/dtoa/dblparse.cpp index 4e1834048a2..49407e46e7d 100644 --- a/runtime/src/main/cpp/dtoa/dblparse.cpp +++ b/runtime/src/main/cpp/dtoa/dblparse.cpp @@ -642,7 +642,9 @@ KDouble Konan_FloatingPointParser_parseDoubleImpl (KString s, KInt e) { const KChar* utf16 = CharArrayAddressOfElementAt(s, 0); KStdString utf8; - utf8::unchecked::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8)); + TRY_CATCH(utf8::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8)), + utf8::unchecked::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8)), + /* Illegal UTF-16 string. */ ThrowNumberFormatException()); const char *str = utf8.c_str(); auto dbl = createDouble (str, e); diff --git a/runtime/src/main/cpp/dtoa/fltparse.cpp b/runtime/src/main/cpp/dtoa/fltparse.cpp index cd26c2e534f..5988121e6fe 100644 --- a/runtime/src/main/cpp/dtoa/fltparse.cpp +++ b/runtime/src/main/cpp/dtoa/fltparse.cpp @@ -542,7 +542,9 @@ Konan_FloatingPointParser_parseFloatImpl(KString s, KInt e) { const KChar* utf16 = CharArrayAddressOfElementAt(s, 0); KStdString utf8; - utf8::unchecked::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8)); + TRY_CATCH(utf8::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8)), + utf8::unchecked::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8)), + /* Illegal UTF-16 string. */ ThrowNumberFormatException()); const char *str = utf8.c_str(); auto flt = createFloat(str, e); diff --git a/runtime/src/main/cpp/utf8.h b/runtime/src/main/cpp/utf8.h index 383db296139..8c7a1dc694c 100644 --- a/runtime/src/main/cpp/utf8.h +++ b/runtime/src/main/cpp/utf8.h @@ -29,5 +29,10 @@ DEALINGS IN THE SOFTWARE. #define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 #include "utf8/unchecked.h" +#include "utf8/with_replacement.h" + +#if !KONAN_NO_EXCEPTIONS +#include "utf8/checked.h" +#endif #endif // header guard diff --git a/runtime/src/main/cpp/utf8/checked.h b/runtime/src/main/cpp/utf8/checked.h index 13311551381..32bde58a090 100644 --- a/runtime/src/main/cpp/utf8/checked.h +++ b/runtime/src/main/cpp/utf8/checked.h @@ -193,6 +193,20 @@ namespace utf8 utf8::next(it, end); } + /** + * Calculates a count of characters needed to represent the string from first to last in UTF-16 + * taking into account surrogate symbols. Throws an exception if the input is invalid. + */ + template + uint32_t utf16_length(octet_iterator first, octet_iterator last) { + uint32_t dist = 0; + while(first < last) { + uint32_t cp = utf8::next(first, last); + dist += (cp > 0xffff) ? 2 : 1; + } + return dist; + } + template typename std::iterator_traits::difference_type distance (octet_iterator first, octet_iterator last) diff --git a/runtime/src/main/cpp/utf8/core.h b/runtime/src/main/cpp/utf8/core.h index 693d388c078..cd4c50f32c5 100644 --- a/runtime/src/main/cpp/utf8/core.h +++ b/runtime/src/main/cpp/utf8/core.h @@ -150,7 +150,7 @@ namespace internal /// get_sequence_x functions decode utf-8 sequences of the length x template - utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point) + utf_error get_sequence_1(octet_iterator& it, const octet_iterator end, uint32_t& code_point) { if (it == end) return NOT_ENOUGH_ROOM; @@ -161,7 +161,7 @@ namespace internal } template - utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point) + utf_error get_sequence_2(octet_iterator& it, const octet_iterator end, uint32_t& code_point) { if (it == end) return NOT_ENOUGH_ROOM; @@ -176,7 +176,7 @@ namespace internal } template - utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point) + utf_error get_sequence_3(octet_iterator& it, const octet_iterator end, uint32_t& code_point) { if (it == end) return NOT_ENOUGH_ROOM; @@ -195,7 +195,7 @@ namespace internal } template - utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point) + utf_error get_sequence_4(octet_iterator& it, const octet_iterator end, uint32_t& code_point) { if (it == end) return NOT_ENOUGH_ROOM; @@ -220,7 +220,7 @@ namespace internal #undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR template - utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point) + utf_error validate_next(octet_iterator& it, const octet_iterator end, uint32_t& code_point) { // Save the original value of it so we can go back in case of failure // Of course, it does not make much sense with i.e. stream iterators diff --git a/runtime/src/main/cpp/utf8/unchecked.h b/runtime/src/main/cpp/utf8/unchecked.h index cb2427166b1..4381f6c395e 100644 --- a/runtime/src/main/cpp/utf8/unchecked.h +++ b/runtime/src/main/cpp/utf8/unchecked.h @@ -116,6 +116,20 @@ namespace utf8 utf8::unchecked::next(it); } + /** + * Calculates a count of characters needed to represent the string from first to last in UTF-16 + * taking into account surrogate symbols. Doesn't validate the input. + */ + template + uint32_t utf16_length(octet_iterator first, const octet_iterator last) { + uint32_t dist = 0; + while (first < last) { + uint32_t cp = utf8::unchecked::next(first); + dist += (cp > 0xffff) ? 2 : 1; + } + return dist; + } + template typename std::iterator_traits::difference_type distance (octet_iterator first, octet_iterator last) @@ -127,7 +141,7 @@ namespace utf8 } template - octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + octet_iterator utf16to8 (u16bit_iterator start, const u16bit_iterator end, octet_iterator result) { while (start != end) { uint32_t cp = utf8::internal::mask16(*start++); @@ -142,7 +156,7 @@ namespace utf8 } template - u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + u16bit_iterator utf8to16 (octet_iterator start, const octet_iterator end, u16bit_iterator result) { while (start < end) { uint32_t cp = utf8::unchecked::next(start); diff --git a/runtime/src/main/cpp/utf8/with_replacement.h b/runtime/src/main/cpp/utf8/with_replacement.h new file mode 100644 index 00000000000..4807006c39d --- /dev/null +++ b/runtime/src/main/cpp/utf8/with_replacement.h @@ -0,0 +1,148 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef UTF_FOR_CPP_WITH_REPLACEMENT_H +#define UTF_FOR_CPP_WITH_REPLACEMENT_H + +#include "core.h" +#include "unchecked.h" + +namespace utf8 { + +namespace with_replacement { + +constexpr uint32_t default_replacement = 0xfffd; + +/* + * Returns the next codepoint replacing any invalid sequence with the replacement codepoint. + */ +template +uint32_t next(octet_iterator &it, const octet_iterator end, uint32_t replacement) { + uint32_t cp = 0; + internal::utf_error err_code = utf8::internal::validate_next(it, end, cp); + switch (err_code) { + case internal::UTF8_OK : + return cp; + case internal::INVALID_LEAD : + it++; + return replacement; + case internal::NOT_ENOUGH_ROOM : + case internal::INCOMPLETE_SEQUENCE : + case internal::OVERLONG_SEQUENCE : + case internal::INVALID_CODE_POINT : + // The whole invalid sequence is replaced with one replacement codepoint. + for (it++; it < end && utf8::internal::is_trail(*it); it++); + return replacement; + } +} + +// Library API + +/** + * Calculates a count of characters needed to represent the string from first to last in UTF-16 + * taking into account surrogate symbols and invalid sequences. + * Assumes that all invalid sequences in the input will be replaced with `replacement` + * so each invalid sequence increases the result by 1 or 2 depending on `replacement`. + */ +template +uint32_t utf16_length(octet_iterator first, + const octet_iterator last, + const uint32_t replacement = default_replacement) { + uint32_t dist = 0; + while (first < last) { + uint32_t cp = next(first, last, replacement); + dist += (cp > 0xffff) ? 2 : 1; + } + return dist; +} + + +template +typename std::iterator_traits::difference_type +distance(octet_iterator first, const octet_iterator last) { + typename std::iterator_traits::difference_type dist; + uint32_t unused = 0; + for (dist = 0; first < last; dist++) { + next(first, last, unused); + } + return dist; +} + +template +octet_iterator utf16to8(u16bit_iterator start, + const u16bit_iterator end, + octet_iterator result, + const uint32_t replacement) { + while (start != end) { + uint32_t cp = utf8::internal::mask16(*start++); + // Process surrogates. + if (utf8::internal::is_lead_surrogate(cp)) { + if (start != end) { + uint32_t trail_surrogate = utf8::internal::mask16(*start); + if (utf8::internal::is_trail_surrogate(trail_surrogate)) { + // Valid surrogate pair. + cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; + start++; + } else { + cp = replacement; // Invalid input: lone lead surrogate. + } + } else { + cp = replacement; // Invalid input: lone lead surrogate at the end of input. + } + } else if (utf8::internal::is_trail_surrogate(cp)) { + cp = replacement; // Invalid input: lone trail surrogate + } + result = utf8::unchecked::append(cp, result); + } + return result; +} + +template +inline octet_iterator utf16to8(u16bit_iterator start, + const u16bit_iterator end, + octet_iterator result) { + return utf16to8(start, end, result, default_replacement); +} + +template +u16bit_iterator utf8to16(octet_iterator start, + const octet_iterator end, + u16bit_iterator result, + const uint32_t replacement) { + while (start != end) { + // The `next` method takes care about replacing invalid sequences. + uint32_t cp = next(start, end, replacement); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + } else + *result++ = static_cast(cp); + } + return result; +} + +template +inline u16bit_iterator utf8to16(octet_iterator start, + const octet_iterator end, + u16bit_iterator result) { + return utf8to16(start, end, result, default_replacement); +} + +} // namespace with_replacement + +} // namespace utf8 + +#endif // UTF_FOR_CPP_WITH_REPLACEMENT_H diff --git a/runtime/src/main/kotlin/konan/internal/RuntimeUtils.kt b/runtime/src/main/kotlin/konan/internal/RuntimeUtils.kt index 14ba32c1f98..4e9211d2c88 100644 --- a/runtime/src/main/kotlin/konan/internal/RuntimeUtils.kt +++ b/runtime/src/main/kotlin/konan/internal/RuntimeUtils.kt @@ -17,7 +17,6 @@ package konan.internal import kotlin.internal.getProgressionLastElement -import kotlin.text.toUtf8Array @ExportForCppRuntime fun ThrowNullPointerException(): Nothing { @@ -66,6 +65,11 @@ internal fun ThrowNotImplementedError(): Nothing { throw NotImplementedError("An operation is not implemented.") } +@ExportForCppRuntime +internal fun ThrowIllegalCharacterConversionException(): Nothing { + throw IllegalCharacterConversionException() +} + @ExportForCppRuntime fun PrintThrowable(throwable: Throwable) { println(throwable) @@ -140,5 +144,5 @@ fun KonanObjectToUtf8Array(value: Any?): ByteArray { is DoubleArray -> value.contentToString() else -> value.toString() } - return toUtf8Array(string, 0, string.length) + return string.toUtf8() } diff --git a/runtime/src/main/kotlin/kotlin/Exceptions.kt b/runtime/src/main/kotlin/kotlin/Exceptions.kt index ec0d36151ee..49587514eb2 100644 --- a/runtime/src/main/kotlin/kotlin/Exceptions.kt +++ b/runtime/src/main/kotlin/kotlin/Exceptions.kt @@ -216,4 +216,9 @@ public open class NumberFormatException : IllegalArgumentException { } constructor(s: String) : super(s) {} +} + +public open class IllegalCharacterConversionException : IllegalArgumentException { + constructor(): super() + constructor(s: String) : super(s) } \ No newline at end of file diff --git a/runtime/src/main/kotlin/kotlin/text/StringBuilder.kt b/runtime/src/main/kotlin/kotlin/text/StringBuilder.kt index 0f934a5d8ec..e3741496e06 100644 --- a/runtime/src/main/kotlin/kotlin/text/StringBuilder.kt +++ b/runtime/src/main/kotlin/kotlin/text/StringBuilder.kt @@ -16,11 +16,50 @@ package kotlin.text -@SymbolName("Kotlin_String_fromUtf8Array") -external fun fromUtf8Array(array: ByteArray, start: Int, size: Int) : String +// ByteArray -> String (UTF-8 -> UTF-16) -@SymbolName("Kotlin_String_toUtf8Array") -external fun toUtf8Array(string: String, start: Int, size: Int) : ByteArray +@Deprecated("Use ByteArray.stringFromUtf8()", ReplaceWith("array.stringFromUtf8(start, end)")) +fun fromUtf8Array(array: ByteArray, start: Int, size: Int) = array.stringFromUtf8Impl(start, size) + +@Deprecated("Use String.toUtf8()", ReplaceWith("string.toUtf8(start, end)")) +fun toUtf8Array(string: String, start: Int, size: Int) : ByteArray = string.toUtf8(start, size) + +/** + * Converts an UTF-8 array into a [String]. Replaces invalid input sequences with a default character. + */ +fun ByteArray.stringFromUtf8(start: Int = 0, size: Int = this.size) : String = + stringFromUtf8Impl(start, size) + +@SymbolName("Kotlin_ByteArray_stringFromUtf8") +private external fun ByteArray.stringFromUtf8Impl(start: Int, size: Int) : String + +/** + * Converts an UTF-8 array into a [String]. Throws [IllegalCharacterConversionException] if the input is invalid. + */ +fun ByteArray.stringFromUtf8OrThrow(start: Int = 0, size: Int = this.size) : String = + stringFromUtf8OrThrowImpl(start, size) + +@SymbolName("Kotlin_ByteArray_stringFromUtf8OrThrow") +private external fun ByteArray.stringFromUtf8OrThrowImpl(start: Int, size: Int) : String + +// String -> ByteArray (UTF-16 -> UTF-8) +/** + * Converts a [String] into an UTF-8 array. Replaces invalid input sequences with a default character. + */ +fun String.toUtf8(start: Int = 0, size: Int = this.length) : ByteArray = + toUtf8Impl(start, size) + +@SymbolName("Kotlin_String_toUtf8") +private external fun String.toUtf8Impl(start: Int, size: Int) : ByteArray + +/** + * Converts a [String] into an UTF-8 array. Throws [IllegalCharacterConversionException] if the input is invalid. + */ +fun String.toUtf8OrThrow(start: Int = 0, size: Int = this.length) : ByteArray = + toUtf8OrThrowImpl(start, size) + +@SymbolName("Kotlin_String_toUtf8OrThrow") +private external fun String.toUtf8OrThrowImpl(start: Int, size: Int) : ByteArray // TODO: make it somewhat private? @SymbolName("Kotlin_String_fromCharArray") diff --git a/samples/libcurl/src/main/kotlin/org/konan/libcurl/CUrl.kt b/samples/libcurl/src/main/kotlin/org/konan/libcurl/CUrl.kt index 876e3efa090..271a2ffbd72 100644 --- a/samples/libcurl/src/main/kotlin/org/konan/libcurl/CUrl.kt +++ b/samples/libcurl/src/main/kotlin/org/konan/libcurl/CUrl.kt @@ -33,7 +33,7 @@ class CUrl(val url: String) { fun CPointer.toKString(length: Int): String { val bytes = this.readBytes(length) - return kotlin.text.fromUtf8Array(bytes, 0, bytes.size) + return bytes.stringFromUtf8() } fun header_callback(buffer: CPointer?, size: size_t, nitems: size_t, userdata: COpaquePointer?): size_t { diff --git a/samples/socket/src/main/kotlin/EchoServer.kt b/samples/socket/src/main/kotlin/EchoServer.kt index fa8c677c9a4..8ad80cb6c67 100644 --- a/samples/socket/src/main/kotlin/EchoServer.kt +++ b/samples/socket/src/main/kotlin/EchoServer.kt @@ -31,7 +31,7 @@ fun main(args: Array) { memScoped { val buffer = ByteArray(1024) - val prefixBuffer = kotlin.text.toUtf8Array("echo: ", 0, 6) + val prefixBuffer = "echo: ".toUtf8() val serverAddr = alloc() val listenFd = socket(AF_INET, SOCK_STREAM, 0) diff --git a/samples/videoplayer/src/main/kotlin/DecoderWorker.kt b/samples/videoplayer/src/main/kotlin/DecoderWorker.kt index 2a596d3519b..ce79eade209 100644 --- a/samples/videoplayer/src/main/kotlin/DecoderWorker.kt +++ b/samples/videoplayer/src/main/kotlin/DecoderWorker.kt @@ -43,7 +43,7 @@ private fun Int.checkError() { if (this != 0) { val buffer = ByteArray(1024) av_strerror(this, buffer.refTo(0), buffer.size.signExtend()) - throw Error("AVError: ${kotlin.text.fromUtf8Array(buffer, 0, buffer.size)}") + throw Error("AVError: ${buffer.stringFromUtf8()}") } }