Fix utf8 conversion (#1121)

Interop/Runtime/src/native/kotlin/kotlinx/cinterop/NativeUtils.kt

@@ -18,9 +18,9 @@ package kotlinx.cinterop
 
 import konan.internal.Intrinsic
 
-internal fun decodeFromUtf8(bytes: ByteArray): String = kotlin.text.fromUtf8Array(bytes, 0, bytes.size)
+internal fun decodeFromUtf8(bytes: ByteArray): String = bytes.stringFromUtf8()
 
-fun encodeToUtf8(str: String): ByteArray = kotlin.text.toUtf8Array(str, 0, str.length)
+fun encodeToUtf8(str: String): ByteArray = str.toUtf8()
 
 @Intrinsic
 external fun bitsToFloat(bits: Int): Float

backend.native/tests/build.gradle

@@ -1520,6 +1520,13 @@ task chars0(type: RunKonanTest) {
     expectedExitStatus = 0
 }
 
+
+task utf8(type: RunKonanTest) {
+    expectedFail = (project.testTarget == 'wasm32') // Uses exceptions.
+    goldValue = "Hello\nПривет\n\uD800\uDC00\n\n\uFFFD\uFFFD\n\uFFFD12\n\uFFFD12\n12\uFFFD\n\uD83D\uDE25\n"
+    source = "runtime/text/utf8.kt"
+}
+
 task catch1(type: RunKonanTest) {
     expectedFail = (project.testTarget == 'wasm32') // Uses exceptions.
     goldValue = "Before\nCaught Throwable\nDone\n"

backend.native/tests/runtime/text/utf8.kt

@@ -0,0 +1,369 @@
+package runtime.text.utf8
+
+import kotlin.test.*
+import kotlin.reflect.KClass
+
+// region Util
+fun assertEquals(expected: ByteArray, actual: ByteArray, message: String) =
+        assertTrue(expected.contentEquals(actual), message)
+
+fun checkUtf16to8(string: String, expected: IntArray, conversion: String.() -> ByteArray) {
+    expected.forEach {
+        assertTrue(it in Byte.MIN_VALUE..Byte.MAX_VALUE, "Expected array contains illegal values")
+    }
+    val expectedBytes = ByteArray(expected.size) { i -> expected[i].toByte() }
+    val actual = string.conversion()
+    assertEquals(expectedBytes, actual, """
+        Assert failed for string: $string
+        Expected: ${expected.joinToString()}
+        Actual: ${actual.joinToString()}
+    """.trimIndent())
+}
+
+fun checkUtf16to8Replacing(string: String, expected: IntArray) = checkUtf16to8(string, expected) { toUtf8() }
+fun checkUtf16to8Throwing(string: String, expected: IntArray) = checkUtf16to8(string, expected) { toUtf8OrThrow() }
+
+fun checkUtf16to8Replacing(string: String, expected: IntArray, start: Int, size: Int) =
+        checkUtf16to8(string, expected) { toUtf8(start, size) }
+
+fun checkUtf16to8Throwing(string: String, expected: IntArray, start: Int, size: Int) =
+        checkUtf16to8(string, expected) { toUtf8OrThrow(start, size) }
+
+fun checkUtf8to16(expected: String, array: IntArray, conversion: ByteArray.() -> String) {
+    array.forEach {
+        assertTrue(it in Byte.MIN_VALUE..Byte.MAX_VALUE, "Expected array contains illegal values")
+    }
+    val arrayBytes = ByteArray(array.size) { i -> array[i].toByte() }
+    val actual = arrayBytes.conversion()
+    assertEquals(expected, actual, """
+        Assert failed for string: $expected
+        Expected: $expected
+        Actual: $actual
+    """.trimIndent())
+}
+
+fun checkUtf8to16Replacing(expected: String, array: IntArray) = checkUtf8to16(expected, array) { stringFromUtf8() }
+fun checkUtf8to16Throwing(expected: String, array: IntArray) = checkUtf8to16(expected, array) { stringFromUtf8OrThrow() }
+
+fun checkUtf8to16Replacing(expected: String, array: IntArray, start: Int, size: Int) =
+        checkUtf8to16(expected, array) { stringFromUtf8(start, size) }
+fun checkUtf8to16Throwing(expected: String, array: IntArray, start: Int, size: Int) =
+        checkUtf8to16(expected, array) { stringFromUtf8OrThrow(start, size) }
+
+fun <T: Any> checkThrows(e: KClass<T>, string: String, action: () -> Unit) {
+    var exception: Throwable? = null
+    try {
+        action()
+    } catch (e: Throwable) {
+        exception = e
+    }
+    assertNotNull(exception, "No excpetion was thrown for string: $string")
+    assertTrue(e.isInstance(exception),"""
+                Wrong exception was thrown for string: $string
+                Expected: ${e.qualifiedName}
+                Actual: ${exception!!::class.qualifiedName}
+    """.trimIndent())
+}
+
+fun checkUtf16to8Throws(string: String) = checkThrows(IllegalCharacterConversionException::class, string) {
+    string.toUtf8OrThrow()
+}
+
+fun checkUtf16to8Throws(string: String, start: Int, size: Int) =
+        checkThrows(IllegalCharacterConversionException::class, string) {
+            string.toUtf8OrThrow(start, size)
+        }
+
+fun checkUtf8to16Throws(string: String, array: IntArray)
+        = checkThrows(IllegalCharacterConversionException::class, string) {
+    array.forEach {
+        assertTrue(it in Byte.MIN_VALUE..Byte.MAX_VALUE, "Expected array contains illegal values")
+    }
+    val arrayBytes = ByteArray(array.size) { i -> array[i].toByte() }
+    arrayBytes.stringFromUtf8OrThrow()
+}
+
+fun checkUtf8to16Throws(string: String, array: IntArray, start: Int, size: Int)
+        = checkThrows(IllegalCharacterConversionException::class, string) {
+    array.forEach {
+        assertTrue(it in Byte.MIN_VALUE..Byte.MAX_VALUE, "Expected array contains illegal values")
+    }
+    val arrayBytes = ByteArray(array.size) { i -> array[i].toByte() }
+    arrayBytes.stringFromUtf8OrThrow(start, size)
+}
+
+
+fun checkDobuleConversionThrows(string: String) = checkThrows(NumberFormatException::class, string) {
+    string.toDouble()
+}
+
+fun checkFloatConversionThrows(string: String) = checkThrows(NumberFormatException::class, string) {
+    string.toFloat()
+}
+// endregion
+
+// region UTF-16 -> UTF-8 conversion
+fun test16to8() {
+    // Test manual conversion with replacement.
+    // Valid strings.
+    checkUtf16to8Replacing("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()))
+    checkUtf16to8Replacing("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126))
+    checkUtf16to8Replacing("\uD800\uDC00", intArrayOf(-16, -112, -128, -128))
+    checkUtf16to8Replacing("", intArrayOf())
+    // Illegal surrogate pair -> replace with default
+    checkUtf16to8Replacing("\uDC00\uD800", intArrayOf(-17, -65, -67, -17, -65, -67))
+    // Different kinds of input
+    checkUtf16to8Replacing("\uD800\uDC001\uDC00\uD800",
+            intArrayOf(-16, -112, -128, -128, '1'.toInt(), -17, -65, -67, -17, -65, -67))
+    // Lone surrogate - replace with default
+    checkUtf16to8Replacing("\uD80012", intArrayOf(-17, -65, -67, '1'.toInt(), '2'.toInt()))
+    checkUtf16to8Replacing("\uDC0012", intArrayOf(-17, -65, -67, '1'.toInt(), '2'.toInt()))
+    checkUtf16to8Replacing("12\uD800", intArrayOf('1'.toInt(), '2'.toInt(), -17, -65, -67))
+
+
+    // Test manual conversion with an exception if an input is invalid.
+    // Valid strings.
+    checkUtf16to8Throwing("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()))
+    checkUtf16to8Throwing("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126))
+    checkUtf16to8Throwing("\uD800\uDC00", intArrayOf(-16, -112, -128, -128))
+    checkUtf16to8Replacing("", intArrayOf())
+
+    // Illegal surrogate pair -> throw
+    checkUtf16to8Throws("\uDC00\uD800")
+    // Different kinds of input (including illegal one) -> throw
+    checkUtf16to8Throws("\uD800\uDC001\uDC00\uD800")
+    // Lone surrogate - throw
+    checkUtf16to8Throws("\uD80012")
+    checkUtf16to8Throws("\uDC0012")
+    checkUtf16to8Throws("12\uD800")
+
+    // Test double parsing.
+    assertEquals(4.2, "4.2".toDouble())
+    // Illegal surrogate pair -> throw
+    checkDobuleConversionThrows("\uDC00\uD800")
+    // Different kinds of input (including illegal one) -> throw
+    checkDobuleConversionThrows("\uD800\uDC001\uDC00\uD800")
+    // Lone surrogate - throw
+    checkDobuleConversionThrows("\uD80012")
+    checkDobuleConversionThrows("\uDC0012")
+    checkDobuleConversionThrows("12\uD800")
+
+    // Test float parsing.
+    assertEquals(4.2F,  "4.2".toFloat())
+    // Illegal surrogate pair -> throw
+    checkFloatConversionThrows("\uDC00\uD800")
+    // Different kinds of input (including illegal one) -> throw
+    checkFloatConversionThrows("\uD800\uDC001\uDC00\uD800")
+    // Lone surrogate - throw
+    checkFloatConversionThrows("\uD80012")
+    checkFloatConversionThrows("\uDC0012")
+    checkFloatConversionThrows("12\uD800")
+}
+
+fun test16to8CustomBorders() {
+    // Test manual conversion with replacement and custom borders.
+    // Valid strings.
+    checkUtf16to8Replacing("Hello!", intArrayOf('H'.toInt(), 'e'.toInt()), 0, 2)
+    checkUtf16to8Replacing("Hello!", intArrayOf('e'.toInt(), 'l'.toInt()), 1, 2)
+    checkUtf16to8Replacing("Hello!", intArrayOf('o'.toInt(), '!'.toInt()), 4, 2)
+    checkUtf16to8Replacing("Hello!", intArrayOf(), 0, 0)
+    checkUtf16to8Replacing("Hello!", intArrayOf(), 10, 0)
+
+    checkUtf16to8Replacing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00",
+            intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 0, 4)
+    checkUtf16to8Replacing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00",
+            intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 2, 4)
+    checkUtf16to8Replacing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00",
+            intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 4, 4)
+
+    // Illegal surrogate pair -> replace with default
+    checkUtf16to8Replacing("\uDC00\uD80012",
+            intArrayOf(-17, -65, -67, -17, -65, -67, '1'.toInt()), 0, 3)
+    checkUtf16to8Replacing("1\uDC00\uD8002",
+            intArrayOf(-17, -65, -67, -17, -65, -67, '2'.toInt()), 1, 3)
+    checkUtf16to8Replacing("12\uDC00\uD800",
+            intArrayOf('2'.toInt(), -17, -65, -67, -17, -65, -67), 1, 3)
+
+    // Lone surrogate - replace with default
+    checkUtf16to8Replacing("1\uD800\uDC002", intArrayOf('1'.toInt(), -17, -65, -67), 0, 2)
+    checkUtf16to8Replacing("1\uD800\uDC002", intArrayOf(-17, -65, -67, '2'.toInt()), 2, 2)
+
+    // Index out of bound
+    checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8(-1, 4)}
+    checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8(5, 10)}
+    checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8(2, 10)}
+    checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8(3, -2)}
+
+
+    // Test manual conversion with an exception if an input is invalid and custom borders.
+    // Valid strings.
+    checkUtf16to8Throwing("Hello!", intArrayOf('H'.toInt(), 'e'.toInt()), 0, 2)
+    checkUtf16to8Throwing("Hello!", intArrayOf('e'.toInt(), 'l'.toInt()), 1, 2)
+    checkUtf16to8Throwing("Hello!", intArrayOf('o'.toInt(), '!'.toInt()), 4, 2)
+    checkUtf16to8Throwing("Hello!", intArrayOf(), 0, 0)
+    checkUtf16to8Throwing("Hello!", intArrayOf(), 10, 0)
+
+    checkUtf16to8Throwing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00",
+            intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 0, 4)
+    checkUtf16to8Throwing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00",
+            intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 2, 4)
+    checkUtf16to8Throwing("\uD800\uDC00\uD800\uDC00\uD800\uDC00\uD800\uDC00",
+            intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128), 4, 4)
+
+    // Illegal surrogate pair -> throw
+    checkUtf16to8Throws("\uDC00\uD80012", 0, 3)
+    checkUtf16to8Throws("1\uDC00\uD8002", 1, 3)
+    checkUtf16to8Throws("12\uDC00\uD800", 1, 3)
+
+    // Lone surrogate -> throw
+    checkUtf16to8Throws("1\uD800\uDC002", 0, 2)
+    checkUtf16to8Throws("1\uD800\uDC002",  2, 2)
+
+    // Index out of bound
+    checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8OrThrow(-1, 4)}
+    checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8OrThrow(5, 10)}
+    checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8OrThrow(2, 10)}
+    checkThrows(IndexOutOfBoundsException::class, "Hello") { "Hello".toUtf8OrThrow(3, -2)}
+}
+
+fun testPrint() {
+    // Valid strings.
+    println("Hello")
+    println("Привет")
+    println("\uD800\uDC00")
+    println("")
+
+    // Illegal surrogate pair -> default output
+    println("\uDC00\uD800")
+    // Lone surrogate -> default output
+    println("\uD80012")
+    println("\uDC0012")
+    println("12\uD800")
+
+    // https://github.com/JetBrains/kotlin-native/issues/1091
+    val array = byteArrayOf(0xF0.toByte(), 0x9F.toByte(), 0x98.toByte(), 0xA5.toByte())
+    val badStr = array.stringFromUtf8()
+    assertEquals(2, badStr.length)
+    println(badStr)
+}
+// endregion
+
+// region UTF-8 -> UTF-16 conversion
+fun test8to16() {
+    // Test manual conversion with replacement.
+    // Valid strings.
+    checkUtf8to16Replacing("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()))
+    checkUtf8to16Replacing("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126))
+    checkUtf8to16Replacing("\uD800\uDC00", intArrayOf(-16, -112, -128, -128))
+    checkUtf8to16Replacing("", intArrayOf())
+
+    // Incorrect UTF-8 lead character.
+    checkUtf8to16Replacing("\uFFFD1", intArrayOf(-1, '1'.toInt()))
+
+    // Incomplete codepoint.
+    checkUtf8to16Replacing("\uFFFD1", intArrayOf(-16, -97, -104, '1'.toInt()))
+    checkUtf8to16Replacing("\uFFFD1\uFFFD", intArrayOf(-16, -97, -104, '1'.toInt(), -16, -97, -104))
+
+    // Test manual conversion with exception throwing
+    // Valid strings.
+    checkUtf8to16Throwing("Hello", intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()))
+    checkUtf8to16Throwing("Привет", intArrayOf(-48, -97, -47, -128, -48, -72, -48, -78, -48, -75, -47, -126))
+    checkUtf8to16Throwing("\uD800\uDC00", intArrayOf(-16, -112, -128, -128))
+    checkUtf8to16Throwing("", intArrayOf())
+
+    // Incorrect UTF-8 lead character -> throw.
+    checkUtf8to16Throws("\uFFFD1", intArrayOf(-1, '1'.toInt()))
+
+    // Incomplete codepoint -> throw.
+    checkUtf8to16Throws("\uFFFD1", intArrayOf(-16, -97, -104, '1'.toInt()))
+    checkUtf8to16Throws("\uFFFD1\uFFFD", intArrayOf(-16, -97, -104, '1'.toInt(), -16, -97, -104))
+}
+
+fun test8to16CustomBorders() {
+    // Conversion with replacement.
+    checkUtf8to16Replacing("He",
+            intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()),0, 2)
+    checkUtf8to16Replacing("ll",
+            intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 2, 2)
+    checkUtf8to16Replacing("lo",
+            intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 3, 2)
+    checkUtf8to16Replacing("",
+            intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 0, 0)
+    checkUtf8to16Replacing("",
+            intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 10, 0)
+
+    checkUtf8to16Replacing("\uD800\uDC00\uD800\uDC00",
+            intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128),
+            0, 8)
+    checkUtf8to16Replacing("\uD800\uDC00\uD800\uDC00",
+            intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128),
+            4, 8)
+    checkUtf8to16Replacing("\uD800\uDC00\uD800\uDC00",
+            intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128),
+            8, 8)
+
+    // Incorrect UTF-8 lead character.
+    checkUtf8to16Replacing("\uFFFD1", intArrayOf(-1, '1'.toInt(), '2'.toInt()), 0, 2)
+    checkUtf8to16Replacing("\uFFFD2", intArrayOf('1'.toInt(), -1, '2'.toInt(), '3'.toInt()), 1, 2)
+    checkUtf8to16Replacing("2\uFFFD", intArrayOf('1'.toInt(), '2'.toInt(), -1), 1, 2)
+
+    // Incomplete codepoint.
+    checkUtf8to16Replacing("\uFFFD1", intArrayOf(-16, -97, -104, '1'.toInt(), '2'.toInt()), 0, 4)
+    checkUtf8to16Replacing("\uFFFD2", intArrayOf('1'.toInt(), -16, -97, -104, '2'.toInt(), '3'.toInt()), 1, 4)
+    checkUtf8to16Replacing("2\uFFFD", intArrayOf('1'.toInt(), '2'.toInt(), -16, -97, -104),  1, 4)
+
+
+    // Index out of bound
+    val helloArray = byteArrayOf('H'.toByte(), 'e'.toByte(), 'l'.toByte(), 'l'.toByte(), 'o'.toByte())
+    checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8(-1, 4)}
+    checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8(5, 10)}
+    checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8(2, 10)}
+    checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8(3, -2)}
+
+    // Conversion with throwing
+    checkUtf8to16Throwing("He",
+            intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()),0, 2)
+    checkUtf8to16Throwing("ll",
+            intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 2, 2)
+    checkUtf8to16Throwing("lo",
+            intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 3, 2)
+    checkUtf8to16Throwing("",
+            intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 0, 0)
+    checkUtf8to16Throwing("",
+            intArrayOf('H'.toInt(), 'e'.toInt(), 'l'.toInt(), 'l'.toInt(), 'o'.toInt()), 10, 0)
+
+    checkUtf8to16Throwing("\uD800\uDC00\uD800\uDC00",
+            intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128),
+            0, 8)
+    checkUtf8to16Throwing("\uD800\uDC00\uD800\uDC00",
+            intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128),
+            4, 8)
+    checkUtf8to16Throwing("\uD800\uDC00\uD800\uDC00",
+            intArrayOf(-16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128, -16, -112, -128, -128),
+            8, 8)
+
+    // Incorrect UTF-8 lead character.
+    checkUtf8to16Throws("\uFFFD1", intArrayOf(-1, '1'.toInt(), '2'.toInt()), 0, 2)
+    checkUtf8to16Throws("\uFFFD2", intArrayOf('1'.toInt(), -1, '2'.toInt(), '3'.toInt()), 1, 2)
+    checkUtf8to16Throws("2\uFFFD", intArrayOf('1'.toInt(), '2'.toInt(), -1), 1, 2)
+
+    // Incomplete codepoint.
+    checkUtf8to16Throws("\uFFFD1", intArrayOf(-16, -97, -104, '1'.toInt(), '2'.toInt()), 0, 4)
+    checkUtf8to16Throws("\uFFFD2", intArrayOf('1'.toInt(), -16, -97, -104, '2'.toInt(), '3'.toInt()), 1, 4)
+    checkUtf8to16Throws("2\uFFFD", intArrayOf('1'.toInt(), '2'.toInt(), -16, -97, -104),  1, 4)
+
+    // Index out of bound
+    checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8OrThrow(-1, 4)}
+    checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8OrThrow(5, 10)}
+    checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8OrThrow(2, 10)}
+    checkThrows(IndexOutOfBoundsException::class, "Hello") { helloArray.stringFromUtf8OrThrow(3, -2)}
+}
+// endregion
+
+@Test fun runTest() {
+    test16to8()
+    test16to8CustomBorders()
+    test8to16()
+    test8to16CustomBorders()
+    testPrint()
+}

runtime/src/main/cpp/Console.cpp

@@ -30,7 +30,8 @@ void Kotlin_io_Console_print(KString message) {
   // TODO: system stdout must be aware about UTF-8.
   const KChar* utf16 = CharArrayAddressOfElementAt(message, 0);
   KStdString utf8;
-  utf8::unchecked::utf16to8(utf16, utf16 + message->count_, back_inserter(utf8));
+  // Replace incorrect sequences with a default codepoint (see utf8::with_replacement::default_replacement)
+  utf8::with_replacement::utf16to8(utf16, utf16 + message->count_, back_inserter(utf8));
   konan::consoleWriteUtf8(utf8.c_str(), utf8.size());
 }
 

runtime/src/main/cpp/Exceptions.h

@@ -48,6 +48,8 @@ void ThrowNumberFormatException();
 void ThrowOutOfMemoryError();
 // Throws not implemented error.
 void ThrowNotImplementedError();
+// Throws illegal character conversion exception (used in UTF8/UTF16 conversions).
+void ThrowIllegalCharacterConversionException();
 // Prints out mesage of Throwable.
 void PrintThrowable(KRef);
 

runtime/src/main/cpp/KString.cpp

@@ -37,16 +37,55 @@
 
 namespace {
 
-OBJ_GETTER(utf8ToUtf16, const char* rawString, size_t rawStringLength) {
-  uint32_t charCount = utf8::unchecked::distance(rawString, rawString + rawStringLength);
-  ArrayHeader* result = AllocArrayInstance(
-    theStringTypeInfo, charCount, OBJ_RESULT)->array();
+typedef std::back_insert_iterator<KStdString> KStdStringInserter;
+typedef KChar* utf8to16(const char*, const char*, KChar*);
+typedef KStdStringInserter utf16to8(const KChar*,const KChar*, KStdStringInserter);
+
+KStdStringInserter utf16toUtf8OrThrow(const KChar* start, const KChar* end, KStdStringInserter result) {
+  TRY_CATCH(result = utf8::utf16to8(start, end, result),
+            result = utf8::unchecked::utf16to8(start, end, result),
+            ThrowIllegalCharacterConversionException());
+  return result;
+}
+
+template<utf8to16 conversion>
+OBJ_GETTER(utf8ToUtf16Impl, const char* rawString, const char* end, uint32_t charCount) {
+  ArrayHeader* result = AllocArrayInstance(theStringTypeInfo, charCount, OBJ_RESULT)->array();
   KChar* rawResult = CharArrayAddressOfElementAt(result, 0);
-  auto convertResult =
-      utf8::unchecked::utf8to16(rawString, rawString + rawStringLength, rawResult);
+  auto convertResult = conversion(rawString, end, rawResult);
   RETURN_OBJ(result->obj());
 }
 
+template<utf16to8 conversion>
+OBJ_GETTER(utf16ToUtf8Impl, KString thiz, KInt start, KInt size) {
+  RuntimeAssert(thiz->type_info() == theStringTypeInfo, "Must use String");
+  if (start < 0 || size < 0 || size > thiz->count_ - start) {
+    ThrowArrayIndexOutOfBoundsException();
+  }
+  const KChar* utf16 = CharArrayAddressOfElementAt(thiz, start);
+  KStdString utf8;
+  conversion(utf16, utf16 + size, back_inserter(utf8));
+  ArrayHeader* result = AllocArrayInstance(theByteArrayTypeInfo, utf8.size(), OBJ_RESULT)->array();
+  ::memcpy(ByteArrayAddressOfElementAt(result, 0), utf8.c_str(), utf8.size());
+  RETURN_OBJ(result->obj());
+}
+
+OBJ_GETTER(utf8ToUtf16OrThrow, const char* rawString, size_t rawStringLength) {
+  const char* end = rawString + rawStringLength;
+  uint32_t charCount;
+  TRY_CATCH(charCount = utf8::utf16_length(rawString, end),
+            charCount = utf8::unchecked::utf16_length(rawString, end),
+            ThrowIllegalCharacterConversionException());
+  RETURN_RESULT_OF(utf8ToUtf16Impl<utf8::unchecked::utf8to16>, rawString, end, charCount);
+}
+
+OBJ_GETTER(utf8ToUtf16, const char* rawString, size_t rawStringLength) {
+  const char* end = rawString + rawStringLength;
+  uint32_t charCount = utf8::with_replacement::utf16_length(rawString, end);
+  RETURN_RESULT_OF(utf8ToUtf16Impl<utf8::with_replacement::utf8to16>, rawString, end, charCount);
+}
+
+
 // Case conversion is derived work from Apache Harmony.
 // Unicode 3.0.1 (same as Unicode 3.0.0)
 enum CharacterClass {
@@ -731,32 +770,37 @@ KInt Kotlin_String_getStringLength(KString thiz) {
   return thiz->count_;
 }
 
-OBJ_GETTER(Kotlin_String_fromUtf8Array, KConstRef thiz, KInt start, KInt size) {
+const char* byteArrayAsCString(KConstRef thiz, KInt start, KInt size) {
   const ArrayHeader* array = thiz->array();
   RuntimeAssert(array->type_info() == theByteArrayTypeInfo, "Must use a byte array");
   if (start < 0 || size < 0 || size > array->count_ - start) {
     ThrowArrayIndexOutOfBoundsException();
   }
+  return reinterpret_cast<const char*>(ByteArrayAddressOfElementAt(array, start));
+}
+
+OBJ_GETTER(Kotlin_ByteArray_stringFromUtf8OrThrow, KConstRef thiz, KInt start, KInt size) {
+  const char* rawString = byteArrayAsCString(thiz, start, size);
+  if (size == 0) {
+    RETURN_RESULT_OF0(TheEmptyString);
+  }
+  RETURN_RESULT_OF(utf8ToUtf16OrThrow, rawString, size);
+}
+
+OBJ_GETTER(Kotlin_ByteArray_stringFromUtf8, KConstRef thiz, KInt start, KInt size) {
+  const char* rawString = byteArrayAsCString(thiz, start, size);
   if (size == 0) {
     RETURN_RESULT_OF0(TheEmptyString);
   }
-  const char* rawString =
-    reinterpret_cast<const char*>(ByteArrayAddressOfElementAt(array, start));
   RETURN_RESULT_OF(utf8ToUtf16, rawString, size);
 }
 
-OBJ_GETTER(Kotlin_String_toUtf8Array, KString thiz, KInt start, KInt size) {
-  RuntimeAssert(thiz->type_info() == theStringTypeInfo, "Must use String");
-  if (start < 0 || size < 0 || size > thiz->count_ - start) {
-    ThrowArrayIndexOutOfBoundsException();
-  }
-  const KChar* utf16 = CharArrayAddressOfElementAt(thiz, start);
-  KStdString utf8;
-  utf8::unchecked::utf16to8(utf16, utf16 + size, back_inserter(utf8));
-  ArrayHeader* result = AllocArrayInstance(
-      theByteArrayTypeInfo, utf8.size(), OBJ_RESULT)->array();
-  ::memcpy(ByteArrayAddressOfElementAt(result, 0), utf8.c_str(), utf8.size());
-  RETURN_OBJ(result->obj());
+OBJ_GETTER(Kotlin_String_toUtf8, KString thiz, KInt start, KInt size) {
+  RETURN_RESULT_OF(utf16ToUtf8Impl<utf8::with_replacement::utf16to8>, thiz, start, size);
+}
+
+OBJ_GETTER(Kotlin_String_toUtf8OrThrow, KString thiz, KInt start, KInt size) {
+  RETURN_RESULT_OF(utf16ToUtf8Impl<utf16toUtf8OrThrow>, thiz, start, size);
 }
 
 OBJ_GETTER(Kotlin_String_fromCharArray, KConstRef thiz, KInt start, KInt size) {

runtime/src/main/cpp/KString.h

@@ -27,7 +27,7 @@ extern "C" {
 #endif
 
 OBJ_GETTER(CreateStringFromCString, const char* cstring);
-OBJ_GETTER(CreateStringFromUtf8, const char* utf8, uint32_t size);
+OBJ_GETTER(CreateStringFromUtf8, const char* utf8, uint32_t lengthBytes);
 char* CreateCStringFromString(KConstRef kstring);
 void DisposeCString(char* cstring);
 

runtime/src/main/cpp/Porting.h

@@ -52,6 +52,16 @@ uint64_t getTimeMillis();
 uint64_t getTimeMicros();
 uint64_t getTimeNanos();
 
+#if KONAN_NO_EXCEPTIONS
+#define TRY_CATCH(tryAction, actionWithoutExceptions, catchAction) actionWithoutExceptions;
+#else
+#define TRY_CATCH(tryAction, actionWithoutExceptions, catchAction) \
+do {                          \
+  try { tryAction; }          \
+  catch(...) { catchAction; } \
+} while(0)
+#endif
+
 }  // namespace konan
 
 #endif  // RUNTIME_PORTING_H

runtime/src/main/cpp/dtoa/dblparse.cpp

@@ -642,7 +642,9 @@ KDouble Konan_FloatingPointParser_parseDoubleImpl (KString s, KInt e)
 {
   const KChar* utf16 = CharArrayAddressOfElementAt(s, 0);
   KStdString utf8;
-  utf8::unchecked::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8));
+  TRY_CATCH(utf8::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8)),
+            utf8::unchecked::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8)),
+            /* Illegal UTF-16 string. */ ThrowNumberFormatException());
   const char *str = utf8.c_str();
   auto dbl = createDouble (str, e);
 

runtime/src/main/cpp/dtoa/fltparse.cpp

@@ -542,7 +542,9 @@ Konan_FloatingPointParser_parseFloatImpl(KString s, KInt e)
 {
   const KChar* utf16 = CharArrayAddressOfElementAt(s, 0);
   KStdString utf8;
-  utf8::unchecked::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8));
+  TRY_CATCH(utf8::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8)),
+            utf8::unchecked::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8)),
+            /* Illegal UTF-16 string. */ ThrowNumberFormatException());
   const char *str = utf8.c_str();
   auto flt = createFloat(str, e);
 

runtime/src/main/cpp/utf8.h

@@ -29,5 +29,10 @@ DEALINGS IN THE SOFTWARE.
 #define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
 
 #include "utf8/unchecked.h"
+#include "utf8/with_replacement.h"
+
+#if !KONAN_NO_EXCEPTIONS
+#include "utf8/checked.h"
+#endif
 
 #endif // header guard

runtime/src/main/cpp/utf8/checked.h

@@ -193,6 +193,20 @@ namespace utf8
             utf8::next(it, end);
     }
 
+    /**
+     * Calculates a count of characters needed to represent the string from first to last in UTF-16
+     * taking into account surrogate symbols. Throws an exception if the input is invalid.
+     */
+    template<typename octet_iterator>
+    uint32_t utf16_length(octet_iterator first, octet_iterator last) {
+      uint32_t dist = 0;
+      while(first < last) {
+        uint32_t cp = utf8::next(first, last);
+        dist += (cp > 0xffff) ? 2 : 1;
+      }
+      return dist;
+    }
+
     template <typename octet_iterator>
     typename std::iterator_traits<octet_iterator>::difference_type
     distance (octet_iterator first, octet_iterator last)

runtime/src/main/cpp/utf8/core.h

@@ -150,7 +150,7 @@ namespace internal
 
     /// get_sequence_x functions decode utf-8 sequences of the length x
     template <typename octet_iterator>
-    utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point)
+    utf_error get_sequence_1(octet_iterator& it, const octet_iterator end, uint32_t& code_point)
     {
         if (it == end)
             return NOT_ENOUGH_ROOM;
@@ -161,7 +161,7 @@ namespace internal
     }
 
     template <typename octet_iterator>
-    utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point)
+    utf_error get_sequence_2(octet_iterator& it, const octet_iterator end, uint32_t& code_point)
     {
         if (it == end) 
             return NOT_ENOUGH_ROOM;
@@ -176,7 +176,7 @@ namespace internal
     }
 
     template <typename octet_iterator>
-    utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point)
+    utf_error get_sequence_3(octet_iterator& it, const octet_iterator end, uint32_t& code_point)
     {
         if (it == end)
             return NOT_ENOUGH_ROOM;
@@ -195,7 +195,7 @@ namespace internal
     }
 
     template <typename octet_iterator>
-    utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point)
+    utf_error get_sequence_4(octet_iterator& it, const octet_iterator end, uint32_t& code_point)
     {
         if (it == end)
            return NOT_ENOUGH_ROOM;
@@ -220,7 +220,7 @@ namespace internal
     #undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR
 
     template <typename octet_iterator>
-    utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point)
+    utf_error validate_next(octet_iterator& it, const octet_iterator end, uint32_t& code_point)
     {
         // Save the original value of it so we can go back in case of failure
         // Of course, it does not make much sense with i.e. stream iterators

runtime/src/main/cpp/utf8/unchecked.h

@@ -116,6 +116,20 @@ namespace utf8
                 utf8::unchecked::next(it);
         }
 
+        /**
+         * Calculates a count of characters needed to represent the string from first to last in UTF-16
+         * taking into account surrogate symbols. Doesn't validate the input.
+         */
+        template<typename octet_iterator>
+        uint32_t utf16_length(octet_iterator first, const octet_iterator last) {
+            uint32_t dist = 0;
+            while (first < last) {
+                uint32_t cp = utf8::unchecked::next(first);
+                dist += (cp > 0xffff) ? 2 : 1;
+            }
+            return dist;
+        }
+
         template <typename octet_iterator>
         typename std::iterator_traits<octet_iterator>::difference_type
         distance (octet_iterator first, octet_iterator last)
@@ -127,7 +141,7 @@ namespace utf8
         }
 
         template <typename u16bit_iterator, typename octet_iterator>
-        octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
+        octet_iterator utf16to8 (u16bit_iterator start, const u16bit_iterator end, octet_iterator result)
         {       
             while (start != end) {
                 uint32_t cp = utf8::internal::mask16(*start++);
@@ -142,7 +156,7 @@ namespace utf8
         }
 
         template <typename u16bit_iterator, typename octet_iterator>
-        u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
+        u16bit_iterator utf8to16 (octet_iterator start, const octet_iterator end, u16bit_iterator result)
         {
             while (start < end) {
                 uint32_t cp = utf8::unchecked::next(start);

runtime/src/main/cpp/utf8/with_replacement.h

@@ -0,0 +1,148 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef UTF_FOR_CPP_WITH_REPLACEMENT_H
+#define UTF_FOR_CPP_WITH_REPLACEMENT_H
+
+#include "core.h"
+#include "unchecked.h"
+
+namespace utf8 {
+
+namespace with_replacement {
+
+constexpr uint32_t default_replacement = 0xfffd;
+
+/*
+ * Returns the next codepoint replacing any invalid sequence with the replacement codepoint.
+ */
+template<typename octet_iterator>
+uint32_t next(octet_iterator &it, const octet_iterator end, uint32_t replacement) {
+    uint32_t cp = 0;
+    internal::utf_error err_code = utf8::internal::validate_next(it, end, cp);
+    switch (err_code) {
+        case internal::UTF8_OK :
+            return cp;
+        case internal::INVALID_LEAD :
+            it++;
+            return replacement;
+        case internal::NOT_ENOUGH_ROOM :
+        case internal::INCOMPLETE_SEQUENCE :
+        case internal::OVERLONG_SEQUENCE :
+        case internal::INVALID_CODE_POINT :
+            // The whole invalid sequence is replaced with one replacement codepoint.
+            for (it++; it < end && utf8::internal::is_trail(*it); it++);
+            return replacement;
+    }
+}
+
+// Library API
+
+/**
+ * Calculates a count of characters needed to represent the string from first to last in UTF-16
+ * taking into account surrogate symbols and invalid sequences.
+ * Assumes that all invalid sequences in the input will be replaced with `replacement`
+ * so each invalid sequence increases the result by 1 or 2 depending on `replacement`.
+ */
+template<typename octet_iterator>
+uint32_t utf16_length(octet_iterator first,
+                      const octet_iterator last,
+                      const uint32_t replacement = default_replacement) {
+    uint32_t dist = 0;
+    while (first < last) {
+        uint32_t cp = next(first, last, replacement);
+        dist += (cp > 0xffff) ? 2 : 1;
+    }
+    return dist;
+}
+
+
+template<typename octet_iterator>
+typename std::iterator_traits<octet_iterator>::difference_type
+distance(octet_iterator first, const octet_iterator last) {
+    typename std::iterator_traits<octet_iterator>::difference_type dist;
+    uint32_t unused = 0;
+    for (dist = 0; first < last; dist++) {
+        next(first, last, unused);
+    }
+    return dist;
+}
+
+template<typename u16bit_iterator, typename octet_iterator>
+octet_iterator utf16to8(u16bit_iterator start,
+                        const u16bit_iterator end,
+                        octet_iterator result,
+                        const uint32_t replacement) {
+    while (start != end) {
+        uint32_t cp = utf8::internal::mask16(*start++);
+        // Process surrogates.
+        if (utf8::internal::is_lead_surrogate(cp)) {
+            if (start != end) {
+                uint32_t trail_surrogate = utf8::internal::mask16(*start);
+                if (utf8::internal::is_trail_surrogate(trail_surrogate)) {
+                    // Valid surrogate pair.
+                    cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
+                    start++;
+                } else {
+                    cp = replacement; // Invalid input: lone lead surrogate.
+                }
+            } else {
+                cp = replacement; // Invalid input: lone lead surrogate at the end of input.
+            }
+        } else if (utf8::internal::is_trail_surrogate(cp)) {
+            cp = replacement; // Invalid input: lone trail surrogate
+        }
+        result = utf8::unchecked::append(cp, result);
+    }
+    return result;
+}
+
+template<typename u16bit_iterator, typename octet_iterator>
+inline octet_iterator utf16to8(u16bit_iterator start,
+                               const u16bit_iterator end,
+                               octet_iterator result) {
+  return utf16to8(start, end, result, default_replacement);
+}
+
+template<typename u16bit_iterator, typename octet_iterator>
+u16bit_iterator utf8to16(octet_iterator start,
+                         const octet_iterator end,
+                         u16bit_iterator result,
+                         const uint32_t replacement) {
+    while (start != end) {
+        // The `next` method takes care about replacing invalid sequences.
+        uint32_t cp = next(start, end, replacement);
+        if (cp > 0xffff) { //make a surrogate pair
+            *result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET);
+            *result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
+        } else
+            *result++ = static_cast<uint16_t>(cp);
+    }
+    return result;
+}
+
+template<typename u16bit_iterator, typename octet_iterator>
+inline u16bit_iterator utf8to16(octet_iterator start,
+                                const octet_iterator end,
+                                u16bit_iterator result) {
+  return utf8to16(start, end, result, default_replacement);
+}
+
+} // namespace with_replacement
+
+} // namespace utf8
+
+#endif // UTF_FOR_CPP_WITH_REPLACEMENT_H

runtime/src/main/kotlin/konan/internal/RuntimeUtils.kt

@@ -17,7 +17,6 @@
 package konan.internal
 
 import kotlin.internal.getProgressionLastElement
-import kotlin.text.toUtf8Array
 
 @ExportForCppRuntime
 fun ThrowNullPointerException(): Nothing {
@@ -66,6 +65,11 @@ internal fun ThrowNotImplementedError(): Nothing {
     throw NotImplementedError("An operation is not implemented.")
 }
 
+@ExportForCppRuntime
+internal fun ThrowIllegalCharacterConversionException(): Nothing {
+    throw IllegalCharacterConversionException()
+}
+
 @ExportForCppRuntime
 fun PrintThrowable(throwable: Throwable) {
     println(throwable)
@@ -140,5 +144,5 @@ fun KonanObjectToUtf8Array(value: Any?): ByteArray {
         is DoubleArray -> value.contentToString()
         else -> value.toString()
     }
-    return toUtf8Array(string, 0, string.length)
+    return string.toUtf8()
 }

runtime/src/main/kotlin/kotlin/Exceptions.kt

@@ -216,4 +216,9 @@ public open class NumberFormatException : IllegalArgumentException {
     }
 
     constructor(s: String) : super(s) {}
+}
+
+public open class IllegalCharacterConversionException : IllegalArgumentException {
+    constructor(): super()
+    constructor(s: String) : super(s)
 }

runtime/src/main/kotlin/kotlin/text/StringBuilder.kt

@@ -16,11 +16,50 @@
 
 package kotlin.text
 
-@SymbolName("Kotlin_String_fromUtf8Array")
-external fun fromUtf8Array(array: ByteArray, start: Int, size: Int) : String
+// ByteArray -> String (UTF-8 -> UTF-16)
 
-@SymbolName("Kotlin_String_toUtf8Array")
-external fun toUtf8Array(string: String, start: Int, size: Int) : ByteArray
+@Deprecated("Use ByteArray.stringFromUtf8()", ReplaceWith("array.stringFromUtf8(start, end)"))
+fun fromUtf8Array(array: ByteArray, start: Int, size: Int) = array.stringFromUtf8Impl(start, size)
+
+@Deprecated("Use String.toUtf8()", ReplaceWith("string.toUtf8(start, end)"))
+fun toUtf8Array(string: String, start: Int, size: Int) : ByteArray = string.toUtf8(start, size)
+
+/**
+ * Converts an UTF-8 array into a [String]. Replaces invalid input sequences with a default character.
+ */
+fun ByteArray.stringFromUtf8(start: Int = 0, size: Int = this.size) : String =
+        stringFromUtf8Impl(start, size)
+
+@SymbolName("Kotlin_ByteArray_stringFromUtf8")
+private external fun ByteArray.stringFromUtf8Impl(start: Int, size: Int) : String
+
+/**
+ * Converts an UTF-8 array into a [String]. Throws [IllegalCharacterConversionException] if the input is invalid.
+ */
+fun ByteArray.stringFromUtf8OrThrow(start: Int = 0, size: Int = this.size) : String =
+        stringFromUtf8OrThrowImpl(start, size)
+
+@SymbolName("Kotlin_ByteArray_stringFromUtf8OrThrow")
+private external fun ByteArray.stringFromUtf8OrThrowImpl(start: Int, size: Int) : String
+
+// String -> ByteArray (UTF-16 -> UTF-8)
+/**
+ * Converts a [String] into an UTF-8 array. Replaces invalid input sequences with a default character.
+ */
+fun String.toUtf8(start: Int = 0, size: Int = this.length) : ByteArray =
+        toUtf8Impl(start, size)
+
+@SymbolName("Kotlin_String_toUtf8")
+private external fun String.toUtf8Impl(start: Int, size: Int) : ByteArray
+
+/**
+ * Converts a [String] into an UTF-8 array. Throws [IllegalCharacterConversionException] if the input is invalid.
+ */
+fun String.toUtf8OrThrow(start: Int = 0, size: Int = this.length) : ByteArray =
+        toUtf8OrThrowImpl(start, size)
+
+@SymbolName("Kotlin_String_toUtf8OrThrow")
+private external fun String.toUtf8OrThrowImpl(start: Int, size: Int) : ByteArray
 
 // TODO: make it somewhat private?
 @SymbolName("Kotlin_String_fromCharArray")

samples/libcurl/src/main/kotlin/org/konan/libcurl/CUrl.kt

@@ -33,7 +33,7 @@ class CUrl(val url: String)  {
 
 fun CPointer<ByteVar>.toKString(length: Int): String {
     val bytes = this.readBytes(length)
-    return kotlin.text.fromUtf8Array(bytes, 0, bytes.size)
+    return bytes.stringFromUtf8()
 }
 
 fun header_callback(buffer: CPointer<ByteVar>?, size: size_t, nitems: size_t, userdata: COpaquePointer?): size_t {

samples/socket/src/main/kotlin/EchoServer.kt

@@ -31,7 +31,7 @@ fun main(args: Array<String>) {
     memScoped {
 
         val buffer = ByteArray(1024)
-        val prefixBuffer = kotlin.text.toUtf8Array("echo: ", 0, 6)
+        val prefixBuffer = "echo: ".toUtf8()
         val serverAddr = alloc<sockaddr_in>()
 
         val listenFd = socket(AF_INET, SOCK_STREAM, 0)

samples/videoplayer/src/main/kotlin/DecoderWorker.kt

@@ -43,7 +43,7 @@ private fun Int.checkError() {
     if (this != 0) {
         val buffer = ByteArray(1024)
         av_strerror(this, buffer.refTo(0), buffer.size.signExtend())
-        throw Error("AVError: ${kotlin.text.fromUtf8Array(buffer, 0, buffer.size)}")
+        throw Error("AVError: ${buffer.stringFromUtf8()}")
     }
 }