azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix utf8 conversion (#1121)

Browse Source
This commit is contained in:
ilmat192
2017-12-12 17:30:50 +07:00
committed by Nikolay Igotti
parent 8fa8e2ebc8
commit cba7319e98
21 changed files with 709 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/src/main/cpp/Console.cpp
+2 -1
View File
@@ -30,7 +30,8 @@ void Kotlin_io_Console_print(KString message) {
// TODO: system stdout must be aware about UTF-8.
const KChar* utf16 = CharArrayAddressOfElementAt(message, 0);
KStdString utf8;
utf8::unchecked::utf16to8(utf16, utf16 + message->count_, back_inserter(utf8));
// Replace incorrect sequences with a default codepoint (see utf8::with_replacement::default_replacement)
utf8::with_replacement::utf16to8(utf16, utf16 + message->count_, back_inserter(utf8));
konan::consoleWriteUtf8(utf8.c_str(), utf8.size());
}
runtime/src/main/cpp/Exceptions.h
+2
View File
@@ -48,6 +48,8 @@ void ThrowNumberFormatException();
void ThrowOutOfMemoryError();
// Throws not implemented error.
void ThrowNotImplementedError();
// Throws illegal character conversion exception (used in UTF8/UTF16 conversions).
void ThrowIllegalCharacterConversionException();
// Prints out mesage of Throwable.
void PrintThrowable(KRef);
runtime/src/main/cpp/KString.cpp
+65 -21
View File
@@ -37,16 +37,55 @@
namespace {
OBJ_GETTER(utf8ToUtf16, const char* rawString, size_t rawStringLength) {
uint32_t charCount = utf8::unchecked::distance(rawString, rawString + rawStringLength);
ArrayHeader* result = AllocArrayInstance(
theStringTypeInfo, charCount, OBJ_RESULT)->array();
typedef std::back_insert_iterator<KStdString> KStdStringInserter;
typedef KChar* utf8to16(const char*, const char*, KChar*);
typedef KStdStringInserter utf16to8(const KChar*,const KChar*, KStdStringInserter);
KStdStringInserter utf16toUtf8OrThrow(const KChar* start, const KChar* end, KStdStringInserter result) {
TRY_CATCH(result = utf8::utf16to8(start, end, result),
result = utf8::unchecked::utf16to8(start, end, result),
ThrowIllegalCharacterConversionException());
return result;
}
template<utf8to16 conversion>
OBJ_GETTER(utf8ToUtf16Impl, const char* rawString, const char* end, uint32_t charCount) {
ArrayHeader* result = AllocArrayInstance(theStringTypeInfo, charCount, OBJ_RESULT)->array();
KChar* rawResult = CharArrayAddressOfElementAt(result, 0);
auto convertResult =
utf8::unchecked::utf8to16(rawString, rawString + rawStringLength, rawResult);
auto convertResult = conversion(rawString, end, rawResult);
RETURN_OBJ(result->obj());
}
template<utf16to8 conversion>
OBJ_GETTER(utf16ToUtf8Impl, KString thiz, KInt start, KInt size) {
RuntimeAssert(thiz->type_info() == theStringTypeInfo, "Must use String");
if (start < 0 || size < 0 || size > thiz->count_ - start) {
ThrowArrayIndexOutOfBoundsException();
}
const KChar* utf16 = CharArrayAddressOfElementAt(thiz, start);
KStdString utf8;
conversion(utf16, utf16 + size, back_inserter(utf8));
ArrayHeader* result = AllocArrayInstance(theByteArrayTypeInfo, utf8.size(), OBJ_RESULT)->array();
::memcpy(ByteArrayAddressOfElementAt(result, 0), utf8.c_str(), utf8.size());
RETURN_OBJ(result->obj());
}
OBJ_GETTER(utf8ToUtf16OrThrow, const char* rawString, size_t rawStringLength) {
const char* end = rawString + rawStringLength;
uint32_t charCount;
TRY_CATCH(charCount = utf8::utf16_length(rawString, end),
charCount = utf8::unchecked::utf16_length(rawString, end),
ThrowIllegalCharacterConversionException());
RETURN_RESULT_OF(utf8ToUtf16Impl<utf8::unchecked::utf8to16>, rawString, end, charCount);
}
OBJ_GETTER(utf8ToUtf16, const char* rawString, size_t rawStringLength) {
const char* end = rawString + rawStringLength;
uint32_t charCount = utf8::with_replacement::utf16_length(rawString, end);
RETURN_RESULT_OF(utf8ToUtf16Impl<utf8::with_replacement::utf8to16>, rawString, end, charCount);
}
// Case conversion is derived work from Apache Harmony.
// Unicode 3.0.1 (same as Unicode 3.0.0)
enum CharacterClass {
@@ -731,32 +770,37 @@ KInt Kotlin_String_getStringLength(KString thiz) {
return thiz->count_;
}
OBJ_GETTER(Kotlin_String_fromUtf8Array, KConstRef thiz, KInt start, KInt size) {
const char* byteArrayAsCString(KConstRef thiz, KInt start, KInt size) {
const ArrayHeader* array = thiz->array();
RuntimeAssert(array->type_info() == theByteArrayTypeInfo, "Must use a byte array");
if (start < 0 || size < 0 || size > array->count_ - start) {
ThrowArrayIndexOutOfBoundsException();
}
return reinterpret_cast<const char*>(ByteArrayAddressOfElementAt(array, start));
}
OBJ_GETTER(Kotlin_ByteArray_stringFromUtf8OrThrow, KConstRef thiz, KInt start, KInt size) {
const char* rawString = byteArrayAsCString(thiz, start, size);
if (size == 0) {
RETURN_RESULT_OF0(TheEmptyString);
}
RETURN_RESULT_OF(utf8ToUtf16OrThrow, rawString, size);
}
OBJ_GETTER(Kotlin_ByteArray_stringFromUtf8, KConstRef thiz, KInt start, KInt size) {
const char* rawString = byteArrayAsCString(thiz, start, size);
if (size == 0) {
RETURN_RESULT_OF0(TheEmptyString);
}
const char* rawString =
reinterpret_cast<const char*>(ByteArrayAddressOfElementAt(array, start));
RETURN_RESULT_OF(utf8ToUtf16, rawString, size);
}
OBJ_GETTER(Kotlin_String_toUtf8Array, KString thiz, KInt start, KInt size) {
RuntimeAssert(thiz->type_info() == theStringTypeInfo, "Must use String");
if (start < 0 || size < 0 || size > thiz->count_ - start) {
ThrowArrayIndexOutOfBoundsException();
}
const KChar* utf16 = CharArrayAddressOfElementAt(thiz, start);
KStdString utf8;
utf8::unchecked::utf16to8(utf16, utf16 + size, back_inserter(utf8));
ArrayHeader* result = AllocArrayInstance(
theByteArrayTypeInfo, utf8.size(), OBJ_RESULT)->array();
::memcpy(ByteArrayAddressOfElementAt(result, 0), utf8.c_str(), utf8.size());
RETURN_OBJ(result->obj());
OBJ_GETTER(Kotlin_String_toUtf8, KString thiz, KInt start, KInt size) {
RETURN_RESULT_OF(utf16ToUtf8Impl<utf8::with_replacement::utf16to8>, thiz, start, size);
}
OBJ_GETTER(Kotlin_String_toUtf8OrThrow, KString thiz, KInt start, KInt size) {
RETURN_RESULT_OF(utf16ToUtf8Impl<utf16toUtf8OrThrow>, thiz, start, size);
}
OBJ_GETTER(Kotlin_String_fromCharArray, KConstRef thiz, KInt start, KInt size) {
runtime/src/main/cpp/KString.h
+1 -1
View File
@@ -27,7 +27,7 @@ extern "C" {
#endif
OBJ_GETTER(CreateStringFromCString, const char* cstring);
OBJ_GETTER(CreateStringFromUtf8, const char* utf8, uint32_t size);
OBJ_GETTER(CreateStringFromUtf8, const char* utf8, uint32_t lengthBytes);
char* CreateCStringFromString(KConstRef kstring);
void DisposeCString(char* cstring);
runtime/src/main/cpp/Porting.h
+10
View File
@@ -52,6 +52,16 @@ uint64_t getTimeMillis();
uint64_t getTimeMicros();
uint64_t getTimeNanos();
#if KONAN_NO_EXCEPTIONS
#define TRY_CATCH(tryAction, actionWithoutExceptions, catchAction) actionWithoutExceptions;
#else
#define TRY_CATCH(tryAction, actionWithoutExceptions, catchAction) \
do { \
try { tryAction; } \
catch(...) { catchAction; } \
} while(0)
#endif
} // namespace konan
#endif // RUNTIME_PORTING_H
runtime/src/main/cpp/dtoa/dblparse.cpp
+3 -1
View File
@@ -642,7 +642,9 @@ KDouble Konan_FloatingPointParser_parseDoubleImpl (KString s, KInt e)
{
const KChar* utf16 = CharArrayAddressOfElementAt(s, 0);
KStdString utf8;
utf8::unchecked::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8));
TRY_CATCH(utf8::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8)),
utf8::unchecked::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8)),
/* Illegal UTF-16 string. */ ThrowNumberFormatException());
const char *str = utf8.c_str();
auto dbl = createDouble (str, e);
runtime/src/main/cpp/dtoa/fltparse.cpp
+3 -1
View File
@@ -542,7 +542,9 @@ Konan_FloatingPointParser_parseFloatImpl(KString s, KInt e)
{
const KChar* utf16 = CharArrayAddressOfElementAt(s, 0);
KStdString utf8;
utf8::unchecked::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8));
TRY_CATCH(utf8::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8)),
utf8::unchecked::utf16to8(utf16, utf16 + s->count_, back_inserter(utf8)),
/* Illegal UTF-16 string. */ ThrowNumberFormatException());
const char *str = utf8.c_str();
auto flt = createFloat(str, e);
runtime/src/main/cpp/utf8.h
+5
View File
@@ -29,5 +29,10 @@ DEALINGS IN THE SOFTWARE.
#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
#include "utf8/unchecked.h"
#include "utf8/with_replacement.h"
#if !KONAN_NO_EXCEPTIONS
#include "utf8/checked.h"
#endif
#endif // header guard
runtime/src/main/cpp/utf8/checked.h
+14
View File
@@ -193,6 +193,20 @@ namespace utf8
utf8::next(it, end);
}
/**
* Calculates a count of characters needed to represent the string from first to last in UTF-16
* taking into account surrogate symbols. Throws an exception if the input is invalid.
*/
template<typename octet_iterator>
uint32_t utf16_length(octet_iterator first, octet_iterator last) {
uint32_t dist = 0;
while(first < last) {
uint32_t cp = utf8::next(first, last);
dist += (cp > 0xffff) ? 2 : 1;
}
return dist;
}
template <typename octet_iterator>
typename std::iterator_traits<octet_iterator>::difference_type
distance (octet_iterator first, octet_iterator last)
runtime/src/main/cpp/utf8/core.h
+5 -5
View File
@@ -150,7 +150,7 @@ namespace internal
/// get_sequence_x functions decode utf-8 sequences of the length x
template <typename octet_iterator>
utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point)
utf_error get_sequence_1(octet_iterator& it, const octet_iterator end, uint32_t& code_point)
{
if (it == end)
return NOT_ENOUGH_ROOM;
@@ -161,7 +161,7 @@ namespace internal
}
template <typename octet_iterator>
utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point)
utf_error get_sequence_2(octet_iterator& it, const octet_iterator end, uint32_t& code_point)
{
if (it == end)
return NOT_ENOUGH_ROOM;
@@ -176,7 +176,7 @@ namespace internal
}
template <typename octet_iterator>
utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point)
utf_error get_sequence_3(octet_iterator& it, const octet_iterator end, uint32_t& code_point)
{
if (it == end)
return NOT_ENOUGH_ROOM;
@@ -195,7 +195,7 @@ namespace internal
}
template <typename octet_iterator>
utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point)
utf_error get_sequence_4(octet_iterator& it, const octet_iterator end, uint32_t& code_point)
{
if (it == end)
return NOT_ENOUGH_ROOM;
@@ -220,7 +220,7 @@ namespace internal
#undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR
template <typename octet_iterator>
utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point)
utf_error validate_next(octet_iterator& it, const octet_iterator end, uint32_t& code_point)
{
// Save the original value of it so we can go back in case of failure
// Of course, it does not make much sense with i.e. stream iterators
runtime/src/main/cpp/utf8/unchecked.h
+16 -2
View File
@@ -116,6 +116,20 @@ namespace utf8
utf8::unchecked::next(it);
}
/**
* Calculates a count of characters needed to represent the string from first to last in UTF-16
* taking into account surrogate symbols. Doesn't validate the input.
*/
template<typename octet_iterator>
uint32_t utf16_length(octet_iterator first, const octet_iterator last) {
uint32_t dist = 0;
while (first < last) {
uint32_t cp = utf8::unchecked::next(first);
dist += (cp > 0xffff) ? 2 : 1;
}
return dist;
}
template <typename octet_iterator>
typename std::iterator_traits<octet_iterator>::difference_type
distance (octet_iterator first, octet_iterator last)
@@ -127,7 +141,7 @@ namespace utf8
}
template <typename u16bit_iterator, typename octet_iterator>
octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
octet_iterator utf16to8 (u16bit_iterator start, const u16bit_iterator end, octet_iterator result)
{
while (start != end) {
uint32_t cp = utf8::internal::mask16(*start++);
@@ -142,7 +156,7 @@ namespace utf8
}
template <typename u16bit_iterator, typename octet_iterator>
u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
u16bit_iterator utf8to16 (octet_iterator start, const octet_iterator end, u16bit_iterator result)
{
while (start < end) {
uint32_t cp = utf8::unchecked::next(start);
runtime/src/main/cpp/utf8/with_replacement.h
+148
View File
@@ -0,0 +1,148 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef UTF_FOR_CPP_WITH_REPLACEMENT_H
#define UTF_FOR_CPP_WITH_REPLACEMENT_H
#include "core.h"
#include "unchecked.h"
namespace utf8 {
namespace with_replacement {
constexpr uint32_t default_replacement = 0xfffd;
/*
* Returns the next codepoint replacing any invalid sequence with the replacement codepoint.
*/
template<typename octet_iterator>
uint32_t next(octet_iterator &it, const octet_iterator end, uint32_t replacement) {
uint32_t cp = 0;
internal::utf_error err_code = utf8::internal::validate_next(it, end, cp);
switch (err_code) {
case internal::UTF8_OK :
return cp;
case internal::INVALID_LEAD :
it++;
return replacement;
case internal::NOT_ENOUGH_ROOM :
case internal::INCOMPLETE_SEQUENCE :
case internal::OVERLONG_SEQUENCE :
case internal::INVALID_CODE_POINT :
// The whole invalid sequence is replaced with one replacement codepoint.
for (it++; it < end && utf8::internal::is_trail(*it); it++);
return replacement;
}
}
// Library API
/**
* Calculates a count of characters needed to represent the string from first to last in UTF-16
* taking into account surrogate symbols and invalid sequences.
* Assumes that all invalid sequences in the input will be replaced with `replacement`
* so each invalid sequence increases the result by 1 or 2 depending on `replacement`.
*/
template<typename octet_iterator>
uint32_t utf16_length(octet_iterator first,
const octet_iterator last,
const uint32_t replacement = default_replacement) {
uint32_t dist = 0;
while (first < last) {
uint32_t cp = next(first, last, replacement);
dist += (cp > 0xffff) ? 2 : 1;
}
return dist;
}
template<typename octet_iterator>
typename std::iterator_traits<octet_iterator>::difference_type
distance(octet_iterator first, const octet_iterator last) {
typename std::iterator_traits<octet_iterator>::difference_type dist;
uint32_t unused = 0;
for (dist = 0; first < last; dist++) {
next(first, last, unused);
}
return dist;
}
template<typename u16bit_iterator, typename octet_iterator>
octet_iterator utf16to8(u16bit_iterator start,
const u16bit_iterator end,
octet_iterator result,
const uint32_t replacement) {
while (start != end) {
uint32_t cp = utf8::internal::mask16(*start++);
// Process surrogates.
if (utf8::internal::is_lead_surrogate(cp)) {
if (start != end) {
uint32_t trail_surrogate = utf8::internal::mask16(*start);
if (utf8::internal::is_trail_surrogate(trail_surrogate)) {
// Valid surrogate pair.
cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
start++;
} else {
cp = replacement; // Invalid input: lone lead surrogate.
}
} else {
cp = replacement; // Invalid input: lone lead surrogate at the end of input.
}
} else if (utf8::internal::is_trail_surrogate(cp)) {
cp = replacement; // Invalid input: lone trail surrogate
}
result = utf8::unchecked::append(cp, result);
}
return result;
}
template<typename u16bit_iterator, typename octet_iterator>
inline octet_iterator utf16to8(u16bit_iterator start,
const u16bit_iterator end,
octet_iterator result) {
return utf16to8(start, end, result, default_replacement);
}
template<typename u16bit_iterator, typename octet_iterator>
u16bit_iterator utf8to16(octet_iterator start,
const octet_iterator end,
u16bit_iterator result,
const uint32_t replacement) {
while (start != end) {
// The `next` method takes care about replacing invalid sequences.
uint32_t cp = next(start, end, replacement);
if (cp > 0xffff) { //make a surrogate pair
*result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET);
*result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
} else
*result++ = static_cast<uint16_t>(cp);
}
return result;
}
template<typename u16bit_iterator, typename octet_iterator>
inline u16bit_iterator utf8to16(octet_iterator start,
const octet_iterator end,
u16bit_iterator result) {
return utf8to16(start, end, result, default_replacement);
}
} // namespace with_replacement
} // namespace utf8
#endif // UTF_FOR_CPP_WITH_REPLACEMENT_H
runtime/src/main/kotlin/konan/internal/RuntimeUtils.kt
+6 -2
View File
@@ -17,7 +17,6 @@
package konan.internal
import kotlin.internal.getProgressionLastElement
import kotlin.text.toUtf8Array
@ExportForCppRuntime
fun ThrowNullPointerException(): Nothing {
@@ -66,6 +65,11 @@ internal fun ThrowNotImplementedError(): Nothing {
throw NotImplementedError("An operation is not implemented.")
}
@ExportForCppRuntime
internal fun ThrowIllegalCharacterConversionException(): Nothing {
throw IllegalCharacterConversionException()
}
@ExportForCppRuntime
fun PrintThrowable(throwable: Throwable) {
println(throwable)
@@ -140,5 +144,5 @@ fun KonanObjectToUtf8Array(value: Any?): ByteArray {
is DoubleArray -> value.contentToString()
else -> value.toString()
}
return toUtf8Array(string, 0, string.length)
return string.toUtf8()
}
runtime/src/main/kotlin/kotlin/Exceptions.kt
+5
View File
@@ -216,4 +216,9 @@ public open class NumberFormatException : IllegalArgumentException {
}
constructor(s: String) : super(s) {}
}
public open class IllegalCharacterConversionException : IllegalArgumentException {
constructor(): super()
constructor(s: String) : super(s)
}
runtime/src/main/kotlin/kotlin/text/StringBuilder.kt
+43 -4
View File
@@ -16,11 +16,50 @@
package kotlin.text
@SymbolName("Kotlin_String_fromUtf8Array")
external fun fromUtf8Array(array: ByteArray, start: Int, size: Int) : String
// ByteArray -> String (UTF-8 -> UTF-16)
@SymbolName("Kotlin_String_toUtf8Array")
external fun toUtf8Array(string: String, start: Int, size: Int) : ByteArray
@Deprecated("Use ByteArray.stringFromUtf8()", ReplaceWith("array.stringFromUtf8(start, end)"))
fun fromUtf8Array(array: ByteArray, start: Int, size: Int) = array.stringFromUtf8Impl(start, size)
@Deprecated("Use String.toUtf8()", ReplaceWith("string.toUtf8(start, end)"))
fun toUtf8Array(string: String, start: Int, size: Int) : ByteArray = string.toUtf8(start, size)
/**
* Converts an UTF-8 array into a [String]. Replaces invalid input sequences with a default character.
*/
fun ByteArray.stringFromUtf8(start: Int = 0, size: Int = this.size) : String =
stringFromUtf8Impl(start, size)
@SymbolName("Kotlin_ByteArray_stringFromUtf8")
private external fun ByteArray.stringFromUtf8Impl(start: Int, size: Int) : String
/**
* Converts an UTF-8 array into a [String]. Throws [IllegalCharacterConversionException] if the input is invalid.
*/
fun ByteArray.stringFromUtf8OrThrow(start: Int = 0, size: Int = this.size) : String =
stringFromUtf8OrThrowImpl(start, size)
@SymbolName("Kotlin_ByteArray_stringFromUtf8OrThrow")
private external fun ByteArray.stringFromUtf8OrThrowImpl(start: Int, size: Int) : String
// String -> ByteArray (UTF-16 -> UTF-8)
/**
* Converts a [String] into an UTF-8 array. Replaces invalid input sequences with a default character.
*/
fun String.toUtf8(start: Int = 0, size: Int = this.length) : ByteArray =
toUtf8Impl(start, size)
@SymbolName("Kotlin_String_toUtf8")
private external fun String.toUtf8Impl(start: Int, size: Int) : ByteArray
/**
* Converts a [String] into an UTF-8 array. Throws [IllegalCharacterConversionException] if the input is invalid.
*/
fun String.toUtf8OrThrow(start: Int = 0, size: Int = this.length) : ByteArray =
toUtf8OrThrowImpl(start, size)
@SymbolName("Kotlin_String_toUtf8OrThrow")
private external fun String.toUtf8OrThrowImpl(start: Int, size: Int) : ByteArray
// TODO: make it somewhat private?
@SymbolName("Kotlin_String_fromCharArray")