diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileDeserializer.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileDeserializer.kt index dd2d4861c5b..cf6eaed71b4 100644 --- a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileDeserializer.kt +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileDeserializer.kt @@ -7,11 +7,14 @@ package org.jetbrains.kotlin.backend.common.serialization import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.descriptors.impl.EmptyPackageFragmentDescriptor -import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.declarations.IrDeclaration +import org.jetbrains.kotlin.ir.declarations.IrFile import org.jetbrains.kotlin.ir.declarations.impl.IrFileImpl import org.jetbrains.kotlin.ir.symbols.impl.IrFileSymbolImpl -import org.jetbrains.kotlin.ir.util.* +import org.jetbrains.kotlin.ir.util.IdSignature +import org.jetbrains.kotlin.ir.util.NaiveSourceBasedFileEntryImpl import org.jetbrains.kotlin.library.IrLibrary +import org.jetbrains.kotlin.library.encodings.WobblyTF8 import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.protobuf.CodedInputStream import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite @@ -146,7 +149,7 @@ class IrLibraryFileFromKlib(private val klib: IrLibrary, private val fileIndex: override fun body(index: Int): ByteArray = klib.body(index, fileIndex) } -internal fun IrLibraryFile.deserializeString(index: Int): String = String(string(index)) +internal fun IrLibraryFile.deserializeString(index: Int): String = WobblyTF8.decode(string(index)) internal fun IrLibraryFile.deserializeFqName(fqn: List): String = fqn.joinToString(".", transform = ::deserializeString) diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileSerializer.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileSerializer.kt index cddc194752d..697d702a3c4 100644 --- a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileSerializer.kt +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileSerializer.kt @@ -21,6 +21,7 @@ import org.jetbrains.kotlin.ir.visitors.acceptVoid import org.jetbrains.kotlin.library.* import org.jetbrains.kotlin.library.impl.IrMemoryArrayWriter import org.jetbrains.kotlin.library.impl.IrMemoryDeclarationWriter +import org.jetbrains.kotlin.library.impl.IrMemoryStringWriter import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.types.Variance import org.jetbrains.kotlin.backend.common.serialization.proto.AccessorIdSignature as ProtoAccessorIdSignature @@ -1324,7 +1325,7 @@ open class IrFileSerializer( file.path, IrMemoryArrayWriter(protoTypeArray.map { it.toByteArray() }).writeIntoMemory(), IrMemoryArrayWriter(protoIdSignatureArray.map { it.toByteArray() }).writeIntoMemory(), - IrMemoryArrayWriter(protoStringArray.map { it.toByteArray() }).writeIntoMemory(), + IrMemoryStringWriter(protoStringArray).writeIntoMemory(), IrMemoryArrayWriter(protoBodyArray.map { it.toByteArray() }).writeIntoMemory(), IrMemoryDeclarationWriter(topLevelDeclarations).writeIntoMemory() ) diff --git a/compiler/util-klib/build.gradle.kts b/compiler/util-klib/build.gradle.kts index eff4b593822..61a8af578e6 100644 --- a/compiler/util-klib/build.gradle.kts +++ b/compiler/util-klib/build.gradle.kts @@ -8,11 +8,12 @@ description = "Common klib reader and writer" dependencies { compile(kotlinStdlib()) compile(project(":kotlin-util-io")) + testImplementation(commonDep("junit:junit")) } sourceSets { "main" { projectDefault() } - "test" { none() } + "test" { projectDefault() } } publish() diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/encodings/WobblyTF8.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/encodings/WobblyTF8.kt new file mode 100644 index 00000000000..ca5a297ca72 --- /dev/null +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/encodings/WobblyTF8.kt @@ -0,0 +1,167 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +@file:Suppress("NOTHING_TO_INLINE") + +package org.jetbrains.kotlin.library.encodings + +/** + * Implementation of Wobbly Transformation Format 8, a superset of UTF-8 + * that encodes surrogate code points if they are not in a pair. + * + * See also https://simonsapin.github.io/wtf-8/ + */ +object WobblyTF8 { + + fun encode(string: String): ByteArray { + val stringLength = string.length + if (stringLength == 0) return EMPTY_BYTE_ARRAY + + val buffer = ByteArray(stringLength * 3) // Allocate for the worse case. + var writtenBytes = 0 + + var index = 0 + while (index < stringLength) { + val char1 = string[index++] + + if (char1 < '\u0080') { + // U+0000..U+007F -> 0xxxxxxx + // 7 meaningful bits -> 1 byte + buffer[writtenBytes++] = char1.toInt() + } else if (char1 < '\u0800') { + // U+0080..U+07FF -> 110xxxxx 10xxxxxx + // 11 meaningful bits -> 2 bytes + val codePoint = char1.toInt() + buffer[writtenBytes++] = (codePoint ushr 6) or 0b1100_0000 + buffer[writtenBytes++] = (codePoint and 0b0011_1111) or 0b1000_0000 + } else { + if (Character.isHighSurrogate(char1) && index < stringLength) { + val char2 = string[index] + if (Character.isLowSurrogate(char2)) { + // a pair of surrogates, encode as in traditional UTF8 + // U+D800..U+DBFF + U+DC00..U+DFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + // 21 meaningful bits -> 4 bytes + index++ + val codePoint = Character.toCodePoint(char1, char2) + buffer[writtenBytes++] = (codePoint ushr 18) or 0b1111_0000 + buffer[writtenBytes++] = ((codePoint ushr 12) and 0b0011_1111) or 0b1000_0000 + buffer[writtenBytes++] = ((codePoint ushr 6) and 0b0011_1111) or 0b1000_0000 + buffer[writtenBytes++] = (codePoint and 0b0011_1111) or 0b1000_0000 + continue + } + } + + // U+0800..U+FFFF -> 1110xxxx 10xxxxxx 10xxxxxx + // 16 meaningful bits -> 3 bytes + val codePoint = char1.toInt() + buffer[writtenBytes++] = (codePoint ushr 12) or 0b1110_0000 + buffer[writtenBytes++] = ((codePoint ushr 6) and 0b0011_1111) or 0b1000_0000 + buffer[writtenBytes++] = (codePoint and 0b0011_1111) or 0b1000_0000 + } + } + + return if (buffer.size == writtenBytes) buffer else buffer.copyOf(writtenBytes) + } + + fun decode(array: ByteArray): String { + val arraySize = array.size + if (arraySize == 0) return EMPTY_STRING + + val buffer = CharArray(arraySize) // Allocate for the worse case. + var charsWritten = 0 + + var index = 0 + while (index < arraySize) { + val byte1 = array.readByteAsInt(index++) + + if (byte1 and 0b1000_0000 == 0) { + // 0xxxxxxx -> U+0000..U+007F + // 1 byte -> 7 meaningful bits + buffer[charsWritten++] = byte1 + continue + } else if (byte1 ushr 5 == 0b000_0110) { + // 110xxxxx 10xxxxxx -> U+0080..U+07FF + // 2 bytes -> 11 meaningful bits + if (index < arraySize) { + val byte2 = array.readByteAsInt(index) + if (isValidContinuation(byte2)) { + index++ + buffer[charsWritten++] = ((byte1 and 0b0001_1111) shl 6) or (byte2 and 0b0011_1111) + continue + } + } + } else if (byte1 ushr 4 == 0b0000_1110) { + // 1110xxxx 10xxxxxx 10xxxxxx -> U+0800..U+FFFF + // 3 bytes -> 16 meaningful bits + if (index < arraySize) { + val byte2 = array.readByteAsInt(index) + if (isValidContinuation(byte2)) { + index++ + if (index < arraySize) { + val byte3 = array.readByteAsInt(index) + if (isValidContinuation(byte3)) { + index++ + buffer[charsWritten++] = ((byte1 and 0b0000_1111) shl 12) or + ((byte2 and 0b0011_1111) shl 6) or + (byte3 and 0b0011_1111) + continue + } + } + } + } + } else if (byte1 ushr 3 == 0b0001_1110) { + // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx -> U+D800..U+DBFF + U+DC00..U+DFFF + // 4 bytes -> 21 meaningful bits + if (index < arraySize) { + val byte2 = array.readByteAsInt(index) + if (isValidContinuation(byte2)) { + index++ + if (index < arraySize) { + val byte3 = array.readByteAsInt(index) + if (isValidContinuation(byte3)) { + index++ + if (index < arraySize) { + val byte4 = array.readByteAsInt(index) + if (isValidContinuation(byte4)) { + index++ + val codePoint = ((byte1 and 0b0000_0111) shl 18) or + ((byte2 and 0b0011_1111) shl 12) or + ((byte3 and 0b0011_1111) shl 6) or + (byte4 and 0b0011_1111) + buffer[charsWritten++] = Character.highSurrogate(codePoint) + buffer[charsWritten++] = Character.lowSurrogate(codePoint) + continue + } + } + } + } + } + } + } + + // unexpected end of the byte sequence or unexpected bit pattern + buffer[charsWritten++] = REPLACEMENT_CHAR + } + + return if (buffer.size == charsWritten) String(buffer) else String(buffer, 0, charsWritten) + } + + private fun ByteArray.readByteAsInt(index: Int): Int = this[index].toInt() and 0b1111_1111 + + private operator fun ByteArray.set(index: Int, value: Int) { + this[index] = value.toByte() + } + + private operator fun CharArray.set(index: Int, value: Int) { + this[index] = value.toChar() + } + + private fun isValidContinuation(byteN: Int) = byteN ushr 6 == 0b0000_0010 + + private val EMPTY_BYTE_ARRAY = byteArrayOf() + private const val EMPTY_STRING = "" + + private const val REPLACEMENT_CHAR = '\ufffd' +} diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrFileWriters.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrFileWriters.kt index 5bdc51f0de2..66f27dceb4e 100644 --- a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrFileWriters.kt +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrFileWriters.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.library.impl import org.jetbrains.kotlin.library.SerializedDeclaration +import org.jetbrains.kotlin.library.encodings.WobblyTF8 import java.io.ByteArrayOutputStream import java.io.DataOutput import java.io.DataOutputStream @@ -62,6 +63,16 @@ class IrMemoryArrayWriter(private val data: List) : IrMemoryWriter() } } +class IrMemoryStringWriter(private val data: List) : IrMemoryWriter() { + override fun writeData(dataOutput: DataOutput) { + dataOutput.writeInt(data.size) + + val transformedData = data.map(WobblyTF8::encode) + + transformedData.forEach { dataOutput.writeInt(it.size) } + transformedData.forEach { dataOutput.write(it) } + } +} class IrByteArrayWriter(private val data: List) : IrFileWriter() { override fun writeData(dataOutput: DataOutput) { diff --git a/compiler/util-klib/test/org/jetbrains/kotlin/library/encodings/WobblyTF8Test.kt b/compiler/util-klib/test/org/jetbrains/kotlin/library/encodings/WobblyTF8Test.kt new file mode 100644 index 00000000000..5455f643bde --- /dev/null +++ b/compiler/util-klib/test/org/jetbrains/kotlin/library/encodings/WobblyTF8Test.kt @@ -0,0 +1,203 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.library.encodings + +import org.junit.Assert.* +import org.junit.Test +import java.lang.Character.* +import kotlin.experimental.and +import kotlin.experimental.inv +import kotlin.experimental.or + +class WobblyTF8Test { + // Empty string decoding and encoding by Wobbly and UTF8. + @Test + fun emptyString() { + val emptyString = "" + + val encodedWithUTF8: ByteArray = emptyString.encodeWithUTF8() + val encodedWithWobbly: ByteArray = emptyString.encodeWithWobbly() + assertArrayEquals(encodedWithUTF8, encodedWithWobbly) + + val decodedWithUTF8: String = encodedWithUTF8.decodeWithUTF8() + assertEquals(emptyString, decodedWithUTF8) + val decodedWithWobbly: String = encodedWithWobbly.decodeWithWobbly() + assertEquals(emptyString, decodedWithWobbly) + } + + // Well-formed UTF16 string decoding and encoding by Wobbly and UTF8. + @Test + fun wellFormedString() { + repeat(10) { + val wellFormedString = generateWellFormedString( + bmpCodePointsBeforeSurrogates = 10_000, + bmpCodePointsAfterSurrogates = 10_000, + supplementaryCodePoints = 80_000 + ) + + val encodedWithUTF8: ByteArray = wellFormedString.encodeWithUTF8() + val encodedWithWobbly: ByteArray = wellFormedString.encodeWithWobbly() + assertArrayEquals(encodedWithUTF8, encodedWithWobbly) + + val decodedWithUTF8: String = encodedWithUTF8.decodeWithUTF8() + assertEquals(wellFormedString, decodedWithUTF8) + val decodedWithWobbly: String = encodedWithWobbly.decodeWithWobbly() + assertEquals(wellFormedString, decodedWithWobbly) + } + } + + // Ill-formed string encoding by UTF8, then decoding it by Wobbly and UTF8. + @Test + fun illFormedStringEncodingByUTF8DecodingByWobbly() { + repeat(10) { + val illFormedString = generateIllFormedString( + bmpCodePointsBeforeSurrogates = 10_000, + isolatedSurrogates = 10_000, + bmpCodePointsAfterSurrogates = 10_000, + supplementaryCodePoints = 80_000 + ) + + val encodedWithUTF8: ByteArray = illFormedString.encodeWithUTF8() + + val decodedWithUTF8: String = encodedWithUTF8.decodeWithUTF8() + assertNotEquals(illFormedString, decodedWithUTF8) + val decodedWithWobbly: String = encodedWithUTF8.decodeWithWobbly() + assertEquals(decodedWithUTF8, decodedWithWobbly) + } + } + + // Ill-formed string lossless encoding and decoding by Wobbly. + @Test + fun illFormedStringEncodingByWobblyDecodingByWobbly() { + repeat(10) { + val illFormedString = generateIllFormedString( + bmpCodePointsBeforeSurrogates = 10_000, + isolatedSurrogates = 10_000, + bmpCodePointsAfterSurrogates = 10_000, + supplementaryCodePoints = 80_000 + ) + + val encodedWithUTF8: ByteArray = illFormedString.encodeWithUTF8() + val encodedWithWobbly: ByteArray = illFormedString.encodeWithWobbly() + assertFalse(encodedWithUTF8.contentEquals(encodedWithWobbly)) + + val decodedWithWobbly: String = encodedWithWobbly.decodeWithWobbly() + assertEquals(illFormedString, decodedWithWobbly) + } + } + + @Test + fun decodingMalformedByteSequence() { + fun ByteArray.assertDecodedSimilarly() { + val decodedWithUTF8 = decodeWithUTF8() + val decodedWithWobbly = decodeWithWobbly() + assertEquals(decodedWithUTF8, decodedWithWobbly) + } + + repeat(10) { + val codePoint = CODE_POINTS_SUPPLEMENTARY.random() + val encodedWithUTF8: ByteArray = buildString { appendCodePoint(codePoint) }.encodeWithUTF8() + + with(encodedWithUTF8.copyOf()) { + setBit(0, 3) // first byte starts with 1111_1000 + assertDecodedSimilarly() + + setBit(0, 2) // first byte starts with 1111_1100 + assertDecodedSimilarly() + + setBit(0, 1) // first byte starts with 1111_1110 + assertDecodedSimilarly() + + setBit(0, 1) // first byte starts with 1111_1111 + assertDecodedSimilarly() + } + + with(encodedWithUTF8.copyOf()) { + for (byteIndex in (size - 1) downTo 0) { + clearBit(byteIndex, 7) // any byte starts with 0 + assertDecodedSimilarly() + } + } + + with(encodedWithUTF8.copyOf()) { + for (byteIndex in (size - 2) downTo 0) { + set(byteIndex, 6) // continuation byte starts with 11 + assertDecodedSimilarly() + } + } + + // chopping byte sequence end + for (newLength in (encodedWithUTF8.size - 1) downTo 1) { + with(encodedWithUTF8.copyOf(newLength)) { + assertDecodedSimilarly() + } + } + + // chopping byte sequence start + for (newLength in (encodedWithUTF8.size - 1) downTo 1) { + with(encodedWithUTF8.copyOfRange(encodedWithUTF8.size - newLength, encodedWithUTF8.size)) { + assertDecodedSimilarly() + } + } + } + } + + private companion object { + fun String.encodeWithWobbly(): ByteArray = WobblyTF8.encode(this) + fun String.encodeWithUTF8(): ByteArray = toByteArray(Charsets.UTF_8) + + fun ByteArray.decodeWithWobbly(): String = WobblyTF8.decode(this) + fun ByteArray.decodeWithUTF8(): String = toString(Charsets.UTF_8) + + val CODE_POINTS_BEFORE_SURROGATES = MIN_CODE_POINT until MIN_SURROGATE.toInt() + val CODE_POINTS_SURROGATES = MIN_SURROGATE.toInt()..MAX_SURROGATE.toInt() + val CODE_POINTS_AFTER_SURROGATES = (MAX_SURROGATE.toInt() + 1) until MIN_SUPPLEMENTARY_CODE_POINT + val CODE_POINTS_SUPPLEMENTARY = MIN_SUPPLEMENTARY_CODE_POINT..MAX_CODE_POINT + + fun generateWellFormedString( + bmpCodePointsBeforeSurrogates: Int, + bmpCodePointsAfterSurrogates: Int, + supplementaryCodePoints: Int + ): String { + val codePoints = ArrayList(bmpCodePointsBeforeSurrogates + bmpCodePointsAfterSurrogates) + repeat(bmpCodePointsBeforeSurrogates) { codePoints += CODE_POINTS_BEFORE_SURROGATES.random() } + repeat(bmpCodePointsAfterSurrogates) { codePoints += CODE_POINTS_AFTER_SURROGATES.random() } + repeat(supplementaryCodePoints) { codePoints += CODE_POINTS_SUPPLEMENTARY.random() } + codePoints.shuffle() + return buildString { codePoints.forEach(::appendCodePoint) } + } + + fun generateIllFormedString( + bmpCodePointsBeforeSurrogates: Int, + isolatedSurrogates: Int, + bmpCodePointsAfterSurrogates: Int, + supplementaryCodePoints: Int + ): String { + val codePoints = ArrayList(bmpCodePointsBeforeSurrogates + isolatedSurrogates + bmpCodePointsAfterSurrogates) + repeat(bmpCodePointsBeforeSurrogates) { codePoints += CODE_POINTS_BEFORE_SURROGATES.random() } + repeat(isolatedSurrogates) { codePoints += CODE_POINTS_SURROGATES.random() } + repeat(bmpCodePointsAfterSurrogates) { codePoints += CODE_POINTS_AFTER_SURROGATES.random() } + repeat(supplementaryCodePoints) { codePoints += CODE_POINTS_SUPPLEMENTARY.random() } + codePoints.shuffle() + + return buildString { + // need to guarantee that at least one isolated surrogate is present + appendCodePoint(CODE_POINTS_SURROGATES.random()) + appendCodePoint(CODE_POINTS_BEFORE_SURROGATES.random()) + + codePoints.forEach(::appendCodePoint) + } + } + + fun ByteArray.setBit(index: Int, bitIndex: Int) { + this[index] = this[index] or (1 shl bitIndex).toByte() + } + + fun ByteArray.clearBit(index: Int, bitIndex: Int) { + this[index] = this[index] and (1 shl bitIndex).toByte().inv() + } + } +}