azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[IR] Use the proper encoding for string serialization

Browse Source 
^KT-33175
This commit is contained in:
Dmitriy Dolovov
2021-03-30 13:17:13 +03:00
parent 7272e5dcad
commit 50326f019b
6 changed files with 391 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileDeserializer.kt
+6 -3
View File
@@ -7,11 +7,14 @@ package org.jetbrains.kotlin.backend.common.serialization
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.impl.EmptyPackageFragmentDescriptor
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.declarations.IrDeclaration
import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.ir.declarations.impl.IrFileImpl
import org.jetbrains.kotlin.ir.symbols.impl.IrFileSymbolImpl
import org.jetbrains.kotlin.ir.util.*
import org.jetbrains.kotlin.ir.util.IdSignature
import org.jetbrains.kotlin.ir.util.NaiveSourceBasedFileEntryImpl
import org.jetbrains.kotlin.library.IrLibrary
import org.jetbrains.kotlin.library.encodings.WobblyTF8
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.protobuf.CodedInputStream
import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
@@ -146,7 +149,7 @@ class IrLibraryFileFromKlib(private val klib: IrLibrary, private val fileIndex:
override fun body(index: Int): ByteArray = klib.body(index, fileIndex)
}
internal fun IrLibraryFile.deserializeString(index: Int): String = String(string(index))
internal fun IrLibraryFile.deserializeString(index: Int): String = WobblyTF8.decode(string(index))
internal fun IrLibraryFile.deserializeFqName(fqn: List<Int>): String =
fqn.joinToString(".", transform = ::deserializeString)
compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileSerializer.kt
+2 -1
View File
@@ -21,6 +21,7 @@ import org.jetbrains.kotlin.ir.visitors.acceptVoid
import org.jetbrains.kotlin.library.*
import org.jetbrains.kotlin.library.impl.IrMemoryArrayWriter
import org.jetbrains.kotlin.library.impl.IrMemoryDeclarationWriter
import org.jetbrains.kotlin.library.impl.IrMemoryStringWriter
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.types.Variance
import org.jetbrains.kotlin.backend.common.serialization.proto.AccessorIdSignature as ProtoAccessorIdSignature
@@ -1324,7 +1325,7 @@ open class IrFileSerializer(
file.path,
IrMemoryArrayWriter(protoTypeArray.map { it.toByteArray() }).writeIntoMemory(),
IrMemoryArrayWriter(protoIdSignatureArray.map { it.toByteArray() }).writeIntoMemory(),
IrMemoryArrayWriter(protoStringArray.map { it.toByteArray() }).writeIntoMemory(),
IrMemoryStringWriter(protoStringArray).writeIntoMemory(),
IrMemoryArrayWriter(protoBodyArray.map { it.toByteArray() }).writeIntoMemory(),
IrMemoryDeclarationWriter(topLevelDeclarations).writeIntoMemory()
)
compiler/util-klib/build.gradle.kts
+2 -1
View File
@@ -8,11 +8,12 @@ description = "Common klib reader and writer"
dependencies {
compile(kotlinStdlib())
compile(project(":kotlin-util-io"))
testImplementation(commonDep("junit:junit"))
}
sourceSets {
"main" { projectDefault() }
"test" { none() }
"test" { projectDefault() }
}
publish()
compiler/util-klib/src/org/jetbrains/kotlin/library/encodings/WobblyTF8.kt
+167
View File
@@ -0,0 +1,167 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
@file:Suppress("NOTHING_TO_INLINE")
package org.jetbrains.kotlin.library.encodings
/**
* Implementation of Wobbly Transformation Format 8, a superset of UTF-8
* that encodes surrogate code points if they are not in a pair.
*
* See also https://simonsapin.github.io/wtf-8/
*/
object WobblyTF8 {
fun encode(string: String): ByteArray {
val stringLength = string.length
if (stringLength == 0) return EMPTY_BYTE_ARRAY
val buffer = ByteArray(stringLength * 3) // Allocate for the worse case.
var writtenBytes = 0
var index = 0
while (index < stringLength) {
val char1 = string[index++]
if (char1 < '\u0080') {
// U+0000..U+007F -> 0xxxxxxx
// 7 meaningful bits -> 1 byte
buffer[writtenBytes++] = char1.toInt()
} else if (char1 < '\u0800') {
// U+0080..U+07FF -> 110xxxxx 10xxxxxx
// 11 meaningful bits -> 2 bytes
val codePoint = char1.toInt()
buffer[writtenBytes++] = (codePoint ushr 6) or 0b1100_0000
buffer[writtenBytes++] = (codePoint and 0b0011_1111) or 0b1000_0000
} else {
if (Character.isHighSurrogate(char1) && index < stringLength) {
val char2 = string[index]
if (Character.isLowSurrogate(char2)) {
// a pair of surrogates, encode as in traditional UTF8
// U+D800..U+DBFF + U+DC00..U+DFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
// 21 meaningful bits -> 4 bytes
index++
val codePoint = Character.toCodePoint(char1, char2)
buffer[writtenBytes++] = (codePoint ushr 18) or 0b1111_0000
buffer[writtenBytes++] = ((codePoint ushr 12) and 0b0011_1111) or 0b1000_0000
buffer[writtenBytes++] = ((codePoint ushr 6) and 0b0011_1111) or 0b1000_0000
buffer[writtenBytes++] = (codePoint and 0b0011_1111) or 0b1000_0000
continue
}
}
// U+0800..U+FFFF -> 1110xxxx 10xxxxxx 10xxxxxx
// 16 meaningful bits -> 3 bytes
val codePoint = char1.toInt()
buffer[writtenBytes++] = (codePoint ushr 12) or 0b1110_0000
buffer[writtenBytes++] = ((codePoint ushr 6) and 0b0011_1111) or 0b1000_0000
buffer[writtenBytes++] = (codePoint and 0b0011_1111) or 0b1000_0000
}
}
return if (buffer.size == writtenBytes) buffer else buffer.copyOf(writtenBytes)
}
fun decode(array: ByteArray): String {
val arraySize = array.size
if (arraySize == 0) return EMPTY_STRING
val buffer = CharArray(arraySize) // Allocate for the worse case.
var charsWritten = 0
var index = 0
while (index < arraySize) {
val byte1 = array.readByteAsInt(index++)
if (byte1 and 0b1000_0000 == 0) {
// 0xxxxxxx -> U+0000..U+007F
// 1 byte -> 7 meaningful bits
buffer[charsWritten++] = byte1
continue
} else if (byte1 ushr 5 == 0b000_0110) {
// 110xxxxx 10xxxxxx -> U+0080..U+07FF
// 2 bytes -> 11 meaningful bits
if (index < arraySize) {
val byte2 = array.readByteAsInt(index)
if (isValidContinuation(byte2)) {
index++
buffer[charsWritten++] = ((byte1 and 0b0001_1111) shl 6) or (byte2 and 0b0011_1111)
continue
}
}
} else if (byte1 ushr 4 == 0b0000_1110) {
// 1110xxxx 10xxxxxx 10xxxxxx -> U+0800..U+FFFF
// 3 bytes -> 16 meaningful bits
if (index < arraySize) {
val byte2 = array.readByteAsInt(index)
if (isValidContinuation(byte2)) {
index++
if (index < arraySize) {
val byte3 = array.readByteAsInt(index)
if (isValidContinuation(byte3)) {
index++
buffer[charsWritten++] = ((byte1 and 0b0000_1111) shl 12) or
((byte2 and 0b0011_1111) shl 6) or
(byte3 and 0b0011_1111)
continue
}
}
}
}
} else if (byte1 ushr 3 == 0b0001_1110) {
// 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx -> U+D800..U+DBFF + U+DC00..U+DFFF
// 4 bytes -> 21 meaningful bits
if (index < arraySize) {
val byte2 = array.readByteAsInt(index)
if (isValidContinuation(byte2)) {
index++
if (index < arraySize) {
val byte3 = array.readByteAsInt(index)
if (isValidContinuation(byte3)) {
index++
if (index < arraySize) {
val byte4 = array.readByteAsInt(index)
if (isValidContinuation(byte4)) {
index++
val codePoint = ((byte1 and 0b0000_0111) shl 18) or
((byte2 and 0b0011_1111) shl 12) or
((byte3 and 0b0011_1111) shl 6) or
(byte4 and 0b0011_1111)
buffer[charsWritten++] = Character.highSurrogate(codePoint)
buffer[charsWritten++] = Character.lowSurrogate(codePoint)
continue
}
}
}
}
}
}
}
// unexpected end of the byte sequence or unexpected bit pattern
buffer[charsWritten++] = REPLACEMENT_CHAR
}
return if (buffer.size == charsWritten) String(buffer) else String(buffer, 0, charsWritten)
}
private fun ByteArray.readByteAsInt(index: Int): Int = this[index].toInt() and 0b1111_1111
private operator fun ByteArray.set(index: Int, value: Int) {
this[index] = value.toByte()
}
private operator fun CharArray.set(index: Int, value: Int) {
this[index] = value.toChar()
}
private fun isValidContinuation(byteN: Int) = byteN ushr 6 == 0b0000_0010
private val EMPTY_BYTE_ARRAY = byteArrayOf()
private const val EMPTY_STRING = ""
private const val REPLACEMENT_CHAR = '\ufffd'
}
compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrFileWriters.kt
+11
View File
@@ -6,6 +6,7 @@
package org.jetbrains.kotlin.library.impl
import org.jetbrains.kotlin.library.SerializedDeclaration
import org.jetbrains.kotlin.library.encodings.WobblyTF8
import java.io.ByteArrayOutputStream
import java.io.DataOutput
import java.io.DataOutputStream
@@ -62,6 +63,16 @@ class IrMemoryArrayWriter(private val data: List<ByteArray>) : IrMemoryWriter()
}
}
class IrMemoryStringWriter(private val data: List<String>) : IrMemoryWriter() {
override fun writeData(dataOutput: DataOutput) {
dataOutput.writeInt(data.size)
val transformedData = data.map(WobblyTF8::encode)
transformedData.forEach { dataOutput.writeInt(it.size) }
transformedData.forEach { dataOutput.write(it) }
}
}
class IrByteArrayWriter(private val data: List<ByteArray>) : IrFileWriter() {
override fun writeData(dataOutput: DataOutput) {
compiler/util-klib/test/org/jetbrains/kotlin/library/encodings/WobblyTF8Test.kt
+203
View File
@@ -0,0 +1,203 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.library.encodings
import org.junit.Assert.*
import org.junit.Test
import java.lang.Character.*
import kotlin.experimental.and
import kotlin.experimental.inv
import kotlin.experimental.or
class WobblyTF8Test {
// Empty string decoding and encoding by Wobbly and UTF8.
@Test
fun emptyString() {
val emptyString = ""
val encodedWithUTF8: ByteArray = emptyString.encodeWithUTF8()
val encodedWithWobbly: ByteArray = emptyString.encodeWithWobbly()
assertArrayEquals(encodedWithUTF8, encodedWithWobbly)
val decodedWithUTF8: String = encodedWithUTF8.decodeWithUTF8()
assertEquals(emptyString, decodedWithUTF8)
val decodedWithWobbly: String = encodedWithWobbly.decodeWithWobbly()
assertEquals(emptyString, decodedWithWobbly)
}
// Well-formed UTF16 string decoding and encoding by Wobbly and UTF8.
@Test
fun wellFormedString() {
repeat(10) {
val wellFormedString = generateWellFormedString(
bmpCodePointsBeforeSurrogates = 10_000,
bmpCodePointsAfterSurrogates = 10_000,
supplementaryCodePoints = 80_000
)
val encodedWithUTF8: ByteArray = wellFormedString.encodeWithUTF8()
val encodedWithWobbly: ByteArray = wellFormedString.encodeWithWobbly()
assertArrayEquals(encodedWithUTF8, encodedWithWobbly)
val decodedWithUTF8: String = encodedWithUTF8.decodeWithUTF8()
assertEquals(wellFormedString, decodedWithUTF8)
val decodedWithWobbly: String = encodedWithWobbly.decodeWithWobbly()
assertEquals(wellFormedString, decodedWithWobbly)
}
}
// Ill-formed string encoding by UTF8, then decoding it by Wobbly and UTF8.
@Test
fun illFormedStringEncodingByUTF8DecodingByWobbly() {
repeat(10) {
val illFormedString = generateIllFormedString(
bmpCodePointsBeforeSurrogates = 10_000,
isolatedSurrogates = 10_000,
bmpCodePointsAfterSurrogates = 10_000,
supplementaryCodePoints = 80_000
)
val encodedWithUTF8: ByteArray = illFormedString.encodeWithUTF8()
val decodedWithUTF8: String = encodedWithUTF8.decodeWithUTF8()
assertNotEquals(illFormedString, decodedWithUTF8)
val decodedWithWobbly: String = encodedWithUTF8.decodeWithWobbly()
assertEquals(decodedWithUTF8, decodedWithWobbly)
}
}
// Ill-formed string lossless encoding and decoding by Wobbly.
@Test
fun illFormedStringEncodingByWobblyDecodingByWobbly() {
repeat(10) {
val illFormedString = generateIllFormedString(
bmpCodePointsBeforeSurrogates = 10_000,
isolatedSurrogates = 10_000,
bmpCodePointsAfterSurrogates = 10_000,
supplementaryCodePoints = 80_000
)
val encodedWithUTF8: ByteArray = illFormedString.encodeWithUTF8()
val encodedWithWobbly: ByteArray = illFormedString.encodeWithWobbly()
assertFalse(encodedWithUTF8.contentEquals(encodedWithWobbly))
val decodedWithWobbly: String = encodedWithWobbly.decodeWithWobbly()
assertEquals(illFormedString, decodedWithWobbly)
}
}
@Test
fun decodingMalformedByteSequence() {
fun ByteArray.assertDecodedSimilarly() {
val decodedWithUTF8 = decodeWithUTF8()
val decodedWithWobbly = decodeWithWobbly()
assertEquals(decodedWithUTF8, decodedWithWobbly)
}
repeat(10) {
val codePoint = CODE_POINTS_SUPPLEMENTARY.random()
val encodedWithUTF8: ByteArray = buildString { appendCodePoint(codePoint) }.encodeWithUTF8()
with(encodedWithUTF8.copyOf()) {
setBit(0, 3) // first byte starts with 1111_1000
assertDecodedSimilarly()
setBit(0, 2) // first byte starts with 1111_1100
assertDecodedSimilarly()
setBit(0, 1) // first byte starts with 1111_1110
assertDecodedSimilarly()
setBit(0, 1) // first byte starts with 1111_1111
assertDecodedSimilarly()
}
with(encodedWithUTF8.copyOf()) {
for (byteIndex in (size - 1) downTo 0) {
clearBit(byteIndex, 7) // any byte starts with 0
assertDecodedSimilarly()
}
}
with(encodedWithUTF8.copyOf()) {
for (byteIndex in (size - 2) downTo 0) {
set(byteIndex, 6) // continuation byte starts with 11
assertDecodedSimilarly()
}
}
// chopping byte sequence end
for (newLength in (encodedWithUTF8.size - 1) downTo 1) {
with(encodedWithUTF8.copyOf(newLength)) {
assertDecodedSimilarly()
}
}
// chopping byte sequence start
for (newLength in (encodedWithUTF8.size - 1) downTo 1) {
with(encodedWithUTF8.copyOfRange(encodedWithUTF8.size - newLength, encodedWithUTF8.size)) {
assertDecodedSimilarly()
}
}
}
}
private companion object {
fun String.encodeWithWobbly(): ByteArray = WobblyTF8.encode(this)
fun String.encodeWithUTF8(): ByteArray = toByteArray(Charsets.UTF_8)
fun ByteArray.decodeWithWobbly(): String = WobblyTF8.decode(this)
fun ByteArray.decodeWithUTF8(): String = toString(Charsets.UTF_8)
val CODE_POINTS_BEFORE_SURROGATES = MIN_CODE_POINT until MIN_SURROGATE.toInt()
val CODE_POINTS_SURROGATES = MIN_SURROGATE.toInt()..MAX_SURROGATE.toInt()
val CODE_POINTS_AFTER_SURROGATES = (MAX_SURROGATE.toInt() + 1) until MIN_SUPPLEMENTARY_CODE_POINT
val CODE_POINTS_SUPPLEMENTARY = MIN_SUPPLEMENTARY_CODE_POINT..MAX_CODE_POINT
fun generateWellFormedString(
bmpCodePointsBeforeSurrogates: Int,
bmpCodePointsAfterSurrogates: Int,
supplementaryCodePoints: Int
): String {
val codePoints = ArrayList<Int>(bmpCodePointsBeforeSurrogates + bmpCodePointsAfterSurrogates)
repeat(bmpCodePointsBeforeSurrogates) { codePoints += CODE_POINTS_BEFORE_SURROGATES.random() }
repeat(bmpCodePointsAfterSurrogates) { codePoints += CODE_POINTS_AFTER_SURROGATES.random() }
repeat(supplementaryCodePoints) { codePoints += CODE_POINTS_SUPPLEMENTARY.random() }
codePoints.shuffle()
return buildString { codePoints.forEach(::appendCodePoint) }
}
fun generateIllFormedString(
bmpCodePointsBeforeSurrogates: Int,
isolatedSurrogates: Int,
bmpCodePointsAfterSurrogates: Int,
supplementaryCodePoints: Int
): String {
val codePoints = ArrayList<Int>(bmpCodePointsBeforeSurrogates + isolatedSurrogates + bmpCodePointsAfterSurrogates)
repeat(bmpCodePointsBeforeSurrogates) { codePoints += CODE_POINTS_BEFORE_SURROGATES.random() }
repeat(isolatedSurrogates) { codePoints += CODE_POINTS_SURROGATES.random() }
repeat(bmpCodePointsAfterSurrogates) { codePoints += CODE_POINTS_AFTER_SURROGATES.random() }
repeat(supplementaryCodePoints) { codePoints += CODE_POINTS_SUPPLEMENTARY.random() }
codePoints.shuffle()
return buildString {
// need to guarantee that at least one isolated surrogate is present
appendCodePoint(CODE_POINTS_SURROGATES.random())
appendCodePoint(CODE_POINTS_BEFORE_SURROGATES.random())
codePoints.forEach(::appendCodePoint)
}
}
fun ByteArray.setBit(index: Int, bitIndex: Int) {
this[index] = this[index] or (1 shl bitIndex).toByte()
}
fun ByteArray.clearBit(index: Int, bitIndex: Int) {
this[index] = this[index] and (1 shl bitIndex).toByte().inv()
}
}
}