[IC KLIB] Serialization infrastructure refactoring

- remove IrModule header
 - put IrDeclaration and its body into separate files
 - use only one type of IrData index
 - separate deserialization of declarations and its bodies
 - introduce FqName message
This commit is contained in:
Roman Artemev
2019-07-31 11:48:26 +03:00
committed by romanart
parent 5a45960c04
commit 94a03c9cb4
118 changed files with 4208 additions and 2371 deletions
@@ -37,6 +37,13 @@ interface IrLibrary {
fun symbol(index: Int): ByteArray
fun type(index: Int): ByteArray
fun string(index: Int): ByteArray
fun irDeclaration(index: Long, isLocal: Boolean, fileIndex: Int): ByteArray
fun symbol(index: Int, fileIndex: Int): ByteArray
fun type(index: Int, fileIndex: Int): ByteArray
fun string(index: Int, fileIndex: Int): ByteArray
fun body(index: Int, fileIndex: Int): ByteArray
fun file(index: Int): ByteArray
fun fileCount(): Int
}
val BaseKotlinLibrary.uniqueName: String
@@ -53,13 +53,35 @@ interface IrKotlinLibraryLayout : KotlinLibraryLayout {
val irHeader
get() = File(irDir, "irHeaders.kni")
val irDeclarations
get() = File(irTablesDir, "irCombined.knd")
get() = File(irDir, "irDeclarations.knd")
val irSymbols
get() = File(irTablesDir, "symbols.knt")
get() = File(irDir, "symbols.knt")
val irTypes
get() = File(irTablesDir, "types.knt")
get() = File(irDir, "types.knt")
val irStrings
get() = File(irTablesDir, "strings.knt")
get() = File(irDir, "strings.knt")
val irBodies
get() = File(irDir, "bodies.knb")
val irFiles
get() = File(irDir, "files.knf")
val dataFlowGraphFile
get() = File(irDir, "module_data_flow_graph")
fun irDeclarations(file: File): File = File(file, "irCombined.knd")
fun irSymbols(file: File): File = File(file, "symbols.knt")
fun irTypes(file: File): File = File(file, "types.knt")
fun irStrings(file: File): File = File(file, "strings.knt")
fun irBodies(file: File): File = File(file, "body.knb")
fun irFile(file: File): File = File(file, "file.knf")
val irFilesX
get() = run {
val fileDirectories = mutableListOf<File>()
irDir.postorder {
if (it.fileName.toString().endsWith(".file")) {
fileDirectories.add(File(it))
}
}
fileDirectories
}
}
@@ -20,7 +20,7 @@ interface MetadataWriter {
}
interface IrWriter {
fun addIr(ir: SerializedIr)
fun addIr(ir: SerializedIrModule)
fun addDataFlowGraph(dataFlowGraph: ByteArray)
}
@@ -33,8 +33,37 @@ class SerializedMetadata(
val fragmentNames: List<String>
)
class SerializedIrSymbol(
val symbolData: ByteArray,
val id: Long,
val isLocal: Boolean
)
//class SerializedIrFile(
// val fileData: ByteArray,
// val symbols: List<SerializedIrSymbol>,
// val types: List<ByteArray>,
// val strings: List<ByteArray>
//)
class IrIrSerializedIrFile(
val fileData: ByteArray,
// val fileProto: ProtoFile,
val fqName: List<String>,
val path: String,
// val symbols: List<SerializedIrSymbol>,
val symbols: List<ByteArray>,
val types: List<ByteArray>,
val strings: List<ByteArray>,
val bodies: List<ByteArray>,
val declarations: List<SerializedDeclaration>
)
class SerializedIrModule(val files: Collection<IrIrSerializedIrFile>)
class SerializedIr(
val module: ByteArray,
// val symbols: List<Pair<Long, ByteArray>>,
val symbols: List<ByteArray>,
val types: List<ByteArray>,
val strings: List<ByteArray>,
@@ -6,12 +6,15 @@
package org.jetbrains.kotlin.library.impl
import org.jetbrains.kotlin.konan.file.File
import java.nio.ByteBuffer
import java.nio.channels.FileChannel
class SimpleIrTableFileReader(file: File) {
class IrArrayReader(file: File) {
private val buffer = file.map(FileChannel.MapMode.READ_ONLY)
private val indexToOffset: IntArray
fun entryCount() = indexToOffset.size - 1
init {
val count = buffer.int
indexToOffset = IntArray(count + 1)
@@ -32,29 +35,141 @@ class SimpleIrTableFileReader(file: File) {
}
}
data class DeclarationId(val id: Long, val isLocal: Boolean)
class CombinedIrFileReader(file: File) {
class IrMultiArrayReader(file: File) {
private val buffer = file.map(FileChannel.MapMode.READ_ONLY)
private val declarationToOffsetSize = mutableMapOf<DeclarationId, Pair<Int, Int>>()
private val indexToOffset: IntArray
private val indexIndexToOffset = mutableMapOf<Int, IntArray>()
private fun readOffsets(position: Int): IntArray {
buffer.position(position)
val count = buffer.int
val result = IntArray(count + 1)
result[0] = 4 * (count + 1)
for (i in 0 until count) {
val size = buffer.int
result[i + 1] = result[i] + size
}
return result
}
init {
val declarationsCount = buffer.int
for (i in 0 until declarationsCount) {
val id = buffer.long
val isLocal = buffer.int != 0
indexToOffset = readOffsets(0)
}
fun tableItemBytes(id: Int): ByteArray {
val offset = indexToOffset[id]
val size = indexToOffset[id + 1] - offset
val result = ByteArray(size)
buffer.position(offset)
buffer.get(result, 0, size)
return result
}
fun tableItemBytes(row: Int, column: Int): ByteArray {
val rowOffset = indexToOffset[row]
val collumnOffsets = indexIndexToOffset.getOrPut(row) {
readOffsets(rowOffset)
}
val dataOffset = collumnOffsets[column]
val dataSize = collumnOffsets[column + 1] - dataOffset
val result = ByteArray(dataSize)
buffer.position(rowOffset + dataOffset)
buffer.get(result, 0, dataSize)
return result
}
}
abstract class IrMultiTableReader<K>(file: File, private val keyReader: ByteBuffer.() -> K) {
private val buffer = file.map(FileChannel.MapMode.READ_ONLY)
private val indexToOffset: IntArray
private val indexToIndexMap = mutableMapOf<Int, Map<K, Pair<Int, Int>>>()
private fun readOffsets(position: Int): IntArray {
buffer.position(position)
val count = buffer.int
val result = IntArray(count + 1)
result[0] = 4 * (count + 1)
for (i in 0 until count) {
val size = buffer.int
result[i + 1] = result[i] + size
}
return result
}
init {
indexToOffset = readOffsets(0)
}
private fun readIndexMap(position: Int): Map<K, Pair<Int, Int>> {
buffer.position(position)
val result = mutableMapOf<K, Pair<Int, Int>>()
val count = buffer.int
for (i in 0 until count) {
val key = keyReader(buffer)
val offset = buffer.int
val size = buffer.int
declarationToOffsetSize[DeclarationId(id, isLocal)] = offset to size
result[key] = offset to size
}
return result
}
fun tableItemBytes(row: Int, id: K): ByteArray {
val rowOffset = indexToOffset[row]
val indexToMap = indexToIndexMap.getOrPut(row) {
readIndexMap(rowOffset)
}
val coordinates = indexToMap[id] ?: error("No coordinates found for $id")
val offset = coordinates.first
val size = coordinates.second
val result = ByteArray(size)
buffer.position(rowOffset + offset)
buffer.get(result, 0, size)
return result
}
}
abstract class IrTableReader<K>(file: File, keyReader: ByteBuffer.() -> K) {
private val buffer = file.map(FileChannel.MapMode.READ_ONLY)
private val indexToOffset = mutableMapOf<K, Pair<Int, Int>>()
init {
val count = buffer.int
for (i in 0 until count) {
val key = keyReader(buffer)
val offset = buffer.int
val size = buffer.int
indexToOffset[key] = offset to size
}
}
fun declarationBytes(id: DeclarationId): ByteArray {
val offsetSize = declarationToOffsetSize[id] ?: throw Error("No declaration with $id here")
val result = ByteArray(offsetSize.second)
buffer.position(offsetSize.first)
buffer.get(result, 0, offsetSize.second)
fun tableItemBytes(id: K): ByteArray {
val coordinates = indexToOffset[id] ?: error("No coordinates found for $id")
val offset = coordinates.first
val size = coordinates.second
val result = ByteArray(size)
buffer.position(offset)
buffer.get(result, 0, size)
return result
}
}
}
class IndexIrTableReader(file: File) : IrTableReader<Long>(file, { long })
data class DeclarationId(val id: Long, val isLocal: Boolean)
class DeclarationIrTableReader(file: File) : IrTableReader<DeclarationId>(file, { DeclarationId(long, int != 0) })
class DeclarationIrMultiTableReader(file: File) : IrMultiTableReader<DeclarationId>(file, { DeclarationId(long, int != 0) })
@@ -5,6 +5,7 @@
package org.jetbrains.kotlin.library.impl
import java.io.ByteArrayOutputStream
import java.io.DataOutput
import java.io.DataOutputStream
import java.io.FileOutputStream
@@ -24,7 +25,25 @@ abstract class IrFileWriter {
}
}
class IrTableWriter(private val data: List<ByteArray>) : IrFileWriter() {
abstract class IrMemoryWriter {
protected abstract fun writeData(dataOutput: DataOutput)
fun writeIntoMemory(): ByteArray {
val memoryStream = ByteArrayOutputStream()
val dataOutputStream = DataOutputStream(memoryStream)
writeData(dataOutputStream)
dataOutputStream.close()
memoryStream.close()
return memoryStream.toByteArray()
}
}
class IrArrayWriter(private val data: List<ByteArray>) : IrFileWriter() {
override fun writeData(dataOutput: DataOutput) {
dataOutput.writeInt(data.size)
@@ -33,16 +52,52 @@ class IrTableWriter(private val data: List<ByteArray>) : IrFileWriter() {
}
}
class IrMemoryArrayWriter(private val data: List<ByteArray>) : IrMemoryWriter() {
override fun writeData(dataOutput: DataOutput) {
dataOutput.writeInt(data.size)
data.forEach { dataOutput.writeInt(it.size) }
data.forEach { dataOutput.write(it) }
}
}
class IrByteArrayWriter(private val data: List<ByteArray>) : IrFileWriter() {
override fun writeData(dataOutput: DataOutput) {
dataOutput.writeInt(data.size)
data.forEach { dataOutput.writeInt(it.size) }
data.forEach { dataOutput.write(it) }
}
}
class IrTableWriter(private val data: List<Pair<Long, ByteArray>>) : IrFileWriter() {
override fun writeData(dataOutput: DataOutput) {
dataOutput.writeInt(data.size)
var dataOffset = 4 + data.size * (8 + 4 + 4)
data.forEach {
dataOutput.writeLong(it.first)
dataOutput.writeInt(dataOffset)
dataOutput.writeInt(it.second.size)
dataOffset += it.second.size
}
data.forEach { dataOutput.write(it.second) }
}
}
sealed class SerializedDeclaration {
abstract val id: Long
abstract val local: Int
abstract val size: Int
abstract val bytes: ByteArray
var offset: Int = -1
abstract val declarationName: String
}
class TopLevelDeclaration(override val id: Long, isLocal: Boolean, override val bytes: ByteArray) : SerializedDeclaration() {
class TopLevelDeclaration(override val id: Long, isLocal: Boolean, override val declarationName: String, override val bytes: ByteArray) : SerializedDeclaration() {
override val local = if (isLocal) 1 else 0
override val size = bytes.size
}
@@ -52,6 +107,9 @@ object SkippedDeclaration : SerializedDeclaration() {
override val local = -1
override val size = 0
override val bytes = ByteArray(0)
override val declarationName: String = "<SKIPPED>"
}
class IrDeclarationWriter(private val declarations: List<SerializedDeclaration>) : IrFileWriter() {
@@ -65,15 +123,36 @@ class IrDeclarationWriter(private val declarations: List<SerializedDeclaration>)
var dataOffset = INDEX_HEADER_SIZE + SINGLE_INDEX_RECORD_SIZE * declarations.size
for (d in declarations) {
d.offset = dataOffset
dataOutput.writeLong(d.id)
dataOutput.writeInt(d.local)
dataOutput.writeInt(dataOffset)
dataOutput.writeInt(d.size)
dataOffset += d.size
}
for (d in declarations) {
dataOutput.write(d.bytes)
}
}
}
class IrMemoryDeclarationWriter(private val declarations: List<SerializedDeclaration>) : IrMemoryWriter() {
private val SINGLE_INDEX_RECORD_SIZE = 20 // sizeof(Long) + 3 * sizeof(Int).
private val INDEX_HEADER_SIZE = 4 // sizeof(Int).
override fun writeData(dataOutput: DataOutput) {
dataOutput.writeInt(declarations.size)
var dataOffset = INDEX_HEADER_SIZE + SINGLE_INDEX_RECORD_SIZE * declarations.size
for (d in declarations) {
dataOutput.writeLong(d.id)
dataOutput.writeInt(d.local)
dataOutput.writeInt(d.offset)
dataOutput.writeInt(dataOffset)
dataOutput.writeInt(d.size)
dataOffset += d.size
}
for (d in declarations) {
@@ -13,13 +13,50 @@ class IrWriterImpl(val irLayout: IrKotlinLibraryLayout) : IrWriter {
irLayout.irTablesDir.mkdirs()
}
override fun addIr(ir: SerializedIr) {
irLayout.irHeader.writeBytes(ir.module)
private fun serializeFile(file: IrIrSerializedIrFile) {
val fqnPath = file.fqName.joinToString(separator = "/")
val fileId = file.path.hashCode().toString(Character.MAX_RADIX)
IrDeclarationWriter(ir.serializedDeclarations).writeIntoFile(irLayout.irDeclarations.absolutePath)
IrTableWriter(ir.symbols).writeIntoFile(irLayout.irSymbols.absolutePath)
IrTableWriter(ir.types).writeIntoFile(irLayout.irTypes.absolutePath)
IrTableWriter(ir.strings).writeIntoFile(irLayout.irStrings.absolutePath)
val fileDirectoryName = "${fileId}.file"
val packageDir = irLayout.irDir.child(fqnPath)
if (!packageDir.exists) {
packageDir.mkdirs()
}
val fileDir = packageDir.child(fileDirectoryName)
assert(!fileDir.exists)
fileDir.mkdirs()
irLayout.irFile(fileDir).writeBytes(file.fileData)
IrDeclarationWriter(file.declarations).writeIntoFile(irLayout.irDeclarations(fileDir).absolutePath)
IrArrayWriter(file.symbols).writeIntoFile(irLayout.irSymbols(fileDir).absolutePath)
IrArrayWriter(file.types).writeIntoFile(irLayout.irTypes(fileDir).absolutePath)
IrArrayWriter(file.strings).writeIntoFile(irLayout.irStrings(fileDir).absolutePath)
IrArrayWriter(file.bodies).writeIntoFile(irLayout.irBodies(fileDir).absolutePath)
fileDir.child("declarations.txt").writeText(file.declarations.joinToString(separator = "\n") { "${it.declarationName} -> (${it.id}, ${it.local})" })
}
override fun addIr(ir: SerializedIrModule) {
// ir.files.forEach {
// serializeFile(it)
// }
with(ir.files.sortedBy { it.path }) {
IrArrayWriter(map { it.fileData }).writeIntoFile(irLayout.irFiles.absolutePath)
IrArrayWriter(map { IrMemoryDeclarationWriter(it.declarations).writeIntoMemory() }).writeIntoFile(irLayout.irDeclarations.absolutePath)
IrArrayWriter(map { IrMemoryArrayWriter(it.symbols).writeIntoMemory() }).writeIntoFile(irLayout.irSymbols.absolutePath)
IrArrayWriter(map { IrMemoryArrayWriter(it.types).writeIntoMemory() }).writeIntoFile(irLayout.irTypes.absolutePath)
IrArrayWriter(map { IrMemoryArrayWriter(it.strings).writeIntoMemory() }).writeIntoFile(irLayout.irStrings.absolutePath)
IrArrayWriter(map { IrMemoryArrayWriter(it.bodies).writeIntoMemory() }).writeIntoFile(irLayout.irBodies.absolutePath)
}
// IrDeclarationWriter(ir.serializedDeclarations).writeIntoFile(irLayout.irDeclarations.absolutePath)
// IrArrayWriter(ir.symbols).writeIntoFile(irLayout.irSymbols.absolutePath)
// IrArrayWriter(ir.types).writeIntoFile(irLayout.irTypes.absolutePath)
// IrArrayWriter(ir.strings).writeIntoFile(irLayout.irStrings.absolutePath)
}
override fun addDataFlowGraph(dataFlowGraph: ByteArray) {
@@ -83,7 +83,7 @@ open class IrLibraryImpl(
}
}
override fun irDeclaration(index: Long, isLocal: Boolean) = loadIrDeclaraton(index, isLocal)
override fun irDeclaration(index: Long, isLocal: Boolean) = error("Should not call this") //loadIrDeclaraton(index, isLocal)
override fun symbol(index: Int) = symbols.tableItemBytes(index)
@@ -91,37 +91,119 @@ open class IrLibraryImpl(
override fun string(index: Int) = strings.tableItemBytes(index)
private val combinedDeclarations: CombinedIrFileReader by lazy {
CombinedIrFileReader(access.realFiles {
override fun file(index: Int): ByteArray = files.tableItemBytes(index)
override fun fileCount(): Int = files.entryCount()
override fun irDeclaration(index: Long, isLocal: Boolean, fileIndex: Int) = loadIrDeclaraton(index, isLocal, fileIndex)
override fun symbol(index: Int, fileIndex: Int) = symbols.tableItemBytes(fileIndex, index)
override fun type(index: Int, fileIndex: Int) = types.tableItemBytes(fileIndex, index)
override fun string(index: Int, fileIndex: Int) = strings.tableItemBytes(fileIndex, index)
override fun body(index: Int, fileIndex: Int) = bodies.tableItemBytes(fileIndex, index)
private val combinedDeclarations: DeclarationIrMultiTableReader by lazy {
DeclarationIrMultiTableReader(access.realFiles {
it.irDeclarations
})
}
private val symbols: SimpleIrTableFileReader by lazy {
SimpleIrTableFileReader(access.realFiles {
private val symbols: IrMultiArrayReader by lazy {
IrMultiArrayReader(access.realFiles {
it.irSymbols
})
}
private val types: SimpleIrTableFileReader by lazy {
SimpleIrTableFileReader(access.realFiles {
private val types: IrMultiArrayReader by lazy {
IrMultiArrayReader(access.realFiles {
it.irTypes
})
}
private val strings: SimpleIrTableFileReader by lazy {
SimpleIrTableFileReader(access.realFiles {
private val strings: IrMultiArrayReader by lazy {
IrMultiArrayReader(access.realFiles {
it.irStrings
})
}
private val bodies: IrMultiArrayReader by lazy {
IrMultiArrayReader(access.realFiles {
it.irBodies
})
}
private val files: IrArrayReader by lazy {
IrArrayReader(access.realFiles {
it.irFiles
})
}
private fun getFileDirectory(fileName: String): File {
val directoryName = "${fileName.hashCode().toString(Character.MAX_RADIX)}.file"
return access.realFiles {
it.irFilesX.first { it.name == directoryName }
}
}
private val fileToDeclarationMap = mutableMapOf<String, DeclarationIrTableReader>()
private fun combinedDeclarations(fileName: String): DeclarationIrTableReader {
return fileToDeclarationMap.getOrPut(fileName) {
DeclarationIrTableReader(access.realFiles {
it.irDeclarations(getFileDirectory(fileName))
})
}
}
private val fileToSymbolMap = mutableMapOf<String, IrArrayReader>()
private fun symbols(fileName: String): IrArrayReader {
return fileToSymbolMap.getOrPut(fileName) {
IrArrayReader(access.realFiles {
it.irSymbols(getFileDirectory(fileName))
})
}
}
private val fileToTypeMap = mutableMapOf<String, IrArrayReader>()
private fun types(fileName: String): IrArrayReader {
return fileToTypeMap.getOrPut(fileName) {
IrArrayReader(access.realFiles {
it.irTypes(getFileDirectory(fileName))
})
}
}
// private val fileToStringMap = mutableMapOf<String, IrArrayReader>()
// private fun strings(fileIndex: Int): IrArrayReader {
// return fileToStringMap.getOrPut(fileName) {
// IrArrayReader(access.realFiles {
// it.irStrings(getFileDirectory(fileName))
// })
// }
// }
private val fileToBodyMap = mutableMapOf<String, IrArrayReader>()
private fun bodies(fileName: String): IrArrayReader {
return fileToBodyMap.getOrPut(fileName) {
IrArrayReader(access.realFiles {
it.irBodies(getFileDirectory(fileName))
})
}
}
private fun loadIrHeader(): ByteArray =
access.inPlace {
it.irHeader.readBytes()
}
private fun loadIrDeclaraton(index: Long, isLocal: Boolean) =
combinedDeclarations.declarationBytes(DeclarationId(index, isLocal))
// private fun loadIrDeclaraton(index: Long, isLocal: Boolean) =
// combinedDeclarations.tableItemBytes(DeclarationId(index, isLocal))
private fun loadIrDeclaraton(index: Long, isLocal: Boolean, fileIndex: Int) =
combinedDeclarations.tableItemBytes(fileIndex, DeclarationId(index, isLocal))
override val dataFlowGraph by lazy {
access.inPlace { it: IrKotlinLibraryLayout ->
@@ -88,7 +88,7 @@ class KoltinLibraryWriterImpl(
fun buildKoltinLibrary(
linkDependencies: List<KotlinLibrary>,
metadata: SerializedMetadata,
ir: SerializedIr,
ir: SerializedIrModule,
versions: KonanLibraryVersioning,
output: String,
moduleName: String,