[IR] Do not interact with file system during Ir Serialization
- make `IrModuleSerializer` produce ByteArrays with serialized data - isolate file system access on Klib writer
This commit is contained in:
Roman Artemev
+14
-38
@@ -25,9 +25,7 @@ import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
|
||||
import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
|
||||
import org.jetbrains.kotlin.ir.visitors.acceptVoid
|
||||
import org.jetbrains.kotlin.library.SerializedIr
|
||||
import org.jetbrains.kotlin.library.impl.CombinedIrFileWriter
|
||||
import org.jetbrains.kotlin.library.impl.DeclarationId
|
||||
import org.jetbrains.kotlin.library.impl.SimpleIrTableFileWriter
|
||||
import org.jetbrains.kotlin.library.impl.*
|
||||
import org.jetbrains.kotlin.name.Name
|
||||
import org.jetbrains.kotlin.types.Variance
|
||||
import org.jetbrains.kotlin.backend.common.serialization.proto.Annotations as ProtoAnnotations
|
||||
@@ -1247,7 +1245,7 @@ open class IrModuleSerializer(
|
||||
open fun backendSpecificExplicitRoot(declaration: IrFunction) = false
|
||||
open fun backendSpecificExplicitRoot(declaration: IrClass) = false
|
||||
|
||||
fun serializeIrFile(file: IrFile): ProtoFile {
|
||||
fun serializeIrFile(file: IrFile, topLevelDeclarations: MutableList<SerializedDeclaration>): ProtoFile {
|
||||
val proto = ProtoFile.newBuilder()
|
||||
.setFileEntry(serializeFileEntry(file.fileEntry))
|
||||
.setFqName(serializeString(file.fqName.toString()))
|
||||
@@ -1255,13 +1253,13 @@ open class IrModuleSerializer(
|
||||
|
||||
file.declarations.forEach {
|
||||
if (it.descriptor.isExpectMember && !it.descriptor.isSerializableExpectClass) {
|
||||
writer.skipDeclaration()
|
||||
topLevelDeclarations.add(SkippedDeclaration)
|
||||
return@forEach
|
||||
}
|
||||
|
||||
val byteArray = serializeDeclaration(it).toByteArray()
|
||||
val uniqId = declarationTable.uniqIdByDeclaration(it)
|
||||
writer.addDeclaration(DeclarationId(uniqId.index, uniqId.isLocal), byteArray)
|
||||
topLevelDeclarations.add(TopLevelDeclaration(uniqId.index, uniqId.isLocal, byteArray))
|
||||
proto.addDeclarationId(protoUniqId(uniqId))
|
||||
}
|
||||
|
||||
@@ -1298,46 +1296,24 @@ open class IrModuleSerializer(
|
||||
return proto.build()
|
||||
}
|
||||
|
||||
lateinit var writer: CombinedIrFileWriter
|
||||
lateinit var symbolTableWriter: SimpleIrTableFileWriter
|
||||
lateinit var typeTableWriter: SimpleIrTableFileWriter
|
||||
lateinit var stringTableWriter: SimpleIrTableFileWriter
|
||||
|
||||
fun serializeModule(module: IrModuleFragment): ProtoModule {
|
||||
fun serializedIrModule(module: IrModuleFragment): SerializedIr {
|
||||
val proto = ProtoModule.newBuilder()
|
||||
.setName(serializeName(module.name))
|
||||
|
||||
val files = module.files.filter { it.packageFragmentDescriptor !is FunctionInterfacePackageFragment }
|
||||
val topLevelDeclarationsCount = files.sumBy { it.declarations.size }
|
||||
val topLevelDeclarations = mutableListOf<SerializedDeclaration>()
|
||||
|
||||
writer = CombinedIrFileWriter(topLevelDeclarationsCount)
|
||||
|
||||
files.forEach {
|
||||
proto.addFile(serializeIrFile(it))
|
||||
module.files.filter { it.packageFragmentDescriptor !is FunctionInterfacePackageFragment }.forEach {
|
||||
proto.addFile(serializeIrFile(it, topLevelDeclarations))
|
||||
}
|
||||
|
||||
symbolTableWriter = SimpleIrTableFileWriter("symbols", protoSymbolArray.size)
|
||||
for (protoSymbol in protoSymbolArray)
|
||||
symbolTableWriter.addItem(protoSymbol.toByteArray())
|
||||
assert(module.files.sumBy { it.declarations.size } == topLevelDeclarations.size)
|
||||
|
||||
typeTableWriter = SimpleIrTableFileWriter("types", protoTypeArray.size)
|
||||
for (protoSymbol in protoTypeArray)
|
||||
typeTableWriter.addItem(protoSymbol.toByteArray())
|
||||
val symbols = protoSymbolArray.map { it.toByteArray() }
|
||||
val types = protoTypeArray.map { it.toByteArray() }
|
||||
val strings = protoStringArray.map { it.toByteArray() }
|
||||
|
||||
stringTableWriter = SimpleIrTableFileWriter("strings", protoStringArray.size)
|
||||
for (protoSymbol in protoStringArray)
|
||||
stringTableWriter.addItem(protoSymbol.toByteArray())
|
||||
val moduleBytes = proto.build().toByteArray()
|
||||
|
||||
return proto.build()
|
||||
}
|
||||
|
||||
fun serializedIrModule(module: IrModuleFragment): SerializedIr {
|
||||
val moduleHeader = serializeModule(module).toByteArray()
|
||||
return SerializedIr(
|
||||
moduleHeader, symbolTableWriter.finishWriting().absolutePath,
|
||||
typeTableWriter.finishWriting().absolutePath,
|
||||
stringTableWriter.finishWriting().absolutePath,
|
||||
writer.finishWriting().absolutePath
|
||||
)
|
||||
return SerializedIr(moduleBytes, symbols, types, strings, topLevelDeclarations)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
|
||||
package org.jetbrains.kotlin.library
|
||||
|
||||
import org.jetbrains.kotlin.library.*
|
||||
import org.jetbrains.kotlin.konan.properties.Properties
|
||||
import org.jetbrains.kotlin.library.impl.SerializedDeclaration
|
||||
|
||||
interface BaseWriter {
|
||||
val versions: KonanLibraryVersioning
|
||||
@@ -33,10 +33,10 @@ class SerializedMetadata(
|
||||
val fragmentNames: List<String>
|
||||
)
|
||||
|
||||
class SerializedIr (
|
||||
class SerializedIr(
|
||||
val module: ByteArray,
|
||||
val symbolTableFilePath: String,
|
||||
val typeTableFilePath: String,
|
||||
val stringTableFilePath: String,
|
||||
val combinedDeclarationFilePath: String
|
||||
val symbols: List<ByteArray>,
|
||||
val types: List<ByteArray>,
|
||||
val strings: List<ByteArray>,
|
||||
val serializedDeclarations: List<SerializedDeclaration>
|
||||
)
|
||||
@@ -1,94 +0,0 @@
|
||||
/**
|
||||
* Copyright 2010-2019 JetBrains s.r.o.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.library.impl
|
||||
|
||||
import org.jetbrains.kotlin.konan.file.File
|
||||
import java.io.RandomAccessFile
|
||||
import java.nio.channels.FileChannel
|
||||
|
||||
data class DeclarationId(val id: Long, val isLocal: Boolean)
|
||||
|
||||
class CombinedIrFileReader(file: File) {
|
||||
private val buffer = file.map(FileChannel.MapMode.READ_ONLY)
|
||||
private val declarationToOffsetSize = mutableMapOf<DeclarationId, Pair<Int, Int>>()
|
||||
|
||||
init {
|
||||
val declarationsCount = buffer.int
|
||||
for (i in 0 until declarationsCount) {
|
||||
val id = buffer.long
|
||||
val isLocal = buffer.int != 0
|
||||
val offset = buffer.int
|
||||
val size = buffer.int
|
||||
declarationToOffsetSize[DeclarationId(id, isLocal)] = offset to size
|
||||
}
|
||||
}
|
||||
|
||||
fun declarationBytes(id: DeclarationId): ByteArray {
|
||||
val offsetSize = declarationToOffsetSize[id] ?: throw Error("No declaration with $id here")
|
||||
val result = ByteArray(offsetSize.second)
|
||||
buffer.position(offsetSize.first)
|
||||
buffer.get(result, 0, offsetSize.second)
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
private const val SINGLE_INDEX_RECORD_SIZE = 20 // sizeof(Long) + 3 * sizeof(Int).
|
||||
private const val INDEX_HEADER_SIZE = 4 // sizeof(Int).
|
||||
|
||||
class CombinedIrFileWriter(val declarationCount: Int) {
|
||||
private var currentDeclaration = 0
|
||||
private var currentPosition = 0
|
||||
private val file = org.jetbrains.kotlin.konan.file.createTempFile("ir").deleteOnExit()
|
||||
private val randomAccessFile = RandomAccessFile(file.path, "rw")
|
||||
|
||||
init {
|
||||
randomAccessFile.writeInt(declarationCount)
|
||||
assert(randomAccessFile.filePointer.toInt() == INDEX_HEADER_SIZE)
|
||||
for (i in 0 until declarationCount) {
|
||||
randomAccessFile.writeLong(-1) // id
|
||||
randomAccessFile.writeInt(-1) // isLocal
|
||||
randomAccessFile.writeInt(-1) // offset
|
||||
randomAccessFile.writeInt(-1) // size
|
||||
}
|
||||
currentPosition = randomAccessFile.filePointer.toInt()
|
||||
assert(currentPosition == INDEX_HEADER_SIZE + SINGLE_INDEX_RECORD_SIZE * declarationCount)
|
||||
}
|
||||
|
||||
fun skipDeclaration() {
|
||||
currentDeclaration++
|
||||
}
|
||||
|
||||
fun addDeclaration(id: DeclarationId, bytes: ByteArray) {
|
||||
randomAccessFile.seek((currentDeclaration * SINGLE_INDEX_RECORD_SIZE + INDEX_HEADER_SIZE).toLong())
|
||||
randomAccessFile.writeLong(id.id)
|
||||
randomAccessFile.writeInt(if (id.isLocal) 1 else 0)
|
||||
randomAccessFile.writeInt(currentPosition)
|
||||
randomAccessFile.writeInt(bytes.size)
|
||||
randomAccessFile.seek(currentPosition.toLong())
|
||||
randomAccessFile.write(bytes)
|
||||
assert(randomAccessFile.filePointer < Int.MAX_VALUE.toLong())
|
||||
currentPosition = randomAccessFile.filePointer.toInt()
|
||||
currentDeclaration++
|
||||
}
|
||||
|
||||
fun finishWriting(): File {
|
||||
assert(currentDeclaration == declarationCount)
|
||||
randomAccessFile.close()
|
||||
return file
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.library.impl
|
||||
|
||||
import org.jetbrains.kotlin.konan.file.File
|
||||
import java.nio.channels.FileChannel
|
||||
|
||||
class SimpleIrTableFileReader(file: File) {
|
||||
private val buffer = file.map(FileChannel.MapMode.READ_ONLY)
|
||||
private val indexToOffset: IntArray
|
||||
|
||||
init {
|
||||
val count = buffer.int
|
||||
indexToOffset = IntArray(count + 1)
|
||||
indexToOffset[0] = 4 * (count + 1)
|
||||
for (i in 0 until count) {
|
||||
val size = buffer.int
|
||||
indexToOffset[i + 1] = indexToOffset[i] + size
|
||||
}
|
||||
}
|
||||
|
||||
fun tableItemBytes(id: Int): ByteArray {
|
||||
val offset = indexToOffset[id]
|
||||
val size = indexToOffset[id + 1] - offset
|
||||
val result = ByteArray(size)
|
||||
buffer.position(offset)
|
||||
buffer.get(result, 0, size)
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
data class DeclarationId(val id: Long, val isLocal: Boolean)
|
||||
|
||||
class CombinedIrFileReader(file: File) {
|
||||
private val buffer = file.map(FileChannel.MapMode.READ_ONLY)
|
||||
private val declarationToOffsetSize = mutableMapOf<DeclarationId, Pair<Int, Int>>()
|
||||
|
||||
init {
|
||||
val declarationsCount = buffer.int
|
||||
for (i in 0 until declarationsCount) {
|
||||
val id = buffer.long
|
||||
val isLocal = buffer.int != 0
|
||||
val offset = buffer.int
|
||||
val size = buffer.int
|
||||
declarationToOffsetSize[DeclarationId(id, isLocal)] = offset to size
|
||||
}
|
||||
}
|
||||
|
||||
fun declarationBytes(id: DeclarationId): ByteArray {
|
||||
val offsetSize = declarationToOffsetSize[id] ?: throw Error("No declaration with $id here")
|
||||
val result = ByteArray(offsetSize.second)
|
||||
buffer.position(offsetSize.first)
|
||||
buffer.get(result, 0, offsetSize.second)
|
||||
return result
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
/*
|
||||
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.library.impl
|
||||
|
||||
import java.io.DataOutput
|
||||
import java.io.DataOutputStream
|
||||
import java.io.FileOutputStream
|
||||
|
||||
abstract class IrFileWriter {
|
||||
|
||||
protected abstract fun writeData(dataOutput: DataOutput)
|
||||
|
||||
fun writeIntoFile(path: String) {
|
||||
val fileStream = FileOutputStream(path)
|
||||
val dataOutputStream = DataOutputStream(fileStream)
|
||||
|
||||
writeData(dataOutputStream)
|
||||
|
||||
dataOutputStream.close()
|
||||
fileStream.close()
|
||||
}
|
||||
}
|
||||
|
||||
class IrTableWriter(private val data: List<ByteArray>) : IrFileWriter() {
|
||||
override fun writeData(dataOutput: DataOutput) {
|
||||
dataOutput.writeInt(data.size)
|
||||
|
||||
data.forEach { dataOutput.writeInt(it.size) }
|
||||
data.forEach { dataOutput.write(it) }
|
||||
}
|
||||
}
|
||||
|
||||
sealed class SerializedDeclaration {
|
||||
abstract val id: Long
|
||||
abstract val local: Int
|
||||
abstract val size: Int
|
||||
abstract val bytes: ByteArray
|
||||
|
||||
var offset: Int = -1
|
||||
}
|
||||
|
||||
class TopLevelDeclaration(override val id: Long, isLocal: Boolean, override val bytes: ByteArray) : SerializedDeclaration() {
|
||||
override val local = if (isLocal) 1 else 0
|
||||
override val size = bytes.size
|
||||
}
|
||||
|
||||
object SkippedDeclaration : SerializedDeclaration() {
|
||||
override val id = -1L
|
||||
override val local = -1
|
||||
override val size = 0
|
||||
override val bytes = ByteArray(0)
|
||||
}
|
||||
|
||||
class IrDeclarationWriter(private val declarations: List<SerializedDeclaration>) : IrFileWriter() {
|
||||
|
||||
private val SINGLE_INDEX_RECORD_SIZE = 20 // sizeof(Long) + 3 * sizeof(Int).
|
||||
private val INDEX_HEADER_SIZE = 4 // sizeof(Int).
|
||||
|
||||
override fun writeData(dataOutput: DataOutput) {
|
||||
dataOutput.writeInt(declarations.size)
|
||||
|
||||
var dataOffset = INDEX_HEADER_SIZE + SINGLE_INDEX_RECORD_SIZE * declarations.size
|
||||
|
||||
for (d in declarations) {
|
||||
d.offset = dataOffset
|
||||
dataOffset += d.size
|
||||
}
|
||||
|
||||
for (d in declarations) {
|
||||
dataOutput.writeLong(d.id)
|
||||
dataOutput.writeInt(d.local)
|
||||
dataOutput.writeInt(d.offset)
|
||||
dataOutput.writeInt(d.size)
|
||||
}
|
||||
|
||||
for (d in declarations) {
|
||||
dataOutput.write(d.bytes)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -5,7 +5,6 @@
|
||||
|
||||
package org.jetbrains.kotlin.library.impl
|
||||
|
||||
import org.jetbrains.kotlin.konan.file.File
|
||||
import org.jetbrains.kotlin.library.*
|
||||
|
||||
class IrWriterImpl(val irLayout: IrKotlinLibraryLayout) : IrWriter {
|
||||
@@ -16,11 +15,11 @@ class IrWriterImpl(val irLayout: IrKotlinLibraryLayout) : IrWriter {
|
||||
|
||||
override fun addIr(ir: SerializedIr) {
|
||||
irLayout.irHeader.writeBytes(ir.module)
|
||||
// TODO: use Files.move.
|
||||
File(ir.combinedDeclarationFilePath).copyTo(irLayout.irDeclarations)
|
||||
File(ir.symbolTableFilePath).copyTo(irLayout.irSymbols)
|
||||
File(ir.typeTableFilePath).copyTo(irLayout.irTypes)
|
||||
File(ir.stringTableFilePath).copyTo(irLayout.irStrings)
|
||||
|
||||
IrDeclarationWriter(ir.serializedDeclarations).writeIntoFile(irLayout.irDeclarations.absolutePath)
|
||||
IrTableWriter(ir.symbols).writeIntoFile(irLayout.irSymbols.absolutePath)
|
||||
IrTableWriter(ir.types).writeIntoFile(irLayout.irTypes.absolutePath)
|
||||
IrTableWriter(ir.strings).writeIntoFile(irLayout.irStrings.absolutePath)
|
||||
}
|
||||
|
||||
override fun addDataFlowGraph(dataFlowGraph: ByteArray) {
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
/*
|
||||
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.library.impl
|
||||
|
||||
import org.jetbrains.kotlin.konan.file.File
|
||||
import java.io.RandomAccessFile
|
||||
import java.nio.channels.FileChannel
|
||||
import java.nio.file.Files
|
||||
|
||||
class SimpleIrTableFileReader(file: File) {
|
||||
private val buffer = file.map(FileChannel.MapMode.READ_ONLY)
|
||||
private val indexToOffset: IntArray
|
||||
|
||||
init {
|
||||
val count = buffer.int
|
||||
indexToOffset = IntArray(count + 1)
|
||||
indexToOffset[0] = 4 * (count + 1)
|
||||
for (i in 0 until count) {
|
||||
val size = buffer.int
|
||||
indexToOffset[i + 1] = indexToOffset[i] + size
|
||||
}
|
||||
}
|
||||
|
||||
fun tableItemBytes(id: Int): ByteArray {
|
||||
val offset = indexToOffset[id]
|
||||
val size = indexToOffset[id + 1] - offset
|
||||
val result = ByteArray(size)
|
||||
buffer.position(offset)
|
||||
buffer.get(result, 0, size)
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
class SimpleIrTableFileWriter(val tableName: String, val itemsCount: Int) {
|
||||
private var currentItemIndex = 0
|
||||
private var currentPosition = 0
|
||||
private val file = Files.createTempFile(tableName, "").toFile()
|
||||
private val randomAccessFile = RandomAccessFile(file.path, "rw")
|
||||
|
||||
init {
|
||||
randomAccessFile.writeInt(itemsCount)
|
||||
assert(randomAccessFile.filePointer.toInt() == 4)
|
||||
for (i in 0 until itemsCount) {
|
||||
randomAccessFile.writeInt(-1) // size
|
||||
}
|
||||
currentPosition = randomAccessFile.filePointer.toInt()
|
||||
assert(currentPosition == 4 + 4 * itemsCount)
|
||||
}
|
||||
|
||||
fun addItem(bytes: ByteArray) {
|
||||
randomAccessFile.seek((currentItemIndex * 4 + 4).toLong())
|
||||
randomAccessFile.writeInt(bytes.size)
|
||||
randomAccessFile.seek(currentPosition.toLong())
|
||||
randomAccessFile.write(bytes)
|
||||
assert(randomAccessFile.filePointer < Int.MAX_VALUE.toLong())
|
||||
currentPosition = randomAccessFile.filePointer.toInt()
|
||||
currentItemIndex++
|
||||
}
|
||||
|
||||
fun finishWriting(): java.io.File {
|
||||
assert(currentItemIndex == itemsCount) { "Expected: $itemsCount, but was: $currentItemIndex" }
|
||||
randomAccessFile.close()
|
||||
return file
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user