From aa313903cc9bb3792971412d55dcb880ff52729e Mon Sep 17 00:00:00 2001 From: Roman Artemev Date: Tue, 13 Aug 2019 17:44:55 +0300 Subject: [PATCH] [IR SERIALIZATION] Support both Per-File and Monolitic klib layout --- .../jetbrains/kotlin/library/KotlinLibrary.kt | 5 - .../kotlin/library/KotlinLibraryLayout.kt | 15 --- .../kotlin/library/impl/IrWriterImpl.kt | 71 +++++----- .../kotlin/library/impl/KotlinLibraryImpl.kt | 127 +++++++++--------- .../library/impl/KotlinLibraryWriterImpl.kt | 3 +- 5 files changed, 97 insertions(+), 124 deletions(-) diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibrary.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibrary.kt index 4d0940ab301..eaa37cfe14e 100644 --- a/compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibrary.kt +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibrary.kt @@ -32,11 +32,6 @@ interface MetadataLibrary { interface IrLibrary { val dataFlowGraph: ByteArray? - val irHeader: ByteArray? - fun irDeclaration(index: Long, isLocal: Boolean): ByteArray - fun symbol(index: Int): ByteArray - fun type(index: Int): ByteArray - fun string(index: Int): ByteArray fun irDeclaration(index: Long, isLocal: Boolean, fileIndex: Int): ByteArray fun symbol(index: Int, fileIndex: Int): ByteArray fun type(index: Int, fileIndex: Int): ByteArray diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibraryLayout.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibraryLayout.kt index 372f77b6d64..5fe3b1f0988 100644 --- a/compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibraryLayout.kt +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibraryLayout.kt @@ -48,10 +48,6 @@ interface MetadataKotlinLibraryLayout : KotlinLibraryLayout { interface IrKotlinLibraryLayout : KotlinLibraryLayout { val irDir get() = File(libDir, "ir") - val irTablesDir - get() = File(irDir, "ir_tables") - val irHeader - get() = File(irDir, "irHeaders.kni") val irDeclarations get() = File(irDir, "irDeclarations.knd") val irSymbols @@ -73,15 +69,4 @@ interface IrKotlinLibraryLayout : KotlinLibraryLayout { fun irStrings(file: File): File = File(file, "strings.knt") fun irBodies(file: File): File = File(file, "body.knb") fun irFile(file: File): File = File(file, "file.knf") - - val irFilesX - get() = run { - val fileDirectories = mutableListOf() - irDir.postorder { - if (it.fileName.toString().endsWith(".file")) { - fileDirectories.add(File(it)) - } - } - fileDirectories - } } diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrWriterImpl.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrWriterImpl.kt index 27344e21a46..28ea5a34d48 100644 --- a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrWriterImpl.kt +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrWriterImpl.kt @@ -7,42 +7,19 @@ package org.jetbrains.kotlin.library.impl import org.jetbrains.kotlin.library.* -class IrWriterImpl(val irLayout: IrKotlinLibraryLayout) : IrWriter { +abstract class IrWriterImpl(val irLayout: IrKotlinLibraryLayout) : IrWriter { init { irLayout.irDir.mkdirs() - irLayout.irTablesDir.mkdirs() } - private fun serializeFile(file: SerializedIrFile) { - val fqnPath = file.fqName.joinToString(separator = "/") - val fileId = file.path.hashCode().toString(Character.MAX_RADIX) - - val fileDirectoryName = "${fileId}.file" - - val packageDir = irLayout.irDir.child(fqnPath) - if (!packageDir.exists) { - packageDir.mkdirs() - } - - val fileDir = packageDir.child(fileDirectoryName) - assert(!fileDir.exists) - fileDir.mkdirs() - - - irLayout.irFile(fileDir).writeBytes(file.fileData) - IrDeclarationWriter(file.declarations).writeIntoFile(irLayout.irDeclarations(fileDir).absolutePath) - IrArrayWriter(file.symbols).writeIntoFile(irLayout.irSymbols(fileDir).absolutePath) - IrArrayWriter(file.types).writeIntoFile(irLayout.irTypes(fileDir).absolutePath) - IrArrayWriter(file.strings).writeIntoFile(irLayout.irStrings(fileDir).absolutePath) - IrArrayWriter(file.bodies).writeIntoFile(irLayout.irBodies(fileDir).absolutePath) - fileDir.child("declarations.txt").writeText(file.declarations.joinToString(separator = "\n") { "${it.declarationName} -> (${it.id}, ${it.local})" }) + override fun addDataFlowGraph(dataFlowGraph: ByteArray) { + irLayout.dataFlowGraphFile.writeBytes(dataFlowGraph) } +} + +class IrMonoliticWriterImpl(irLayout: IrKotlinLibraryLayout) : IrWriterImpl(irLayout) { override fun addIr(ir: SerializedIrModule) { -// ir.files.forEach { -// serializeFile(it) -// } - with(ir.files.sortedBy { it.path }) { IrArrayWriter(map { it.fileData }).writeIntoFile(irLayout.irFiles.absolutePath) IrArrayWriter(map { IrMemoryDeclarationWriter(it.declarations).writeIntoMemory() }).writeIntoFile(irLayout.irDeclarations.absolutePath) @@ -51,15 +28,31 @@ class IrWriterImpl(val irLayout: IrKotlinLibraryLayout) : IrWriter { IrArrayWriter(map { IrMemoryArrayWriter(it.strings).writeIntoMemory() }).writeIntoFile(irLayout.irStrings.absolutePath) IrArrayWriter(map { IrMemoryArrayWriter(it.bodies).writeIntoMemory() }).writeIntoFile(irLayout.irBodies.absolutePath) } - - -// IrDeclarationWriter(ir.serializedDeclarations).writeIntoFile(irLayout.irDeclarations.absolutePath) -// IrArrayWriter(ir.symbols).writeIntoFile(irLayout.irSymbols.absolutePath) -// IrArrayWriter(ir.types).writeIntoFile(irLayout.irTypes.absolutePath) -// IrArrayWriter(ir.strings).writeIntoFile(irLayout.irStrings.absolutePath) - } - - override fun addDataFlowGraph(dataFlowGraph: ByteArray) { - irLayout.dataFlowGraphFile.writeBytes(dataFlowGraph) } } + +class IrPerFileWriterImpl(irLayout: IrKotlinLibraryLayout) : IrWriterImpl(irLayout) { + override fun addIr(ir: SerializedIrModule) { + ir.files.forEach { + serializeFile(it) + } + } + + private fun serializeFile(file: SerializedIrFile) { + val fqnPath = file.fqName.joinToString(separator = ".") + val fileId = file.path.hashCode().toString(Character.MAX_RADIX) + val irFileDirectory = "$fqnPath.$fileId.file" + val fileDir = irLayout.irDir.child(irFileDirectory) + + assert(!fileDir.exists) + fileDir.mkdirs() + + irLayout.irFile(fileDir).writeBytes(file.fileData) + IrDeclarationWriter(file.declarations).writeIntoFile(irLayout.irDeclarations(fileDir).absolutePath) + IrArrayWriter(file.symbols).writeIntoFile(irLayout.irSymbols(fileDir).absolutePath) + IrArrayWriter(file.types).writeIntoFile(irLayout.irTypes(fileDir).absolutePath) + IrArrayWriter(file.strings).writeIntoFile(irLayout.irStrings(fileDir).absolutePath) + IrArrayWriter(file.bodies).writeIntoFile(irLayout.irBodies(fileDir).absolutePath) +// fileDir.child("declarations.txt").writeText(file.declarations.joinToString(separator = "\n") { "${it.declarationName} -> (${it.id}, ${it.local})" }) + } +} \ No newline at end of file diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryImpl.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryImpl.kt index fd373faef7b..da9b902af33 100644 --- a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryImpl.kt +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryImpl.kt @@ -71,31 +71,20 @@ open class MetadataLibraryImpl( } } -open class IrLibraryImpl( - private val access: IrLibraryAccess +abstract class IrLibraryImpl( + protected val access: IrLibraryAccess ) : IrLibrary { - - override val irHeader: ByteArray? by lazy { - access.inPlace { library: IrKotlinLibraryLayout -> - library.irHeader.let { - if (it.exists) loadIrHeader() else null - } + override val dataFlowGraph by lazy { + access.inPlace { it: IrKotlinLibraryLayout -> + it.dataFlowGraphFile.let { if (it.exists) it.readBytes() else null } } } +} - override fun irDeclaration(index: Long, isLocal: Boolean) = error("Should not call this") //loadIrDeclaraton(index, isLocal) - - override fun symbol(index: Int) = symbols.tableItemBytes(index) - - override fun type(index: Int) = types.tableItemBytes(index) - - override fun string(index: Int) = strings.tableItemBytes(index) - - override fun file(index: Int): ByteArray = files.tableItemBytes(index) - +class IrMonoliticLibraryImpl(_access: IrLibraryAccess) : IrLibraryImpl(_access) { override fun fileCount(): Int = files.entryCount() - override fun irDeclaration(index: Long, isLocal: Boolean, fileIndex: Int) = loadIrDeclaraton(index, isLocal, fileIndex) + override fun irDeclaration(index: Long, isLocal: Boolean, fileIndex: Int) = loadIrDeclaration(index, isLocal, fileIndex) override fun symbol(index: Int, fileIndex: Int) = symbols.tableItemBytes(fileIndex, index) @@ -105,6 +94,11 @@ open class IrLibraryImpl( override fun body(index: Int, fileIndex: Int) = bodies.tableItemBytes(fileIndex, index) + override fun file(index: Int) = files.tableItemBytes(index) + + private fun loadIrDeclaration(index: Long, isLocal: Boolean, fileIndex: Int) = + combinedDeclarations.tableItemBytes(fileIndex, DeclarationId(index, isLocal)) + private val combinedDeclarations: DeclarationIrMultiTableReader by lazy { DeclarationIrMultiTableReader(access.realFiles { it.irDeclarations @@ -140,75 +134,79 @@ open class IrLibraryImpl( it.irFiles }) } +} +class IrPerFileLibraryImpl(_access: IrLibraryAccess) : IrLibraryImpl(_access) { - private fun getFileDirectory(fileName: String): File { - val directoryName = "${fileName.hashCode().toString(Character.MAX_RADIX)}.file" - return access.realFiles { - it.irFilesX.first { it.name == directoryName } + private val directories by lazy { + access.realFiles { + it.irDir.listFiles.filter { f -> f.isDirectory && f.name.endsWith(".file") } } } - private val fileToDeclarationMap = mutableMapOf() - private fun combinedDeclarations(fileName: String): DeclarationIrTableReader { - return fileToDeclarationMap.getOrPut(fileName) { + private val fileToDeclarationMap = mutableMapOf() + override fun irDeclaration(index: Long, isLocal: Boolean, fileIndex: Int): ByteArray { + val dataReader = fileToDeclarationMap.getOrPut(fileIndex) { + val fileDirectory = directories[fileIndex] DeclarationIrTableReader(access.realFiles { - it.irDeclarations(getFileDirectory(fileName)) + it.irDeclarations(fileDirectory) }) } + return dataReader.tableItemBytes(DeclarationId(index, isLocal)) } - private val fileToSymbolMap = mutableMapOf() - private fun symbols(fileName: String): IrArrayReader { - return fileToSymbolMap.getOrPut(fileName) { + private val fileToSymbolMap = mutableMapOf() + override fun symbol(index: Int, fileIndex: Int): ByteArray { + val dataReader = fileToSymbolMap.getOrPut(fileIndex) { + val fileDirectory = directories[fileIndex] IrArrayReader(access.realFiles { - it.irSymbols(getFileDirectory(fileName)) + it.irSymbols(fileDirectory) }) } + return dataReader.tableItemBytes(index) } - private val fileToTypeMap = mutableMapOf() - private fun types(fileName: String): IrArrayReader { - return fileToTypeMap.getOrPut(fileName) { + private val fileToTypeMap = mutableMapOf() + override fun type(index: Int, fileIndex: Int): ByteArray { + val dataReader = fileToTypeMap.getOrPut(fileIndex) { + val fileDirectory = directories[fileIndex] IrArrayReader(access.realFiles { - it.irTypes(getFileDirectory(fileName)) + it.irTypes(fileDirectory) }) } + return dataReader.tableItemBytes(index) } -// private val fileToStringMap = mutableMapOf() -// private fun strings(fileIndex: Int): IrArrayReader { -// return fileToStringMap.getOrPut(fileName) { -// IrArrayReader(access.realFiles { -// it.irStrings(getFileDirectory(fileName)) -// }) -// } -// } - - private val fileToBodyMap = mutableMapOf() - private fun bodies(fileName: String): IrArrayReader { - return fileToBodyMap.getOrPut(fileName) { + private val fileToStringMap = mutableMapOf() + override fun string(index: Int, fileIndex: Int): ByteArray { + val dataReader = fileToStringMap.getOrPut(fileIndex) { + val fileDirectory = directories[fileIndex] IrArrayReader(access.realFiles { - it.irBodies(getFileDirectory(fileName)) + it.irStrings(fileDirectory) }) } + return dataReader.tableItemBytes(index) + } + + private val fileToBodyMap = mutableMapOf() + override fun body(index: Int, fileIndex: Int): ByteArray { + val dataReader = fileToBodyMap.getOrPut(fileIndex) { + val fileDirectory = directories[fileIndex] + IrArrayReader(access.realFiles { + it.irBodies(fileDirectory) + }) + } + return dataReader.tableItemBytes(index) + } + + override fun file(index: Int): ByteArray { + return access.realFiles { + it.irFile(directories[index]).readBytes() + } } - private fun loadIrHeader(): ByteArray = - access.inPlace { - it.irHeader.readBytes() - } - -// private fun loadIrDeclaraton(index: Long, isLocal: Boolean) = -// combinedDeclarations.tableItemBytes(DeclarationId(index, isLocal)) - - private fun loadIrDeclaraton(index: Long, isLocal: Boolean, fileIndex: Int) = - combinedDeclarations.tableItemBytes(fileIndex, DeclarationId(index, isLocal)) - - override val dataFlowGraph by lazy { - access.inPlace { it: IrKotlinLibraryLayout -> - it.dataFlowGraphFile.let { if (it.exists) it.readBytes() else null } - } + override fun fileCount(): Int { + return directories.size } } @@ -231,7 +229,8 @@ fun createKotlinLibrary( val base = BaseKotlinLibraryImpl(baseAccess, isDefault) val metadata = MetadataLibraryImpl(metadataAccess) - val ir = IrLibraryImpl(irAccess) +// val ir = IrMonoliticLibraryImpl(irAccess) + val ir = IrPerFileLibraryImpl(irAccess) return KotlinLibraryImpl(base, metadata, ir) } \ No newline at end of file diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryWriterImpl.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryWriterImpl.kt index 5e796c74add..c070d645db6 100644 --- a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryWriterImpl.kt +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryWriterImpl.kt @@ -81,7 +81,8 @@ class KoltinLibraryWriterImpl( base: BaseWriter = BaseWriterImpl(layout, moduleName, versions, nopack), metadata: MetadataWriter = MetadataWriterImpl(layout), - ir: IrWriter = IrWriterImpl(layout) +// ir: IrWriter = IrMonoliticWriterImpl(layout) + ir: IrWriter = IrPerFileWriterImpl(layout) ) : BaseWriter by base, MetadataWriter by metadata, IrWriter by ir, KotlinLibraryWriter