Combine all IR files to a single mmaped file with an index. (#2620)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt

@@ -263,7 +263,6 @@ internal class Context(config: KonanConfig) : KonanBackendContext(config) {
     // But we have to wait until the code generation phase,
     // to dump this information into generated file.
     var serializedLinkData: LinkData? = null
-    var serializedIr: ByteArray? = null
     var dataFlowGraph: ByteArray? = null
 
     @Deprecated("")

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt

@@ -19,6 +19,7 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.ir.util.patchDeclarationParents
+import org.jetbrains.kotlin.konan.library.impl.CombinedIrFileWriter
 import org.jetbrains.kotlin.psi2ir.Psi2IrConfiguration
 import org.jetbrains.kotlin.psi2ir.Psi2IrTranslator
 
@@ -117,8 +118,8 @@ fun runTopLevelPhases(konanConfig: KonanConfig, environment: KotlinCoreEnvironme
 
     phaser.phase(KonanPhase.SERIALIZER) {
         val declarationTable = DeclarationTable(context.irModule!!.irBuiltins, DescriptorTable())
-        val serializedIr = IrModuleSerializer(context, declarationTable, bodiesOnlyForInlines = context.config.isInteropStubs).serializedIrModule(context.irModule!!)
-
+        val serializedIr = IrModuleSerializer(
+                context, declarationTable, bodiesOnlyForInlines = context.config.isInteropStubs).serializedIrModule(context.irModule!!)
         val serializer = KonanSerializationUtil(context, context.config.configuration.get(CommonConfigurationKeys.METADATA_VERSION)!!, declarationTable)
         context.serializedLinkData =
             serializer.serializeModule(context.moduleDescriptor, /*if (!context.config.isInteropStubs) serializedIr else null*/ serializedIr)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryWriter.kt

@@ -33,7 +33,7 @@ class LinkData(
 
 class SerializedIr (
     val module: ByteArray,
-    val declarations: Map<UniqId, ByteArray>,
+    val combinedDeclarationFilePath: String,
     val debugIndex: Map<UniqId, String>
 )
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/MetadataWriterImpl.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.backend.konan.library.impl
 
 import org.jetbrains.kotlin.backend.konan.library.LinkData
+import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
 
 internal class MetadataWriterImpl(libraryLayout: KonanLibraryLayout): KonanLibraryLayout by libraryLayout {
@@ -25,13 +26,9 @@ internal class MetadataWriterImpl(libraryLayout: KonanLibraryLayout): KonanLibra
                 packageFragmentFile(packageFqName, "${withLeadingZeros(i)}_$shortName").writeBytes(fragment)
             }
         }
-        linkData.ir?.declarations?.forEach {
-            val index = it.key.index.toULong().toString(16)
-            val file = if (it.key.isLocal)
-                hiddenDeclarationFile(index)
-            else
-                visibleDeclarationFile(index)
-            file.writeBytes(it.value)
+        linkData.ir?.combinedDeclarationFilePath?.let {
+            // TODO: use Files.move.
+            File(it).copyTo(irFile)
         }
         val lines = linkData.ir?.debugIndex?.map { entry -> "${entry.key}: ${entry.value}" }
         if (lines != null) irIndex.writeLines(lines)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/IrModuleSerializer.kt

@@ -39,6 +39,8 @@ import org.jetbrains.kotlin.ir.expressions.impl.IrUnaryPrimitiveImpl
 import org.jetbrains.kotlin.ir.symbols.*
 import org.jetbrains.kotlin.ir.types.*
 import org.jetbrains.kotlin.ir.types.impl.IrStarProjectionImpl
+import org.jetbrains.kotlin.konan.library.impl.CombinedIrFileWriter
+import org.jetbrains.kotlin.konan.library.impl.DeclarationId
 import org.jetbrains.kotlin.metadata.KonanIr
 import org.jetbrains.kotlin.types.SimpleType
 import org.jetbrains.kotlin.types.StarProjectionImpl
@@ -1050,30 +1052,35 @@ internal class IrModuleSerializer(
         .addAllLineStartOffsets(entry.lineStartOffsets.asIterable())
         .build()
 
-    val topLevelDeclarations = mutableMapOf<UniqId, ByteArray>()
-
     fun serializeIrFile(file: IrFile): KonanIr.IrFile {
         val proto = KonanIr.IrFile.newBuilder()
             .setFileEntry(serializeFileEntry(file.fileEntry))
             .setFqName(serializeString(file.fqName.toString()))
 
         file.declarations.forEach {
-            if (it is IrTypeAlias) return@forEach
-            if (it.descriptor.isExpectMember && !it.descriptor.isSerializableExpectClass) {
+            if (it is IrTypeAlias || (it.descriptor.isExpectMember && !it.descriptor.isSerializableExpectClass)) {
+                writer.skipDeclaration()
                 return@forEach
             }
 
             val byteArray = serializeDeclaration(it).toByteArray()
             val uniqId = declarationTable.uniqIdByDeclaration(it)
-            topLevelDeclarations.put(uniqId, byteArray)
+            writer.addDeclaration(DeclarationId(uniqId.index, uniqId.isLocal), byteArray)
             proto.addDeclarationId(protoUniqId(uniqId))
         }
         return proto.build()
     }
 
+    lateinit var writer: CombinedIrFileWriter
+
     fun serializeModule(module: IrModuleFragment): KonanIr.IrModule {
         val proto = KonanIr.IrModule.newBuilder()
             .setName(serializeString(module.name.toString()))
+
+        val topLevelDeclarationsCount = module.files.sumBy { it.declarations.size }
+
+        writer = CombinedIrFileWriter(topLevelDeclarationsCount)
+
         module.files.forEach {
             proto.addFile(serializeIrFile(it))
         }
@@ -1095,6 +1102,6 @@ internal class IrModuleSerializer(
 
     fun serializedIrModule(module: IrModuleFragment): SerializedIr {
         val moduleHeader = serializeModule(module).toByteArray()
-        return SerializedIr(moduleHeader, topLevelDeclarations, declarationTable.debugIndex)
+        return SerializedIr(moduleHeader, writer.finishWriting().absolutePath, declarationTable.debugIndex)
     }
 }

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanLibraryLayout.kt

@@ -1,3 +1,19 @@
+/**
+ * Copyright 2010-2019 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.jetbrains.kotlin.konan.library
 
 import org.jetbrains.kotlin.konan.file.File
@@ -46,10 +62,8 @@ interface KonanLibraryLayout {
             File(packageFragmentsDir(packageFqName), "$partName$KLIB_METADATA_FILE_EXTENSION_WITH_DOT")
     val irDir
         get() = File(libDir, "ir")
-    fun hiddenDeclarationFile(declarationId: String)
-        = File(irDir, "hidden_$declarationId.knd")
-    fun visibleDeclarationFile(declarationId: String)
-        = File(irDir, "visible_$declarationId.knd")
+    val irFile
+        get() = File(irDir, "irCombined.knd")
     val irHeader
         get() = File(irDir, "irHeaders.kni")
     val irIndex: File

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/KonanMetadataReader.kt

@@ -3,6 +3,4 @@ package org.jetbrains.kotlin.konan.library
 interface MetadataReader {
     fun loadSerializedModule(libraryLayout: KonanLibraryLayout): ByteArray
     fun loadSerializedPackageFragment(libraryLayout: KonanLibraryLayout, fqName: String, partName: String): ByteArray
-    fun loadIrHeader(libraryLayout: KonanLibraryLayout): ByteArray
-    fun loadIrDeclaraton(libraryLayout: KonanLibraryLayout, index: Long, isLocal: Boolean): ByteArray
 }

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/CombinedIrFileAccessor.kt

@@ -0,0 +1,94 @@
+/**
+ * Copyright 2010-2019 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.konan.library.impl
+
+import org.jetbrains.kotlin.konan.file.File
+import java.io.RandomAccessFile
+import java.nio.channels.FileChannel
+
+data class DeclarationId(val id: Long, val isLocal: Boolean)
+
+class CombinedIrFileReader(file: File) {
+    private val buffer = file.map(FileChannel.MapMode.READ_ONLY)
+    private val declarationToOffsetSize = mutableMapOf<DeclarationId, Pair<Int, Int>>()
+
+    init {
+        val declarationsCount = buffer.int
+        for (i in 0 until declarationsCount) {
+            val id = buffer.long
+            val isLocal = buffer.int != 0
+            val offset = buffer.int
+            val size = buffer.int
+            declarationToOffsetSize[DeclarationId(id, isLocal)] = offset to size
+        }
+    }
+
+    fun declarationBytes(id: DeclarationId): ByteArray {
+        val offsetSize = declarationToOffsetSize[id] ?: throw Error("No declaration with $id here")
+        val result = ByteArray(offsetSize.second)
+        buffer.position(offsetSize.first)
+        buffer.get(result, 0, offsetSize.second)
+        return result
+    }
+}
+
+private const val SINGLE_INDEX_RECORD_SIZE = 20  // sizeof(Long) + 3 * sizeof(Int).
+private const val INDEX_HEADER_SIZE = 4  // sizeof(Int).
+
+class CombinedIrFileWriter(val declarationCount: Int) {
+    private var currentDeclaration = 0
+    private var currentPosition = 0
+    private val file = org.jetbrains.kotlin.konan.file.createTempFile("ir").deleteOnExit()
+    private val randomAccessFile = RandomAccessFile(file.path, "rw")
+
+    init {
+        randomAccessFile.writeInt(declarationCount)
+        assert(randomAccessFile.filePointer.toInt() == INDEX_HEADER_SIZE)
+        for (i in 0 until declarationCount) {
+            randomAccessFile.writeLong(-1) // id
+            randomAccessFile.writeInt(-1)  // isLocal
+            randomAccessFile.writeInt(-1)  // offset
+            randomAccessFile.writeInt(-1)  // size
+        }
+        currentPosition = randomAccessFile.filePointer.toInt()
+        assert(currentPosition == INDEX_HEADER_SIZE + SINGLE_INDEX_RECORD_SIZE * declarationCount)
+    }
+
+    fun skipDeclaration() {
+        currentDeclaration++
+    }
+
+    fun addDeclaration(id: DeclarationId, bytes: ByteArray) {
+        randomAccessFile.seek((currentDeclaration * SINGLE_INDEX_RECORD_SIZE + INDEX_HEADER_SIZE).toLong())
+        randomAccessFile.writeLong(id.id)
+        randomAccessFile.writeInt(if (id.isLocal) 1 else 0)
+        randomAccessFile.writeInt(currentPosition)
+        randomAccessFile.writeInt(bytes.size)
+        randomAccessFile.seek(currentPosition.toLong())
+        randomAccessFile.write(bytes)
+        assert(randomAccessFile.filePointer < Int.MAX_VALUE.toLong())
+        currentPosition = randomAccessFile.filePointer.toInt()
+        currentDeclaration++
+    }
+
+    fun finishWriting(): File {
+        assert(currentDeclaration == declarationCount)
+        randomAccessFile.close()
+        return file
+    }
+}
+

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/DefaultMetadataReaderImpl.kt

@@ -11,15 +11,4 @@ internal object DefaultMetadataReaderImpl : MetadataReader {
     override fun loadSerializedPackageFragment(libraryLayout: KonanLibraryLayout, fqName: String, partName: String): ByteArray =
             libraryLayout.packageFragmentFile(fqName, partName).readBytes()
 
-    override fun loadIrHeader(libraryLayout: KonanLibraryLayout): ByteArray =
-            libraryLayout.irHeader.readBytes()
-
-    override fun loadIrDeclaraton(libraryLayout: KonanLibraryLayout, index: Long, isLocal: Boolean): ByteArray {
-        val name = index.toULong().toString(16)
-        val file = if (isLocal)
-            libraryLayout.hiddenDeclarationFile(name)
-        else
-            libraryLayout.visibleDeclarationFile(name)
-        return file.readBytes()
-    }
 }

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/KonanLibraryImpl.kt

@@ -84,9 +84,22 @@ class KonanLibraryImpl(
                 }
             }
 
-    override val irHeader: ByteArray? by lazy { layout.inPlace { library -> library.irHeader.let { if (it.exists) metadataReader.loadIrHeader(library) else null }}}
+    override val irHeader: ByteArray? by lazy { layout.inPlace { library -> library.irHeader.let {
+        if (it.exists) loadIrHeader() else null }
+    }}
 
-    override fun irDeclaration(index: Long, isLocal: Boolean) = layout.inPlace { metadataReader.loadIrDeclaraton(it, index, isLocal) }
+    override fun irDeclaration(index: Long, isLocal: Boolean) = loadIrDeclaraton(index, isLocal)
 
     override fun toString() = "$libraryName[default=$isDefault]"
+
+    private val combinedDeclarations: CombinedIrFileReader by lazy {
+        CombinedIrFileReader(layout.realFiles {
+            it.irFile
+        })
+    }
+    private fun loadIrHeader(): ByteArray =
+            layout.inPlace {  it.irHeader.readBytes() }
+
+    private fun loadIrDeclaraton(index: Long, isLocal: Boolean) =
+            combinedDeclarations.declarationBytes(DeclarationId(index, isLocal))
 }

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/KonanLibraryLayoutImpl.kt

@@ -19,7 +19,7 @@ private class ZippedKonanLibraryLayout(val klibFile: File, override val target:
 
     override val libraryName = klibFile.path.removeSuffixIfPresent(KLIB_FILE_EXTENSION_WITH_DOT)
 
-    override val libDir: File= File("/")
+    override val libDir = File("/")
 
     override fun <T> realFiles(action: (KonanLibraryLayout) -> T): T {
         return action(FileExtractor(this))!!
@@ -70,6 +70,8 @@ private class FileExtractor(val zippedLibraryLayout: ZippedKonanLibraryLayout):
 
     override val linkdataDir: File by lazy { extractDir(super.linkdataDir) }
 
+    override val irFile: File by lazy { extract(super.irFile) }
+
     fun extract(file: File): File = zippedLibraryLayout.klibFile.withZipFileSystem { zipFileSystem ->
         val temporary = createTempFile(file.name)
         zipFileSystem.file(file).copyTo(temporary)

shared/src/main/kotlin/org/jetbrains/kotlin/konan/KonanAbiVersion.kt

@@ -1,3 +1,19 @@
+/**
+ * Copyright 2010-2019 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.jetbrains.kotlin.konan
 
 fun String.parseKonanAbiVersion(): KonanAbiVersion {
@@ -6,7 +22,7 @@ fun String.parseKonanAbiVersion(): KonanAbiVersion {
 
 data class KonanAbiVersion(val version: Int) {
     companion object {
-        val CURRENT = KonanAbiVersion(6)
+        val CURRENT = KonanAbiVersion(7)
     }
     override fun toString() = "$version"
 }

shared/src/main/kotlin/org/jetbrains/kotlin/konan/file/File.kt

@@ -22,7 +22,10 @@ import org.jetbrains.kotlin.konan.util.removeSuffixIfPresent
 import java.io.BufferedReader
 import java.io.InputStream
 import java.io.InputStreamReader
+import java.io.RandomAccessFile
 import java.net.URI
+import java.nio.MappedByteBuffer
+import java.nio.channels.FileChannel
 import java.nio.file.*
 import java.nio.file.attribute.BasicFileAttributes
 
@@ -108,8 +111,17 @@ data class File constructor(internal val javaPath: Path) {
             return FileVisitResult.CONTINUE
         }
         })
-
     }
+
+    fun map(mode: FileChannel.MapMode = FileChannel.MapMode.READ_ONLY,
+            start: Long = 0, size: Long = -1): MappedByteBuffer {
+        val file = RandomAccessFile(path,
+                if (mode == FileChannel.MapMode.READ_ONLY) "r" else "rw")
+        val fileSize = if (mode == FileChannel.MapMode.READ_ONLY)
+            file.length() else size.also { assert(size != -1L) }
+        return file.channel.map(mode, start, fileSize) // Shall we .also { file.close() }?
+    }
+
     fun deleteOnExit(): File {
         // Works only on the default file system, 
         // but that's okay for now.