[IR] Do not interact with file system during Ir Serialization

- make `IrModuleSerializer` produce ByteArrays with serialized data
 - isolate file system access on Klib writer

compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrModuleSerializer.kt

@@ -25,9 +25,7 @@ import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
 import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
 import org.jetbrains.kotlin.ir.visitors.acceptVoid
 import org.jetbrains.kotlin.library.SerializedIr
-import org.jetbrains.kotlin.library.impl.CombinedIrFileWriter
+import org.jetbrains.kotlin.library.impl.*
-import org.jetbrains.kotlin.library.impl.DeclarationId
-import org.jetbrains.kotlin.library.impl.SimpleIrTableFileWriter
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.types.Variance
 import org.jetbrains.kotlin.backend.common.serialization.proto.Annotations as ProtoAnnotations
@@ -1247,7 +1245,7 @@ open class IrModuleSerializer(
     open fun backendSpecificExplicitRoot(declaration: IrFunction) = false
     open fun backendSpecificExplicitRoot(declaration: IrClass) = false
 
-    fun serializeIrFile(file: IrFile): ProtoFile {
+    fun serializeIrFile(file: IrFile, topLevelDeclarations: MutableList<SerializedDeclaration>): ProtoFile {
         val proto = ProtoFile.newBuilder()
             .setFileEntry(serializeFileEntry(file.fileEntry))
             .setFqName(serializeString(file.fqName.toString()))
@@ -1255,13 +1253,13 @@ open class IrModuleSerializer(
 
         file.declarations.forEach {
             if (it.descriptor.isExpectMember && !it.descriptor.isSerializableExpectClass) {
-                writer.skipDeclaration()
+                topLevelDeclarations.add(SkippedDeclaration)
                 return@forEach
             }
 
             val byteArray = serializeDeclaration(it).toByteArray()
             val uniqId = declarationTable.uniqIdByDeclaration(it)
-            writer.addDeclaration(DeclarationId(uniqId.index, uniqId.isLocal), byteArray)
+            topLevelDeclarations.add(TopLevelDeclaration(uniqId.index, uniqId.isLocal, byteArray))
             proto.addDeclarationId(protoUniqId(uniqId))
         }
 
@@ -1298,46 +1296,24 @@ open class IrModuleSerializer(
         return proto.build()
     }
 
-    lateinit var writer: CombinedIrFileWriter
+    fun serializedIrModule(module: IrModuleFragment): SerializedIr {
-    lateinit var symbolTableWriter: SimpleIrTableFileWriter
-    lateinit var typeTableWriter: SimpleIrTableFileWriter
-    lateinit var stringTableWriter: SimpleIrTableFileWriter
-
-    fun serializeModule(module: IrModuleFragment): ProtoModule {
         val proto = ProtoModule.newBuilder()
             .setName(serializeName(module.name))
 
-        val files = module.files.filter { it.packageFragmentDescriptor !is FunctionInterfacePackageFragment }
+        val topLevelDeclarations = mutableListOf<SerializedDeclaration>()
-        val topLevelDeclarationsCount = files.sumBy { it.declarations.size }
 
-        writer = CombinedIrFileWriter(topLevelDeclarationsCount)
+        module.files.filter { it.packageFragmentDescriptor !is FunctionInterfacePackageFragment }.forEach {
-
+            proto.addFile(serializeIrFile(it, topLevelDeclarations))
-        files.forEach {
-            proto.addFile(serializeIrFile(it))
         }
 
-        symbolTableWriter = SimpleIrTableFileWriter("symbols", protoSymbolArray.size)
+        assert(module.files.sumBy { it.declarations.size } == topLevelDeclarations.size)
-        for (protoSymbol in protoSymbolArray)
-            symbolTableWriter.addItem(protoSymbol.toByteArray())
 
-        typeTableWriter = SimpleIrTableFileWriter("types", protoTypeArray.size)
+        val symbols = protoSymbolArray.map { it.toByteArray() }
-        for (protoSymbol in protoTypeArray)
+        val types = protoTypeArray.map { it.toByteArray() }
-            typeTableWriter.addItem(protoSymbol.toByteArray())
+        val strings = protoStringArray.map { it.toByteArray() }
 
-        stringTableWriter = SimpleIrTableFileWriter("strings", protoStringArray.size)
+        val moduleBytes = proto.build().toByteArray()
-        for (protoSymbol in protoStringArray)
-            stringTableWriter.addItem(protoSymbol.toByteArray())
 
-        return proto.build()
+        return SerializedIr(moduleBytes, symbols, types, strings, topLevelDeclarations)
-    }
-
-    fun serializedIrModule(module: IrModuleFragment): SerializedIr {
-        val moduleHeader = serializeModule(module).toByteArray()
-        return SerializedIr(
-            moduleHeader, symbolTableWriter.finishWriting().absolutePath,
-            typeTableWriter.finishWriting().absolutePath,
-            stringTableWriter.finishWriting().absolutePath,
-            writer.finishWriting().absolutePath
-        )
     }
 }

compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinLibraryWriter.kt

@@ -5,8 +5,8 @@
 
 package org.jetbrains.kotlin.library
 
-import org.jetbrains.kotlin.library.*
 import org.jetbrains.kotlin.konan.properties.Properties
+import org.jetbrains.kotlin.library.impl.SerializedDeclaration
 
 interface BaseWriter {
     val versions: KonanLibraryVersioning
@@ -33,10 +33,10 @@ class SerializedMetadata(
     val fragmentNames: List<String>
 )
 
-class SerializedIr (
+class SerializedIr(
     val module: ByteArray,
-    val symbolTableFilePath: String,
+    val symbols: List<ByteArray>,
-    val typeTableFilePath: String,
+    val types: List<ByteArray>,
-    val stringTableFilePath: String,
+    val strings: List<ByteArray>,
-    val combinedDeclarationFilePath: String
+    val serializedDeclarations: List<SerializedDeclaration>
 )

compiler/util-klib/src/org/jetbrains/kotlin/library/impl/CombinedIrFileAccessor.kt

@@ -1,94 +0,0 @@
-/**
- * Copyright 2010-2019 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.jetbrains.kotlin.library.impl
-
-import org.jetbrains.kotlin.konan.file.File
-import java.io.RandomAccessFile
-import java.nio.channels.FileChannel
-
-data class DeclarationId(val id: Long, val isLocal: Boolean)
-
-class CombinedIrFileReader(file: File) {
-    private val buffer = file.map(FileChannel.MapMode.READ_ONLY)
-    private val declarationToOffsetSize = mutableMapOf<DeclarationId, Pair<Int, Int>>()
-
-    init {
-        val declarationsCount = buffer.int
-        for (i in 0 until declarationsCount) {
-            val id = buffer.long
-            val isLocal = buffer.int != 0
-            val offset = buffer.int
-            val size = buffer.int
-            declarationToOffsetSize[DeclarationId(id, isLocal)] = offset to size
-        }
-    }
-
-    fun declarationBytes(id: DeclarationId): ByteArray {
-        val offsetSize = declarationToOffsetSize[id] ?: throw Error("No declaration with $id here")
-        val result = ByteArray(offsetSize.second)
-        buffer.position(offsetSize.first)
-        buffer.get(result, 0, offsetSize.second)
-        return result
-    }
-}
-
-private const val SINGLE_INDEX_RECORD_SIZE = 20  // sizeof(Long) + 3 * sizeof(Int).
-private const val INDEX_HEADER_SIZE = 4  // sizeof(Int).
-
-class CombinedIrFileWriter(val declarationCount: Int) {
-    private var currentDeclaration = 0
-    private var currentPosition = 0
-    private val file = org.jetbrains.kotlin.konan.file.createTempFile("ir").deleteOnExit()
-    private val randomAccessFile = RandomAccessFile(file.path, "rw")
-
-    init {
-        randomAccessFile.writeInt(declarationCount)
-        assert(randomAccessFile.filePointer.toInt() == INDEX_HEADER_SIZE)
-        for (i in 0 until declarationCount) {
-            randomAccessFile.writeLong(-1) // id
-            randomAccessFile.writeInt(-1)  // isLocal
-            randomAccessFile.writeInt(-1)  // offset
-            randomAccessFile.writeInt(-1)  // size
-        }
-        currentPosition = randomAccessFile.filePointer.toInt()
-        assert(currentPosition == INDEX_HEADER_SIZE + SINGLE_INDEX_RECORD_SIZE * declarationCount)
-    }
-
-    fun skipDeclaration() {
-        currentDeclaration++
-    }
-
-    fun addDeclaration(id: DeclarationId, bytes: ByteArray) {
-        randomAccessFile.seek((currentDeclaration * SINGLE_INDEX_RECORD_SIZE + INDEX_HEADER_SIZE).toLong())
-        randomAccessFile.writeLong(id.id)
-        randomAccessFile.writeInt(if (id.isLocal) 1 else 0)
-        randomAccessFile.writeInt(currentPosition)
-        randomAccessFile.writeInt(bytes.size)
-        randomAccessFile.seek(currentPosition.toLong())
-        randomAccessFile.write(bytes)
-        assert(randomAccessFile.filePointer < Int.MAX_VALUE.toLong())
-        currentPosition = randomAccessFile.filePointer.toInt()
-        currentDeclaration++
-    }
-
-    fun finishWriting(): File {
-        assert(currentDeclaration == declarationCount)
-        randomAccessFile.close()
-        return file
-    }
-}
-

compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrFileReaders.kt

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.library.impl
+
+import org.jetbrains.kotlin.konan.file.File
+import java.nio.channels.FileChannel
+
+class SimpleIrTableFileReader(file: File) {
+    private val buffer = file.map(FileChannel.MapMode.READ_ONLY)
+    private val indexToOffset: IntArray
+
+    init {
+        val count = buffer.int
+        indexToOffset = IntArray(count + 1)
+        indexToOffset[0] = 4 * (count + 1)
+        for (i in 0 until count) {
+            val size = buffer.int
+            indexToOffset[i + 1] = indexToOffset[i] + size
+        }
+    }
+
+    fun tableItemBytes(id: Int): ByteArray {
+        val offset = indexToOffset[id]
+        val size = indexToOffset[id + 1] - offset
+        val result = ByteArray(size)
+        buffer.position(offset)
+        buffer.get(result, 0, size)
+        return result
+    }
+}
+
+
+data class DeclarationId(val id: Long, val isLocal: Boolean)
+
+class CombinedIrFileReader(file: File) {
+    private val buffer = file.map(FileChannel.MapMode.READ_ONLY)
+    private val declarationToOffsetSize = mutableMapOf<DeclarationId, Pair<Int, Int>>()
+
+    init {
+        val declarationsCount = buffer.int
+        for (i in 0 until declarationsCount) {
+            val id = buffer.long
+            val isLocal = buffer.int != 0
+            val offset = buffer.int
+            val size = buffer.int
+            declarationToOffsetSize[DeclarationId(id, isLocal)] = offset to size
+        }
+    }
+
+    fun declarationBytes(id: DeclarationId): ByteArray {
+        val offsetSize = declarationToOffsetSize[id] ?: throw Error("No declaration with $id here")
+        val result = ByteArray(offsetSize.second)
+        buffer.position(offsetSize.first)
+        buffer.get(result, 0, offsetSize.second)
+        return result
+    }
+}

compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrFileWriters.kt

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.library.impl
+
+import java.io.DataOutput
+import java.io.DataOutputStream
+import java.io.FileOutputStream
+
+abstract class IrFileWriter {
+
+    protected abstract fun writeData(dataOutput: DataOutput)
+
+    fun writeIntoFile(path: String) {
+        val fileStream = FileOutputStream(path)
+        val dataOutputStream = DataOutputStream(fileStream)
+
+        writeData(dataOutputStream)
+
+        dataOutputStream.close()
+        fileStream.close()
+    }
+}
+
+class IrTableWriter(private val data: List<ByteArray>) : IrFileWriter() {
+    override fun writeData(dataOutput: DataOutput) {
+        dataOutput.writeInt(data.size)
+
+        data.forEach { dataOutput.writeInt(it.size) }
+        data.forEach { dataOutput.write(it) }
+    }
+}
+
+sealed class SerializedDeclaration {
+    abstract val id: Long
+    abstract val local: Int
+    abstract val size: Int
+    abstract val bytes: ByteArray
+
+    var offset: Int = -1
+}
+
+class TopLevelDeclaration(override val id: Long, isLocal: Boolean, override val bytes: ByteArray) : SerializedDeclaration() {
+    override val local = if (isLocal) 1 else 0
+    override val size = bytes.size
+}
+
+object SkippedDeclaration : SerializedDeclaration() {
+    override val id = -1L
+    override val local = -1
+    override val size = 0
+    override val bytes = ByteArray(0)
+}
+
+class IrDeclarationWriter(private val declarations: List<SerializedDeclaration>) : IrFileWriter() {
+
+    private val SINGLE_INDEX_RECORD_SIZE = 20  // sizeof(Long) + 3 * sizeof(Int).
+    private val INDEX_HEADER_SIZE = 4  // sizeof(Int).
+
+    override fun writeData(dataOutput: DataOutput) {
+        dataOutput.writeInt(declarations.size)
+
+        var dataOffset = INDEX_HEADER_SIZE + SINGLE_INDEX_RECORD_SIZE * declarations.size
+
+        for (d in declarations) {
+            d.offset = dataOffset
+            dataOffset += d.size
+        }
+
+        for (d in declarations) {
+            dataOutput.writeLong(d.id)
+            dataOutput.writeInt(d.local)
+            dataOutput.writeInt(d.offset)
+            dataOutput.writeInt(d.size)
+        }
+
+        for (d in declarations) {
+            dataOutput.write(d.bytes)
+        }
+    }
+
+}

compiler/util-klib/src/org/jetbrains/kotlin/library/impl/IrWriterImpl.kt

@@ -5,7 +5,6 @@
 
 package org.jetbrains.kotlin.library.impl
 
-import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.library.*
 
 class IrWriterImpl(val irLayout: IrKotlinLibraryLayout) : IrWriter {
@@ -16,11 +15,11 @@ class IrWriterImpl(val irLayout: IrKotlinLibraryLayout) : IrWriter {
 
     override fun addIr(ir: SerializedIr) {
         irLayout.irHeader.writeBytes(ir.module)
-        // TODO: use Files.move.
+
-        File(ir.combinedDeclarationFilePath).copyTo(irLayout.irDeclarations)
+        IrDeclarationWriter(ir.serializedDeclarations).writeIntoFile(irLayout.irDeclarations.absolutePath)
-        File(ir.symbolTableFilePath).copyTo(irLayout.irSymbols)
+        IrTableWriter(ir.symbols).writeIntoFile(irLayout.irSymbols.absolutePath)
-        File(ir.typeTableFilePath).copyTo(irLayout.irTypes)
+        IrTableWriter(ir.types).writeIntoFile(irLayout.irTypes.absolutePath)
-        File(ir.stringTableFilePath).copyTo(irLayout.irStrings)
+        IrTableWriter(ir.strings).writeIntoFile(irLayout.irStrings.absolutePath)
     }
 
     override fun addDataFlowGraph(dataFlowGraph: ByteArray) {

compiler/util-klib/src/org/jetbrains/kotlin/library/impl/SimpleIrTableAccessor.kt

@@ -1,68 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.library.impl
-
-import org.jetbrains.kotlin.konan.file.File
-import java.io.RandomAccessFile
-import java.nio.channels.FileChannel
-import java.nio.file.Files
-
-class SimpleIrTableFileReader(file: File) {
-    private val buffer = file.map(FileChannel.MapMode.READ_ONLY)
-    private val indexToOffset: IntArray
-
-    init {
-        val count = buffer.int
-        indexToOffset = IntArray(count + 1)
-        indexToOffset[0] = 4 * (count + 1)
-        for (i in 0 until count) {
-            val size = buffer.int
-            indexToOffset[i + 1] = indexToOffset[i] + size
-        }
-    }
-
-    fun tableItemBytes(id: Int): ByteArray {
-        val offset = indexToOffset[id]
-        val size = indexToOffset[id + 1] - offset
-        val result = ByteArray(size)
-        buffer.position(offset)
-        buffer.get(result, 0, size)
-        return result
-    }
-}
-
-class SimpleIrTableFileWriter(val tableName: String, val itemsCount: Int) {
-    private var currentItemIndex = 0
-    private var currentPosition = 0
-    private val file = Files.createTempFile(tableName, "").toFile()
-    private val randomAccessFile = RandomAccessFile(file.path, "rw")
-
-    init {
-        randomAccessFile.writeInt(itemsCount)
-        assert(randomAccessFile.filePointer.toInt() == 4)
-        for (i in 0 until itemsCount) {
-            randomAccessFile.writeInt(-1)  // size
-        }
-        currentPosition = randomAccessFile.filePointer.toInt()
-        assert(currentPosition == 4 + 4 * itemsCount)
-    }
-
-    fun addItem(bytes: ByteArray) {
-        randomAccessFile.seek((currentItemIndex * 4 + 4).toLong())
-        randomAccessFile.writeInt(bytes.size)
-        randomAccessFile.seek(currentPosition.toLong())
-        randomAccessFile.write(bytes)
-        assert(randomAccessFile.filePointer < Int.MAX_VALUE.toLong())
-        currentPosition = randomAccessFile.filePointer.toInt()
-        currentItemIndex++
-    }
-
-    fun finishWriting(): java.io.File {
-        assert(currentItemIndex == itemsCount) { "Expected: $itemsCount, but was: $currentItemIndex" }
-        randomAccessFile.close()
-        return file
-    }
-}