[JS KLIB IC] Implement incremental cache for IR

build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt

@@ -18,6 +18,7 @@ package org.jetbrains.kotlin.incremental
 
 import com.intellij.util.io.DataExternalizer
 import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl
+import org.jetbrains.kotlin.incremental.js.IrTranslationResultValue
 import org.jetbrains.kotlin.incremental.js.TranslationResultValue
 import org.jetbrains.kotlin.incremental.storage.*
 import org.jetbrains.kotlin.metadata.ProtoBuf
@@ -40,6 +41,7 @@ open class IncrementalJsCache(
 ) : AbstractIncrementalCache<FqName>(cachesDir, pathConverter) {
     companion object {
         private const val TRANSLATION_RESULT_MAP = "translation-result"
+        private const val IR_TRANSLATION_RESULT_MAP = "ir-translation-result"
         private const val INLINE_FUNCTIONS = "inline-functions"
         private const val HEADER_FILE_NAME = "header.meta"
         private const val PACKAGE_META_FILE = "packages-meta"
@@ -50,6 +52,7 @@ open class IncrementalJsCache(
     override val sourceToClassesMap = registerMap(SourceToFqNameMap(SOURCE_TO_CLASSES.storageFile, pathConverter))
     override val dirtyOutputClassesMap = registerMap(DirtyClassesFqNameMap(DIRTY_OUTPUT_CLASSES.storageFile))
     private val translationResults = registerMap(TranslationResultMap(TRANSLATION_RESULT_MAP.storageFile, pathConverter))
+    private val irTranslationResults = registerMap(IrTranslationResultMap(IR_TRANSLATION_RESULT_MAP.storageFile, pathConverter))
     private val inlineFunctions = registerMap(InlineFunctionsMap(INLINE_FUNCTIONS.storageFile, pathConverter))
     private val packageMetadata = registerMap(PackageMetadataMap(PACKAGE_META_FILE.storageFile))
 
@@ -108,6 +111,11 @@ open class IncrementalJsCache(
         for ((packageName, metadata) in incrementalResults.packageMetadata) {
             packageMetadata.put(packageName, metadata)
         }
+
+        for ((srcFile, irData) in incrementalResults.irFileData) {
+            val (fileData, symbols, types, strings, declarations, bodies, fqn) = irData
+            irTranslationResults.put(srcFile, fileData, symbols, types, strings, declarations, bodies, fqn)
+        }
     }
 
     private fun registerOutputForFile(srcFile: File, name: FqName) {
@@ -118,6 +126,7 @@ open class IncrementalJsCache(
     override fun clearCacheForRemovedClasses(changesCollector: ChangesCollector) {
         dirtySources.forEach {
             translationResults.remove(it, changesCollector)
+            irTranslationResults.remove(it)
             inlineFunctions.remove(it)
         }
         removeAllFromClassStorage(dirtyOutputClassesMap.getDirtyOutputClasses(), changesCollector)
@@ -140,6 +149,16 @@ open class IncrementalJsCache(
             put(fqNameString, packageMetadata[fqNameString]!!)
         }
     }
+
+    fun nonDirtyIrParts(): Map<File, IrTranslationResultValue> =
+        hashMapOf<File, IrTranslationResultValue>().apply {
+            for (file in irTranslationResults.keys()) {
+
+                if (file !in dirtySources) {
+                    put(file, irTranslationResults[file]!!)
+                }
+            }
+        }
 }
 
 private object TranslationResultValueExternalizer : DataExternalizer<TranslationResultValue> {
@@ -202,6 +221,67 @@ private class TranslationResultMap(
     }
 }
 
+private object IrTranslationResultValueExternalizer : DataExternalizer<IrTranslationResultValue> {
+    override fun save(output: DataOutput, value: IrTranslationResultValue) {
+        output.writeArray(value.fileData)
+        output.writeArray(value.symbols)
+        output.writeArray(value.types)
+        output.writeArray(value.strings)
+        output.writeArray(value.declarations)
+        output.writeArray(value.bodies)
+        output.writeArray(value.fqn)
+    }
+
+    private fun DataOutput.writeArray(array: ByteArray) {
+        writeInt(array.size)
+        write(array)
+    }
+
+    private fun DataInput.readArray(): ByteArray {
+        val dataSize = readInt()
+        val filedata = ByteArray(dataSize)
+        readFully(filedata)
+        return filedata
+    }
+
+    override fun read(input: DataInput): IrTranslationResultValue {
+        val fileData = input.readArray()
+        val symbols = input.readArray()
+        val types = input.readArray()
+        val strings = input.readArray()
+        val declarations = input.readArray()
+        val bodies = input.readArray()
+        val fqn = input.readArray()
+
+        return IrTranslationResultValue(fileData, symbols, types, strings, declarations, bodies, fqn)
+    }
+}
+
+private class IrTranslationResultMap(
+    storageFile: File,
+    private val pathConverter: FileToPathConverter
+) :
+    BasicStringMap<IrTranslationResultValue>(storageFile, IrTranslationResultValueExternalizer) {
+    override fun dumpValue(value: IrTranslationResultValue): String =
+        "Filedata: ${value.fileData.md5()}, Symbols: ${value.symbols.md5()}, Types: ${value.types.md5()}, Strings: ${value.strings.md5()}, Declarations: ${value.declarations.md5()}, Bodies: ${value.bodies.md5()}"
+
+    fun put(sourceFile: File, newFiledata: ByteArray, newSymbols: ByteArray, newTypes: ByteArray, newStrings: ByteArray, newDeclarations: ByteArray, newBodies: ByteArray, fqn: ByteArray) {
+        storage[pathConverter.toPath(sourceFile)] =
+            IrTranslationResultValue(newFiledata, newSymbols, newTypes, newStrings, newDeclarations, newBodies, fqn)
+    }
+
+    operator fun get(sourceFile: File): IrTranslationResultValue? =
+        storage[pathConverter.toPath(sourceFile)]
+
+    fun keys(): Collection<File> =
+        storage.keys.map { pathConverter.toFile(it) }
+
+    fun remove(sourceFile: File) {
+        val path = pathConverter.toPath(sourceFile)
+        storage.remove(path)
+    }
+}
+
 fun getProtoData(sourceFile: File, metadata: ByteArray): Map<ClassId, ProtoData> {
     val classes = hashMapOf<ClassId, ProtoData>()
     val proto = ProtoBuf.PackageFragment.parseFrom(metadata, JsSerializerProtocol.extensionRegistry)

build-common/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProviderFromCache.kt

@@ -21,4 +21,7 @@ class IncrementalDataProviderFromCache(private val cache: IncrementalJsCache) :
 
     override val packageMetadata: Map<String, ByteArray>
         get() = cache.packageMetadata()
+
+    override val serializedIrFiles: Map<File, IrTranslationResultValue>
+        get() = cache.nonDirtyIrParts()
 }

compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalDataProvider.kt

@@ -9,11 +9,15 @@ import org.jetbrains.kotlin.daemon.common.CompilerCallbackServicesFacade
 import org.jetbrains.kotlin.daemon.common.Profiler
 import org.jetbrains.kotlin.daemon.common.withMeasure
 import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider
+import org.jetbrains.kotlin.incremental.js.IrTranslationResultValue
 import org.jetbrains.kotlin.incremental.js.TranslationResultValue
 import java.io.File
 
 class RemoteIncrementalDataProvider(val facade: CompilerCallbackServicesFacade, val rpcProfiler: Profiler) :
     IncrementalDataProvider {
+    override val serializedIrFiles: Map<File, IrTranslationResultValue>
+        get() = TODO("not implemented") //To change initializer of created properties use File | Settings | File Templates.
+
     override val headerMetadata: ByteArray
         get() = rpcProfiler.withMeasure(this) {
             facade.incrementalDataProvider_getHeaderMetadata()

compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalResultsConsumer.kt

@@ -15,6 +15,18 @@ import java.io.File
 
 class RemoteIncrementalResultsConsumer(val facade: CompilerCallbackServicesFacade, eventManager: EventManager, val rpcProfiler: Profiler) :
     IncrementalResultsConsumer {
+    override fun processIrFile(
+        sourceFile: File,
+        fileData: ByteArray,
+        symbols: ByteArray,
+        types: ByteArray,
+        strings: ByteArray,
+        declarations: ByteArray,
+        bodies: ByteArray,
+        fqn: ByteArray
+    ) {
+        TODO("not implemented") //To change body of created functions use File | Settings | File Templates.
+    }
 
     init {
         eventManager.

jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt

@@ -123,7 +123,7 @@ abstract class AbstractJsLookupTrackerTest : AbstractLookupTrackerTest() {
         if (header != null) {
             register(
                 IncrementalDataProvider::class.java,
-                IncrementalDataProviderImpl(header!!, packageParts, JsMetadataVersion.INSTANCE.toArray(), emptyMap()) // TODO pass correct metadata
+                IncrementalDataProviderImpl(header!!, packageParts, JsMetadataVersion.INSTANCE.toArray(), emptyMap(), emptyMap()) // TODO pass correct metadata
             )
         }
 

jps-plugin/src/org/jetbrains/kotlin/jps/incremental/local.kt

@@ -7,7 +7,7 @@ package org.jetbrains.kotlin.jps.incremental
 
 import java.io.File
 
-private val NORMAL_VERSION = 12
+private val NORMAL_VERSION = 13
 private val NORMAL_VERSION_FILE_NAME = "format-version.txt"
 
 fun localCacheVersionManager(dataRoot: File, isCachesEnabled: Boolean) =

js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProvider.kt

@@ -31,11 +31,14 @@ interface IncrementalDataProvider {
 
     /** gets non-dirty package metadata from previous compilation */
     val packageMetadata: Map<String, ByteArray>
+
+    val serializedIrFiles: Map<File, IrTranslationResultValue>
 }
 
 class IncrementalDataProviderImpl(
     override val headerMetadata: ByteArray,
     override val compiledPackageParts: Map<File, TranslationResultValue>,
     override val metadataVersion: IntArray,
-    override val packageMetadata: Map<String, ByteArray>
+    override val packageMetadata: Map<String, ByteArray>,
+    override val serializedIrFiles: Map<File, IrTranslationResultValue>
 ) : IncrementalDataProvider

js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalResultsConsumer.kt

@@ -16,7 +16,6 @@
 
 package org.jetbrains.kotlin.incremental.js
 
-import org.jetbrains.kotlin.name.FqName
 import java.io.File
 import java.io.Serializable
 import java.security.MessageDigest
@@ -42,6 +41,17 @@ interface IncrementalResultsConsumer {
     fun processInlineFunctions(functions: Collection<JsInlineFunctionHash>)
 
     fun processPackageMetadata(packageName: String, metadata: ByteArray)
+
+    fun processIrFile(
+        sourceFile: File,
+        fileData: ByteArray,
+        symbols: ByteArray,
+        types: ByteArray,
+        strings: ByteArray,
+        declarations: ByteArray,
+        bodies: ByteArray,
+        fqn: ByteArray
+    )
 }
 
 class JsInlineFunctionHash(val sourceFilePath: String, val fqName: String, val inlineFunctionMd5Hash: Long): Serializable
@@ -108,6 +118,24 @@ class IncrementalResultsConsumerImpl : IncrementalResultsConsumer {
     override fun processPackageMetadata(packageName: String, metadata: ByteArray) {
         _packageMetadata[packageName] = metadata
     }
+
+//    class IrFileData(fileData: ByteArray, symbols: ByteArray, types: ByteArray, strings: ByteArray, bodies: ByteArray, declarations: ByteArray)
+    private val _irFileData = hashMapOf<File, IrTranslationResultValue>()
+    val irFileData: Map<File, IrTranslationResultValue>
+        get() = _irFileData
+
+    override fun processIrFile(
+        sourceFile: File,
+        fileData: ByteArray,
+        symbols: ByteArray,
+        types: ByteArray,
+        strings: ByteArray,
+        declarations: ByteArray,
+        bodies: ByteArray,
+        fqn: ByteArray
+    ) {
+        _irFileData[sourceFile] = IrTranslationResultValue(fileData, symbols, types, strings, declarations, bodies, fqn)
+    }
 }
 
 private fun ByteArray.md5(): Long {

js/js.frontend/src/org/jetbrains/kotlin/incremental/js/TranslationResultValue.kt

@@ -18,3 +18,13 @@ package org.jetbrains.kotlin.incremental.js
 
 data class TranslationResultValue(val metadata: ByteArray, val binaryAst: ByteArray, val inlineData: ByteArray)
 
+
+data class IrTranslationResultValue(
+    val fileData: ByteArray,
+    val symbols: ByteArray,
+    val types: ByteArray,
+    val strings: ByteArray,
+    val declarations: ByteArray,
+    val bodies: ByteArray,
+    val fqn: ByteArray
+)

js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt

@@ -654,7 +654,7 @@ abstract class BasicBoxTest(
             if (header != null) {
                 configuration.put(
                     JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER,
-                    IncrementalDataProviderImpl(header, incrementalData.translatedFiles, JsMetadataVersion.INSTANCE.toArray(), incrementalData.packageMetadata)
+                    IncrementalDataProviderImpl(header, incrementalData.translatedFiles, JsMetadataVersion.INSTANCE.toArray(), incrementalData.packageMetadata, emptyMap())
                 )
             }