[JS IR] Move the JS AST serialization right before the IC data committing

It allows not to keep the binary representation in memory

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/compilerWithIC.kt

@@ -53,7 +53,7 @@ class JsIrCompilerWithIC(
         allModules: Collection<IrModuleFragment>,
         dirtyFiles: Collection<IrFile>,
         mainArguments: List<String>?
-    ): List<JsIrFragmentAndBinaryAst> {
+    ): List<Pair<IrFile, JsIrProgramFragment>> {
         val shouldGeneratePolyfills = context.configuration.getBoolean(JSConfigurationKeys.GENERATE_POLYFILLS)
 
         allModules.forEach {

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt

@@ -9,7 +9,7 @@ import org.jetbrains.kotlin.descriptors.DescriptorVisibilities
 import org.jetbrains.kotlin.ir.backend.js.*
 import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity
 import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
-import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrFragmentAndBinaryAst
+import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrProgramFragment
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.ir.symbols.IrSymbol
 import org.jetbrains.kotlin.ir.util.*
@@ -27,7 +27,7 @@ fun interface JsIrCompilerICInterface {
         allModules: Collection<IrModuleFragment>,
         dirtyFiles: Collection<IrFile>,
         mainArguments: List<String>?
-    ): List<JsIrFragmentAndBinaryAst>
+    ): List<Pair<IrFile, JsIrProgramFragment>>
 }
 
 fun interface JsIrCompilerICInterfaceFactory {
@@ -567,13 +567,13 @@ class CacheUpdater(
     private fun buildModuleArtifactsAndCommitCache(
         jsIrLinker: JsIrLinker,
         loadedFragments: Map<KotlinLibraryFile, IrModuleFragment>,
-        rebuiltFileFragments: List<JsIrFragmentAndBinaryAst>
+        rebuiltFileFragments: List<Pair<IrFile, JsIrProgramFragment>>
     ): List<ModuleArtifact> {
         stopwatch.startNext("Cache committing")
         val fragmentToLibName = loadedFragments.entries.associate { it.value to it.key }
 
         val rebuiltSrcFiles = rebuiltFileFragments.groupBy {
-            fragmentToLibName[it.irFile.module] ?: notFoundIcError("loaded fragment lib name", srcFile = KotlinSourceFile(it.irFile))
+            fragmentToLibName[it.first.module] ?: notFoundIcError("loaded fragment lib name", srcFile = KotlinSourceFile(it.first))
         }
 
         val visited = hashSetOf<KotlinLibrary>()
@@ -585,7 +585,7 @@ class CacheUpdater(
                 val libFile = KotlinLibraryFile(lib)
                 val incrementalCache = getLibIncrementalCache(libFile)
                 val libFragment = loadedFragments[libFile] ?: notFoundIcError("loaded fragment", libFile)
-                val libRebuiltFiles = rebuiltSrcFiles[libFile]?.associateBy { KotlinSourceFile(it.irFile) } ?: emptyMap()
+                val libRebuiltFiles = rebuiltSrcFiles[libFile]?.associate { KotlinSourceFile(it.first) to it.second } ?: emptyMap()
                 val moduleDeserializer = jsIrLinker.moduleDeserializer(libFragment.descriptor)
 
                 val signatureToIndexMapping = moduleDeserializer.fileDeserializers().associate {
@@ -675,7 +675,7 @@ class CacheUpdater(
         compilerForIC: JsIrCompilerICInterface,
         loadedFragments: Map<KotlinLibraryFile, IrModuleFragment>,
         dirtyFiles: Map<KotlinLibraryFile, Set<KotlinSourceFile>>
-    ): List<JsIrFragmentAndBinaryAst> {
+    ): List<Pair<IrFile, JsIrProgramFragment>> {
         stopwatch.startNext("Processing IR - lowering")
         val result = compilerForIC.compile(
             allModules = loadedFragments.values,
@@ -779,7 +779,7 @@ fun rebuildCacheForDirtyFiles(
     irFactory: IrFactory,
     exportedDeclarations: Set<FqName>,
     mainArguments: List<String>?,
-): Pair<IrModuleFragment, List<JsIrFragmentAndBinaryAst>> {
+): Pair<IrModuleFragment, List<Pair<IrFile, JsIrProgramFragment>>> {
     val emptyMetadata = object : KotlinSourceFileExports() {
         override val inverseDependencies = KotlinSourceFileMap<Set<IdSignature>>(emptyMap())
     }

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt

@@ -6,12 +6,14 @@
 package org.jetbrains.kotlin.ir.backend.js.ic
 
 import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer
-import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrFragmentAndBinaryAst
+import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrProgramFragment
+import org.jetbrains.kotlin.ir.backend.js.utils.serialization.JsIrAstSerializer
 import org.jetbrains.kotlin.ir.util.IdSignature
 import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.library.impl.javaFile
 import org.jetbrains.kotlin.protobuf.CodedInputStream
 import org.jetbrains.kotlin.protobuf.CodedOutputStream
+import java.io.BufferedOutputStream
 import java.io.File
 
 
@@ -85,18 +87,23 @@ class IncrementalCache(private val library: KotlinLibrary, cachePath: String) {
     fun buildModuleArtifactAndCommitCache(
         moduleName: String,
         externalModuleName: String?,
-        rebuiltFileFragments: Map<KotlinSourceFile, JsIrFragmentAndBinaryAst>,
+        rebuiltFileFragments: Map<KotlinSourceFile, JsIrProgramFragment>,
         signatureToIndexMapping: Map<KotlinSourceFile, Map<IdSignature, Int>>
     ): ModuleArtifact {
+        val serializer = JsIrAstSerializer()
+
         val fileArtifacts = kotlinLibraryHeader.sourceFiles.map { srcFile ->
             val binaryAstFile = srcFile.getCacheFile(BINARY_AST_SUFFIX)
             val rebuiltFileFragment = rebuiltFileFragments[srcFile]
             if (rebuiltFileFragment != null) {
-                binaryAstFile.apply { recreate() }.writeBytes(rebuiltFileFragment.binaryAst)
+                binaryAstFile.recreate()
+                BufferedOutputStream(binaryAstFile.outputStream()).use { bufferedOutStream ->
+                    serializer.serialize(rebuiltFileFragment, bufferedOutStream)
+                }
             }
 
             commitSourceFileMetadata(srcFile, signatureToIndexMapping[srcFile] ?: emptyMap())
-            SrcFileArtifact(srcFile.path, rebuiltFileFragment?.fragment, binaryAstFile)
+            SrcFileArtifact(srcFile.path, rebuiltFileFragment, binaryAstFile)
         }
 
         return ModuleArtifact(moduleName, fileArtifacts, cacheDir, forceRebuildJs, externalModuleName)

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/transformers/irToJs/IrModuleToJsTransformer.kt

@@ -11,7 +11,6 @@ import org.jetbrains.kotlin.ir.backend.js.dce.eliminateDeadDeclarations
 import org.jetbrains.kotlin.ir.backend.js.export.*
 import org.jetbrains.kotlin.ir.backend.js.lower.StaticMembersLowering
 import org.jetbrains.kotlin.ir.backend.js.utils.*
-import org.jetbrains.kotlin.ir.backend.js.utils.serialization.JsIrAstSerializer
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.ir.util.isInterface
 import org.jetbrains.kotlin.ir.util.render
@@ -72,8 +71,6 @@ enum class TranslationMode(
     }
 }
 
-class JsIrFragmentAndBinaryAst(val irFile: IrFile, val fragment: JsIrProgramFragment, val binaryAst: ByteArray)
-
 class JsCodeGenerator(
     private val program: JsIrProgram,
     private val multiModule: Boolean,
@@ -177,20 +174,15 @@ class IrModuleToJsTransformer(
         return makeJsCodeGeneratorFromIr(exportData, mode) to dts
     }
 
-    fun generateBinaryAst(files: Collection<IrFile>, allModules: Collection<IrModuleFragment>): List<JsIrFragmentAndBinaryAst> {
+    fun generateBinaryAst(files: Collection<IrFile>, allModules: Collection<IrModuleFragment>): List<Pair<IrFile, JsIrProgramFragment>> {
         val exportModelGenerator = ExportModelGenerator(backendContext, generateNamespacesForPackages = !isEsModules)
 
         val exportData = files.map { it to exportModelGenerator.generateExportWithExternals(it) }
 
         doStaticMembersLowering(allModules)
 
-        val serializer = JsIrAstSerializer()
         return exportData.map { (file, exports) ->
-            val fragment = generateProgramFragment(file, exports, minimizedMemberNames = false)
-            val output = ByteArrayOutputStream()
-            serializer.serialize(fragment, output)
-            val binaryAst = output.toByteArray()
-            JsIrFragmentAndBinaryAst(file, fragment, binaryAst)
+            file to generateProgramFragment(file, exports, minimizedMemberNames = false)
         }
     }
 
@@ -490,4 +482,3 @@ fun generateSingleWrappedModuleBody(
         sourceMapBuilder?.build()
     )
 }
-

js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt

@@ -10,6 +10,7 @@ import org.jetbrains.kotlin.ir.backend.js.WholeWorldStageController
 import org.jetbrains.kotlin.ir.backend.js.ic.*
 import org.jetbrains.kotlin.ir.backend.js.moduleName
 import org.jetbrains.kotlin.ir.backend.js.utils.serialization.JsIrAstDeserializer
+import org.jetbrains.kotlin.ir.backend.js.utils.serialization.JsIrAstSerializer
 import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC
 import org.jetbrains.kotlin.js.test.handlers.JsBoxRunner
 import org.jetbrains.kotlin.konan.properties.propertyList
@@ -24,6 +25,7 @@ import org.jetbrains.kotlin.test.services.compilerConfigurationProvider
 import org.jetbrains.kotlin.test.services.configuration.JsEnvironmentConfigurator
 import org.jetbrains.kotlin.test.services.jsLibraryProvider
 import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
 import java.io.File
 
 private class TestArtifactCache(val moduleName: String, val binaryAsts: MutableMap<String, ByteArray> = mutableMapOf()) {
@@ -34,7 +36,7 @@ private class TestArtifactCache(val moduleName: String, val binaryAsts: MutableM
             fileArtifacts = binaryAsts.entries.map {
                 SrcFileArtifact(
                     srcFilePath = it.key,
-                    // TODO: It will be better to use saved fragments (from JsIrFragmentAndBinaryAst), but it doesn't work
+                    // TODO: It will be better to use saved fragments, but it doesn't work
                     //  Merger.merge() + JsNode.resolveTemporaryNames() modify fragments,
                     //  therefore the sequential calls produce different results
                     fragment = deserializer.deserialize(ByteArrayInputStream(it.value))
@@ -146,9 +148,13 @@ class JsIrIncrementalDataProvider(private val testServices: TestServices) : Test
         )
 
         val moduleCache = icCache[canonicalPath] ?: TestArtifactCache(mainModuleIr.name.asString())
+
+        val serializer = JsIrAstSerializer()
         for (rebuiltFile in rebuiltFiles) {
-            if (rebuiltFile.irFile.module == mainModuleIr) {
-                moduleCache.binaryAsts[rebuiltFile.irFile.fileEntry.name] = rebuiltFile.binaryAst
+            if (rebuiltFile.first.module == mainModuleIr) {
+                val output = ByteArrayOutputStream()
+                serializer.serialize(rebuiltFile.second, output)
+                moduleCache.binaryAsts[rebuiltFile.first.fileEntry.name] = output.toByteArray()
             }
         }