[JS IR IC] Do not read non modified JS into memory

Optimizaion: instead of reading all non modified JS files
 from the cache into the memory at once, the patch allows
 copying non-modified JS files from disk one by one
 during the compilation output writing routine.

compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt

@@ -77,6 +77,7 @@ import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerIr
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformer
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsCodeGenerator
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.TranslationMode
+import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.CompilationOutputsBuilt
 import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl
 import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC
 import org.jetbrains.kotlin.ir.symbols.IrSymbol
@@ -155,7 +156,7 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
             return transformer.makeJsCodeGenerator(ir.allModules, mode)
         }
 
-        fun compileAndTransformIrNew(): CompilationOutputs {
+        fun compileAndTransformIrNew(): CompilationOutputsBuilt {
             return makeJsCodeGenerator().generateJsCode(relativeRequirePath = true, outJsProgram = false)
         }
     }
@@ -317,7 +318,7 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
                 )
 
                 val (outputs, rebuiltModules) = jsExecutableProducer.buildExecutable(arguments.irPerModule, outJsProgram = false)
-                outputs.write(outputDir, outputName, arguments.generateDts, moduleName, moduleKind)
+                outputs.writeAll(outputDir, outputName, arguments.generateDts, moduleName, moduleKind)
 
                 messageCollector.report(INFO, "Executable production duration (IC): ${System.currentTimeMillis() - beforeIc2Js}ms")
                 for ((event, duration) in jsExecutableProducer.getStopwatchLaps()) {
@@ -393,7 +394,7 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
 
                 messageCollector.report(INFO, "Executable production duration: ${System.currentTimeMillis() - start}ms")
 
-                outputs.write(outputDir, outputName, arguments.generateDts, moduleName, moduleKind)
+                outputs.writeAll(outputDir, outputName, arguments.generateDts, moduleName, moduleKind)
             } catch (e: CompilationException) {
                 messageCollector.report(
                     ERROR,
@@ -704,28 +705,6 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
         return icCaches
     }
 
-    private fun CompilationOutputs.write(outputDir: File, outputName: String, genDTS: Boolean, moduleName: String, moduleKind: ModuleKind) {
-        val outputFile = outputDir.resolve("$outputName.js")
-        outputFile.parentFile.mkdirs()
-        outputFile.write(this)
-        dependencies.forEach { (name, content) ->
-            outputDir.resolve("$name.js").let {
-                it.parentFile.mkdirs()
-                it.write(content)
-            }
-        }
-
-        if (genDTS) {
-            val dtsFile = outputDir.resolve("$outputName.d.ts")
-            dtsFile.writeText(getFullTsDefinition(moduleName, moduleKind))
-        }
-    }
-
-    private fun File.write(outputs: CompilationOutputs) {
-        writeText(outputs.jsCode)
-        outputs.writeSourceMapIfPresent(this)
-    }
-
     override fun setupPlatformSpecificArgumentsAndServices(
         configuration: CompilerConfiguration,
         arguments: K2JSCompilerArguments,

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/compiler.kt

@@ -11,46 +11,24 @@ import org.jetbrains.kotlin.backend.common.serialization.linkerissues.checkNoUnb
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.ir.IrBuiltIns
 import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity
-import org.jetbrains.kotlin.ir.backend.js.export.TypeScriptFragment
-import org.jetbrains.kotlin.ir.backend.js.export.toTypeScript
 import org.jetbrains.kotlin.ir.backend.js.lower.collectNativeImplementations
 import org.jetbrains.kotlin.ir.backend.js.lower.generateJsTests
 import org.jetbrains.kotlin.ir.backend.js.lower.moveBodilessDeclarationsToSeparatePlace
 import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
+import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.CompilationOutputs
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.TranslationMode
 import org.jetbrains.kotlin.ir.declarations.IrFactory
 import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
 import org.jetbrains.kotlin.ir.util.ExternalDependenciesGenerator
 import org.jetbrains.kotlin.ir.util.SymbolTable
-import org.jetbrains.kotlin.js.backend.ast.JsProgram
 import org.jetbrains.kotlin.js.config.JSConfigurationKeys
 import org.jetbrains.kotlin.js.config.RuntimeDiagnostic
 import org.jetbrains.kotlin.name.FqName
-import org.jetbrains.kotlin.serialization.js.ModuleKind
-import org.jetbrains.kotlin.utils.addToStdlib.ifNotEmpty
-import java.io.File
 
 class CompilerResult(
     val outputs: Map<TranslationMode, CompilationOutputs>,
 )
 
-class CompilationOutputs(
-    val jsCode: String,
-    val tsDefinitions: TypeScriptFragment? = null,
-    val jsProgram: JsProgram? = null,
-    val sourceMap: String? = null,
-    val dependencies: Iterable<Pair<String, CompilationOutputs>> = emptyList()
-) {
-    fun addDependencies(depends: Iterable<Pair<String, CompilationOutputs>>): CompilationOutputs {
-        return CompilationOutputs(jsCode, tsDefinitions, jsProgram, sourceMap, depends)
-    }
-
-    fun getFullTsDefinition(name: String, moduleKind: ModuleKind): String {
-        val allTsDefinitions = dependencies.mapNotNull { it.second.tsDefinitions } + listOfNotNull(tsDefinitions)
-        return allTsDefinitions.toTypeScript(name, moduleKind)
-    }
-}
-
 class LoweredIr(
     val context: JsIrBackendContext,
     val mainModule: IrModuleFragment,
@@ -163,11 +141,3 @@ fun compileIr(
 
     return LoweredIr(context, moduleFragment, allModules, moduleToName)
 }
-
-fun CompilationOutputs.writeSourceMapIfPresent(outputJsFile: File) {
-    sourceMap?.let {
-        val mapFile = outputJsFile.resolveSibling("${outputJsFile.name}.map")
-        outputJsFile.appendText("\n//# sourceMappingURL=${mapFile.name}\n")
-        mapFile.writeText(it)
-    }
-}

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsExecutableProducer.kt

@@ -5,7 +5,6 @@
 
 package org.jetbrains.kotlin.ir.backend.js.ic
 
-import org.jetbrains.kotlin.ir.backend.js.CompilationOutputs
 import org.jetbrains.kotlin.ir.backend.js.SourceMapsInfo
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.*
 import org.jetbrains.kotlin.serialization.js.ModuleKind
@@ -96,11 +95,10 @@ class JsExecutableProducer(
         val mainModule = cachedMainModule.compileModule(mainModuleName, true)
 
         val cachedOtherModules = cachedProgram.dropLast(1)
-        val dependencies = cachedOtherModules.map {
+        mainModule.dependencies = cachedOtherModules.map {
             it.jsIrHeader.externalModuleName to it.compileModule(it.jsIrHeader.externalModuleName, false)
         }
         stopwatch.stop()
-        val compilationOut = mainModule.addDependencies(dependencies)
-        return BuildResult(compilationOut, rebuildModules)
+        return BuildResult(mainModule, rebuildModules)
     }
 }

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsMultiModuleCache.kt

@@ -5,10 +5,7 @@
 
 package org.jetbrains.kotlin.ir.backend.js.ic
 
-import org.jetbrains.kotlin.ir.backend.js.CompilationOutputs
-import org.jetbrains.kotlin.ir.backend.js.export.TypeScriptFragment
-import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.CrossModuleReferences
-import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrModuleHeader
+import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.*
 import java.io.File
 
 class JsMultiModuleCache(private val moduleArtifacts: List<ModuleArtifact>) {
@@ -99,17 +96,19 @@ class JsMultiModuleCache(private val moduleArtifacts: List<ModuleArtifact>) {
     }
 
     fun fetchCompiledJsCode(artifact: ModuleArtifact) = artifact.artifactsDir?.let { cacheDir ->
-        val jsCode = File(cacheDir, CACHED_MODULE_JS).ifExists { readText() }
-        val sourceMap = File(cacheDir, CACHED_MODULE_JS_MAP).ifExists { readText() }
-        val tsDefinitions = File(cacheDir, CACHED_MODULE_D_TS).ifExists { TypeScriptFragment(readText()) }
-        jsCode?.let { CompilationOutputs(it, tsDefinitions, null, sourceMap) }
+        val jsCodeFilePath = File(cacheDir, CACHED_MODULE_JS).ifExists { absolutePath }
+        val sourceMapFilePath = File(cacheDir, CACHED_MODULE_JS_MAP).ifExists { absolutePath }
+        val tsDefinitionsFilePath = File(cacheDir, CACHED_MODULE_D_TS).ifExists { absolutePath }
+        jsCodeFilePath?.let { CompilationOutputsCached(it, sourceMapFilePath, tsDefinitionsFilePath) }
     }
 
-    fun commitCompiledJsCode(artifact: ModuleArtifact, compilationOutputs: CompilationOutputs) = artifact.artifactsDir?.let { cacheDir ->
-        File(cacheDir, CACHED_MODULE_JS).writeIfNotNull(compilationOutputs.jsCode)
-        File(cacheDir, CACHED_MODULE_JS_MAP).writeIfNotNull(compilationOutputs.sourceMap)
-        File(cacheDir, CACHED_MODULE_D_TS).writeIfNotNull(compilationOutputs.tsDefinitions?.raw)
-    }
+    fun commitCompiledJsCode(artifact: ModuleArtifact, compilationOutputs: CompilationOutputsBuilt) =
+        artifact.artifactsDir?.let { cacheDir ->
+            val jsCodeFile = File(cacheDir, CACHED_MODULE_JS)
+            val jsMapFile = File(cacheDir, CACHED_MODULE_JS_MAP)
+            compilationOutputs.writeJsCode(jsCodeFile, jsMapFile)
+            File(cacheDir, CACHED_MODULE_D_TS).writeIfNotNull(compilationOutputs.tsDefinitions?.raw)
+        }
 
     fun loadProgramHeadersFromCache(): List<CachedModuleInfo> {
         return moduleArtifacts.map { artifact ->

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/transformers/irToJs/CompilationOutputs.kt

@@ -0,0 +1,95 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.ir.backend.js.transformers.irToJs
+
+import org.jetbrains.kotlin.ir.backend.js.export.TypeScriptFragment
+import org.jetbrains.kotlin.ir.backend.js.export.toTypeScript
+import org.jetbrains.kotlin.js.backend.ast.JsProgram
+import org.jetbrains.kotlin.serialization.js.ModuleKind
+import java.io.File
+
+abstract class CompilationOutputs {
+    var dependencies: Collection<Pair<String, CompilationOutputs>> = emptyList()
+
+    abstract val tsDefinitions: TypeScriptFragment?
+
+    abstract val jsProgram: JsProgram?
+
+    abstract fun writeJsCode(outputJsFile: File, outputJsMapFile: File)
+
+    fun writeAll(outputDir: File, outputName: String, genDTS: Boolean, moduleName: String, moduleKind: ModuleKind): List<String> {
+        val writtenJsFiles = ArrayList<String>(dependencies.size + 1)
+
+        val outputJsFile = outputDir.resolve("$outputName.js")
+
+        outputJsFile.parentFile.mkdirs()
+        writeJsCode(outputJsFile, outputJsFile.mapForJsFile)
+
+        dependencies.forEach { (name, content) ->
+            outputDir.resolve("$name.js").let { depJsFile ->
+                depJsFile.parentFile.mkdirs()
+                content.writeJsCode(depJsFile, depJsFile.mapForJsFile)
+                writtenJsFiles += depJsFile.absolutePath
+            }
+        }
+
+        writtenJsFiles += outputJsFile.absolutePath
+
+        if (genDTS) {
+            outputJsFile.dtsForJsFile.writeText(getFullTsDefinition(moduleName, moduleKind))
+        }
+
+        return writtenJsFiles
+    }
+
+    fun getFullTsDefinition(moduleName: String, moduleKind: ModuleKind): String {
+        val allTsDefinitions = dependencies.mapNotNull { it.second.tsDefinitions } + listOfNotNull(tsDefinitions)
+        return allTsDefinitions.toTypeScript(moduleName, moduleKind)
+    }
+
+    private val File.mapForJsFile
+        get() = resolveSibling("$name.map")
+
+    private val File.dtsForJsFile
+        get() = resolveSibling("$nameWithoutExtension.d.ts")
+}
+
+class CompilationOutputsBuilt(
+    private val rawJsCode: String,
+    private val sourceMap: String?,
+    override val tsDefinitions: TypeScriptFragment?,
+    override val jsProgram: JsProgram?,
+) : CompilationOutputs() {
+    override fun writeJsCode(outputJsFile: File, outputJsMapFile: File) {
+        var jsCodeWithSourceMap = rawJsCode
+
+        sourceMap?.let {
+            outputJsMapFile.writeText(it)
+            jsCodeWithSourceMap = "$jsCodeWithSourceMap\n//# sourceMappingURL=${outputJsMapFile.name}\n"
+        }
+        outputJsFile.writeText(jsCodeWithSourceMap)
+    }
+}
+
+class CompilationOutputsCached(
+    private val jsCodeFilePath: String,
+    private val sourceMapFilePath: String?,
+    private val tsDefinitionsFilePath: String?
+) : CompilationOutputs() {
+    override val tsDefinitions: TypeScriptFragment?
+        get() = tsDefinitionsFilePath?.let { TypeScriptFragment(File(it).readText()) }
+
+    override val jsProgram: JsProgram?
+        get() = null
+
+    override fun writeJsCode(outputJsFile: File, outputJsMapFile: File) {
+        File(jsCodeFilePath).copyTo(outputJsFile, true)
+
+        sourceMapFilePath?.let {
+            File(it).copyTo(outputJsMapFile, true)
+        }
+    }
+}

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/transformers/irToJs/IrModuleToJsTransformer.kt

@@ -81,7 +81,7 @@ class JsCodeGenerator(
     private val moduleKind: ModuleKind,
     private val sourceMapsInfo: SourceMapsInfo?
 ) {
-    fun generateJsCode(relativeRequirePath: Boolean, outJsProgram: Boolean): CompilationOutputs {
+    fun generateJsCode(relativeRequirePath: Boolean, outJsProgram: Boolean): CompilationOutputsBuilt {
         return generateWrappedModuleBody(
             multiModule,
             mainModuleName,
@@ -380,7 +380,7 @@ private fun generateWrappedModuleBody(
     sourceMapsInfo: SourceMapsInfo?,
     relativeRequirePath: Boolean,
     outJsProgram: Boolean
-): CompilationOutputs {
+): CompilationOutputsBuilt {
     if (multiModule) {
         // mutable container allows explicitly remove elements from itself,
         // so we are able to help GC to free heavy JsIrModule objects
@@ -398,7 +398,7 @@ private fun generateWrappedModuleBody(
             )
         }
 
-        val dependencies = buildList(moduleToRef.size) {
+        mainModule.dependencies = buildList(moduleToRef.size) {
             while (moduleToRef.isNotEmpty()) {
                 moduleToRef.removeFirst().let { (module, moduleRef) ->
                     val moduleName = module.externalModuleName
@@ -416,7 +416,7 @@ private fun generateWrappedModuleBody(
             }
         }
 
-        return mainModule.addDependencies(dependencies)
+        return mainModule
     } else {
         return generateSingleWrappedModuleBody(
             mainModuleName,
@@ -437,7 +437,7 @@ fun generateSingleWrappedModuleBody(
     generateCallToMain: Boolean,
     crossModuleReferences: CrossModuleReferences = CrossModuleReferences.Empty(moduleKind),
     outJsProgram: Boolean = true
-): CompilationOutputs {
+): CompilationOutputsBuilt {
     val program = Merger(
         moduleName,
         moduleKind,
@@ -476,10 +476,10 @@ fun generateSingleWrappedModuleBody(
 
     program.accept(JsToStringGenerationVisitor(jsCode, sourceMapBuilderConsumer))
 
-    return CompilationOutputs(
+    return CompilationOutputsBuilt(
         jsCode.toString(),
+        sourceMapBuilder?.build(),
         fragments.mapNotNull { it.dts }.ifNotEmpty { joinTypeScriptFragments() },
-        program.takeIf { outJsProgram },
-        sourceMapBuilder?.build()
+        program.takeIf { outJsProgram }
     )
 }