diff --git a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt index 12e7c96a758..e739ba5378e 100644 --- a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt +++ b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt @@ -38,6 +38,8 @@ import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider import org.jetbrains.kotlin.incremental.js.IncrementalNextRoundChecker import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer import org.jetbrains.kotlin.ir.backend.js.* +import org.jetbrains.kotlin.ir.backend.js.ic.buildCache +import org.jetbrains.kotlin.ir.backend.js.ic.checkCaches import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrFactory import org.jetbrains.kotlin.js.config.* @@ -179,6 +181,43 @@ class K2JsIrCompiler : CLICompiler() { // TODO: Handle non-empty main call arguments val mainCallArguments = if (K2JsArgumentConstants.NO_CALL == arguments.main) null else emptyList() + val icCaches = configureLibraries(arguments.cacheDirectories) + + if (arguments.irBuildCache) { + messageCollector.report(INFO, "") + messageCollector.report(INFO, "Building cache:") + messageCollector.report(INFO, "to: ${outputFilePath}") + messageCollector.report(INFO, arguments.cacheDirectories ?: "") + messageCollector.report(INFO, resolvedLibraries.getFullList().map { it.libraryName }.toString()) + + val includes = arguments.includes!! + + // TODO: deduplicate + val mainModule = run { + if (sourcesFiles.isNotEmpty()) { + messageCollector.report(ERROR, "Source files are not supported when -Xinclude is present") + } + val allLibraries = resolvedLibraries.getFullList() + val mainLib = allLibraries.find { it.libraryFile.absolutePath == File(includes).absolutePath }!! + MainModule.Klib(mainLib) + } + + val start = System.currentTimeMillis() + buildCache( + cachePath = outputFilePath, + project = projectJs, + mainModule = mainModule, + analyzer = AnalyzerWithCompilerReport(config.configuration), + configuration = config.configuration, + allDependencies = resolvedLibraries, + friendDependencies = friendDependencies, + icCache = checkCaches(resolvedLibraries, icCaches, skipLib = mainModule.lib.libraryFile.absolutePath) + ) + + messageCollector.report(INFO, "IC cache building duration: ${System.currentTimeMillis() - start}ms") + return OK + } + if (arguments.irProduceKlibDir || arguments.irProduceKlibFile) { if (arguments.irProduceKlibFile) { require(outputFile.extension == KLIB_FILE_EXTENSION) { "Please set up .klib file as output" } @@ -199,6 +238,9 @@ class K2JsIrCompiler : CLICompiler() { } if (arguments.irProduceJs) { + messageCollector.report(INFO,"Produce executable: $outputFilePath") + messageCollector.report(INFO, arguments.cacheDirectories ?: "") + val phaseConfig = createPhaseConfig(jsPhases, arguments, messageCollector) val includes = arguments.includes @@ -243,6 +285,8 @@ class K2JsIrCompiler : CLICompiler() { return OK } + val start = System.currentTimeMillis() + val compiledModule = compile( projectJs, mainModule, @@ -270,8 +314,13 @@ class K2JsIrCompiler : CLICompiler() { arguments.irSafeExternalBooleanDiagnostic, messageCollector ), + lowerPerModule = icCaches.isNotEmpty(), + useStdlibCache = icCaches.isNotEmpty(), + icCache = if (icCaches.isNotEmpty()) checkCaches(resolvedLibraries, icCaches, skipLib = (mainModule as MainModule.Klib).lib.libraryFile.absolutePath).data else emptyMap(), ) + messageCollector.report(INFO, "Executable production duration: ${System.currentTimeMillis() - start}ms") + val outputs = if (arguments.irDce && !arguments.irDceDriven) compiledModule.outputsAfterDce!! else compiledModule.outputs!! outputFile.write(outputs) outputs.dependencies.forEach { (name, content) -> diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/Dce.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/Dce.kt index 3e16b3a85b5..02619a9c1ab 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/Dce.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/Dce.kt @@ -136,7 +136,9 @@ private fun processUselessDeclarations( // Remove annotations for `findAssociatedObject` feature, which reference objects eliminated by the DCE. // Otherwise `JsClassGenerator.generateAssociatedKeyProperties` will try to reference the object factory (which is removed). // That will result in an error from the Namer. It cannot generate a name for an absent declaration. - declaration.annotations = declaration.annotations.filter { it.shouldKeepAnnotation() } + if (declaration.annotations.any { !it.shouldKeepAnnotation() }) { + declaration.annotations = declaration.annotations.filter { it.shouldKeepAnnotation() } + } } // TODO bring back the primary constructor fix diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/JsIrBackendContext.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/JsIrBackendContext.kt index 52c9838e5e2..55fe1f92890 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/JsIrBackendContext.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/JsIrBackendContext.kt @@ -60,6 +60,7 @@ class JsIrBackendContext( val baseClassIntoMetadata: Boolean = false, val safeExternalBoolean: Boolean = false, val safeExternalBooleanDiagnostic: RuntimeDiagnostic? = null, + override val mapping: JsMapping = JsMapping(symbolTable.irFactory) ) : JsCommonBackendContext { val fileToInitializationFuns: MutableMap = mutableMapOf() val fileToInitializerPureness: MutableMap = mutableMapOf() @@ -137,8 +138,6 @@ class JsIrBackendContext( val testRoots: Map get() = testContainerFuns - override val mapping = JsMapping(irFactory) - override val inlineClassesUtils = JsInlineClassesUtils(this) val innerClassesSupport = JsInnerClassesSupport(mapping, irFactory) diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/WholeWorldStageController.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/WholeWorldStageController.kt index d4bb8fcb880..cbd061f963a 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/WholeWorldStageController.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/WholeWorldStageController.kt @@ -7,7 +7,9 @@ package org.jetbrains.kotlin.ir.backend.js import org.jetbrains.kotlin.descriptors.DescriptorVisibilities import org.jetbrains.kotlin.ir.declarations.IrClass +import org.jetbrains.kotlin.ir.declarations.IrDeclaration import org.jetbrains.kotlin.ir.declarations.StageController +import org.jetbrains.kotlin.ir.util.IdSignature // Only allows to apply a lowering to the whole world and save the result class WholeWorldStageController : StageController() { @@ -15,8 +17,30 @@ class WholeWorldStageController : StageController() { // TODO assert lowered + override var currentDeclaration: IrDeclaration? = null + private var index: Int = 0 + var declarationListsRestricted: Boolean = false + override fun restrictTo(declaration: IrDeclaration, fn: () -> T): T { + val previousCurrentDeclaration = currentDeclaration + val previousIndex = index + + currentDeclaration = declaration + index = 0 + + return try { + fn() + } finally { + currentDeclaration = previousCurrentDeclaration + index = previousIndex + } + } + + override fun createSignature(parentSignature: IdSignature): IdSignature { + return IdSignature.LoweredDeclarationSignature(parentSignature, currentStage, index++) + } + override fun withStage(stage: Int, fn: () -> T): T { val oldStage = currentStage currentStage = stage diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/compiler.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/compiler.kt index 20126c6aa30..b05421e4e7d 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/compiler.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/compiler.kt @@ -11,6 +11,8 @@ import org.jetbrains.kotlin.backend.common.lower import org.jetbrains.kotlin.backend.common.phaser.PhaseConfig import org.jetbrains.kotlin.backend.common.phaser.invokeToplevel import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.ir.backend.js.ic.SerializedIcData +import org.jetbrains.kotlin.ir.backend.js.ic.icCompile import org.jetbrains.kotlin.ir.backend.js.lower.generateTests import org.jetbrains.kotlin.ir.backend.js.lower.moveBodilessDeclarationsToSeparatePlace import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformer @@ -19,8 +21,7 @@ import org.jetbrains.kotlin.ir.declarations.IrFactory import org.jetbrains.kotlin.ir.declarations.IrModuleFragment import org.jetbrains.kotlin.ir.declarations.StageController import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrFactory -import org.jetbrains.kotlin.ir.util.ExternalDependenciesGenerator -import org.jetbrains.kotlin.ir.util.noUnboundLeft +import org.jetbrains.kotlin.ir.util.* import org.jetbrains.kotlin.js.config.RuntimeDiagnostic import org.jetbrains.kotlin.name.FqName @@ -61,7 +62,32 @@ fun compile( lowerPerModule: Boolean = false, safeExternalBoolean: Boolean = false, safeExternalBooleanDiagnostic: RuntimeDiagnostic? = null, + useStdlibCache: Boolean = false, + icCache: Map = emptyMap(), ): CompilerResult { + + if (lowerPerModule) { + return icCompile( + project, + mainModule, + analyzer, + configuration, + allDependencies, + friendDependencies, + mainArguments, + exportedDeclarations, + generateFullJs, + generateDceJs, + dceRuntimeDiagnostic, + es6mode, + multiModule, + relativeRequirePath, + propertyLazyInitialization, + useStdlibCache, + icCache, + ) + } + val (moduleFragment: IrModuleFragment, dependencyModules, irBuiltIns, symbolTable, deserializer, moduleToName) = loadIr(project, mainModule, analyzer, configuration, dependencies, friendDependencies, irFactory, verifySignatures) @@ -125,6 +151,7 @@ fun compile( ) return transformer.generateModule(allModules) } else { + // TODO is this reachable when lowerPerModule == true? if (lowerPerModule) { val controller = WholeWorldStageController() check(irFactory is PersistentIrFactory) @@ -132,6 +159,7 @@ fun compile( allModules.forEach { lowerPreservingIcData(it, context, controller) } + irFactory.stageController = object : StageController(irFactory.stageController.currentStage) {} } else { jsPhases.invokeToplevel(phaseConfig, context, allModules) } @@ -177,4 +205,4 @@ fun generateJsCode( val transformer = IrModuleToJsTransformer(context, null, true, nameTables) return transformer.generateModule(listOf(moduleFragment)).outputs!!.jsCode -} +} \ No newline at end of file diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/fileUtil.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/fileUtil.kt new file mode 100644 index 00000000000..f3cbb37d3db --- /dev/null +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/fileUtil.kt @@ -0,0 +1,153 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.js.ic + +import com.intellij.openapi.project.Project +import org.jetbrains.kotlin.analyzer.AbstractAnalyzerWithCompilerReport +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.ir.backend.js.MainModule +import org.jetbrains.kotlin.ir.backend.js.toByteArray +import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.library.impl.javaFile +import org.jetbrains.kotlin.library.resolver.KotlinLibraryResolveResult +import org.jetbrains.kotlin.library.resolver.TopologicalLibraryOrder +import org.jetbrains.kotlin.name.FqName +import java.io.File +import java.io.PrintWriter +import java.security.MessageDigest +import kotlin.random.Random +import kotlin.random.nextULong + +// TODO: Proper version of the compiler (should take changes to lowerings into account) +private val compilerVersion = Random.nextULong() + +// TODO more parameters for lowerings +fun buildCache( + cachePath: String, + project: Project, + mainModule: MainModule.Klib, + analyzer: AbstractAnalyzerWithCompilerReport, + configuration: CompilerConfiguration, + allDependencies: KotlinLibraryResolveResult, + friendDependencies: List, + exportedDeclarations: Set = emptySet(), + forceClean: Boolean = false, + icCache: IcCacheInfo = IcCacheInfo.EMPTY, +) { + val dependencyHashes = allDependencies.getFullList(TopologicalLibraryOrder).mapNotNull { + val path = it.libraryFile.absolutePath + icCache.md5[path] + } + compilerVersion + + val md5 = mainModule.lib.libraryFile.javaFile().md5(dependencyHashes) + + if (!forceClean) { + val oldCacheInfo = CacheInfo.load(cachePath) + if (oldCacheInfo != null && md5 == oldCacheInfo.md5) return + } + + val icDir = File(cachePath) + icDir.deleteRecursively() + icDir.mkdirs() + + val icData = prepareSingleLibraryIcCache(project, analyzer, configuration, mainModule.lib, allDependencies, friendDependencies, exportedDeclarations, icCache.data) + + icData.writeTo(File(cachePath)) + + CacheInfo(cachePath, mainModule.lib.libraryFile.absolutePath, md5).save() +} + +private fun File.md5(additional: Iterable = emptyList()): ULong { + val md5 = MessageDigest.getInstance("MD5") + + for (ul in additional) { + md5.update(ul.toLong().toByteArray()) + } + + fun File.process(prefix: String = "") { + if (isDirectory) { + this.listFiles()!!.sortedBy { it.name }.forEach { + md5.update((prefix + it.name).toByteArray()) + it.process(prefix + it.name + "/") + } + } else { + md5.update(readBytes()) + } + } + + this.process() + + val d = md5.digest() + + return ((d[0].toULong() and 0xFFUL) + or ((d[1].toULong() and 0xFFUL) shl 8) + or ((d[2].toULong() and 0xFFUL) shl 16) + or ((d[3].toULong() and 0xFFUL) shl 24) + or ((d[4].toULong() and 0xFFUL) shl 32) + or ((d[5].toULong() and 0xFFUL) shl 40) + or ((d[6].toULong() and 0xFFUL) shl 48) + or ((d[7].toULong() and 0xFFUL) shl 56) + ) +} + +fun checkCaches( + allDependencies: KotlinLibraryResolveResult, + cachePaths: List, + skipLib: String? = null, +): IcCacheInfo { + val allLibs = allDependencies.getFullList().map { it.libraryFile.absolutePath }.toSet() - skipLib + + val caches = cachePaths.map { CacheInfo.load(it) ?: error("Cannot load IC cache from ${it}") } + + val missedLibs = allLibs - caches.map { it.libPath } + if (!missedLibs.isEmpty()) { + error("Missing caches for libraries: ${missedLibs}") + } + + val result = mutableMapOf() + val md5 = mutableMapOf() + + for (c in caches) { + if (c.libPath !in allLibs) error("Missing library: ${c.libPath}") + + result[c.libPath] = File(c.path).readIcData() + md5[c.libPath] = c.md5 + } + + return IcCacheInfo(result, md5) +} + +// TODO md5 hash +data class CacheInfo(val path: String, val libPath: String, val md5: ULong) { + fun save() { + PrintWriter(File(File(path), "info")).use { + it.println(libPath) + it.println(md5.toString(16)) + } + } + + companion object { + fun load(path: String): CacheInfo? { + val info = File(File(path), "info") + + if (!info.exists()) return null + + val (libPath, md5) = info.readLines() + + return CacheInfo(path, libPath, md5.toULong(16)) + } + } +} + + +class IcCacheInfo( + val data: Map, + val md5: Map, +) { + companion object { + val EMPTY = IcCacheInfo(emptyMap(), emptyMap()) + } +} \ No newline at end of file diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ic.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ic.kt new file mode 100644 index 00000000000..722686f0642 --- /dev/null +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ic.kt @@ -0,0 +1,277 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.js.ic + +import com.intellij.openapi.project.Project +import org.jetbrains.kotlin.analyzer.AbstractAnalyzerWithCompilerReport +import org.jetbrains.kotlin.backend.common.lower +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.ir.backend.js.* +import org.jetbrains.kotlin.ir.backend.js.lower.generateTests +import org.jetbrains.kotlin.ir.backend.js.lower.moveBodilessDeclarationsToSeparatePlace +import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker +import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformer +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrFactory +import org.jetbrains.kotlin.ir.util.* +import org.jetbrains.kotlin.js.config.RuntimeDiagnostic +import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.library.resolver.KotlinLibraryResolveResult +import org.jetbrains.kotlin.library.resolver.KotlinResolvedLibrary +import org.jetbrains.kotlin.name.FqName +import java.io.PrintWriter + +fun prepareSingleLibraryIcCache( + project: Project, + analyzer: AbstractAnalyzerWithCompilerReport, + configuration: CompilerConfiguration, + library: KotlinLibrary, + dependencies: KotlinLibraryResolveResult, + friendDependencies: List = emptyList(), + exportedDeclarations: Set = emptySet(), + icCache: Map = emptyMap(), +): SerializedIcData { + val irFactory = PersistentIrFactory() + val controller = WholeWorldStageController() + irFactory.stageController = controller + + val (context, deserializer, allModules) = prepareIr( + project, + MainModule.Klib(library), + analyzer, + configuration, + dependencies, + friendDependencies, + exportedDeclarations, + null, + false, + false, + irFactory, + useGlobalSignatures = true, + useStdlibCache = true, + icCache = icCache + ) + + val moduleFragment = allModules.last() + + moveBodilessDeclarationsToSeparatePlace(context, moduleFragment) + +// generateTests(context, moduleFragment) + + lowerPreservingIcData(moduleFragment, context, controller) + + return IcSerializer( + context.irBuiltIns, + context.mapping, + irFactory, + deserializer, + moduleFragment + ).serializeDeclarations(irFactory.allDeclarations) +} + +private fun KotlinResolvedLibrary.allDependencies(): List { + val visited = mutableSetOf() + + val result = mutableListOf() + + fun KotlinResolvedLibrary.dfs() { + visited += this + + resolvedDependencies.forEach { + if (it !in visited) { + it.dfs() + result += it + } + } + } + + dfs() + + return result +} + +private fun dumpIr(module: IrModuleFragment, fileName: String) { + val dumpOptions = KotlinLikeDumpOptions(printElseAsTrue = true) + + var actual = "" + + for (file in module.files) { + actual += file.path + "\n" + actual += run { + var r = "" + + file.declarations.map { it.dumpKotlinLike(dumpOptions) }.sorted().forEach { r += it } + + r + } + actual += "\n" + } + PrintWriter("/home/ab/vcs/kotlin/$fileName.txt").use { + it.print(actual) + } +} + +fun icCompile( + project: Project, + mainModule: MainModule, + analyzer: AbstractAnalyzerWithCompilerReport, + configuration: CompilerConfiguration, + allDependencies: KotlinLibraryResolveResult, + friendDependencies: List, + mainArguments: List?, + exportedDeclarations: Set = emptySet(), + generateFullJs: Boolean = true, + generateDceJs: Boolean = false, + dceRuntimeDiagnostic: RuntimeDiagnostic? = null, + es6mode: Boolean = false, + multiModule: Boolean = false, + relativeRequirePath: Boolean = false, + propertyLazyInitialization: Boolean, + useStdlibCache: Boolean, + icCache: Map = emptyMap() +): CompilerResult { + + val irFactory = PersistentIrFactory() + val controller = WholeWorldStageController() + irFactory.stageController = controller + + val (context, _, allModules, moduleToName, loweredIrLoaded) = prepareIr( + project, + mainModule, + analyzer, + configuration, + allDependencies, + friendDependencies, + exportedDeclarations, + dceRuntimeDiagnostic, + es6mode, + propertyLazyInitialization, + irFactory, + useStdlibCache, + useStdlibCache, + icCache, + ) + + val modulesToLower = allModules.filter { it !in loweredIrLoaded } + + + + if (!modulesToLower.isEmpty()) { + // This won't work incrementally + modulesToLower.forEach { module -> + moveBodilessDeclarationsToSeparatePlace(context, module) + } + + generateTests(context, modulesToLower.last()) + + modulesToLower.forEach { + lowerPreservingIcData(it, context, controller) + } + } + +// dumpIr(allModules.first(), "simple-dump${if (useStdlibCache) "-actual" else ""}") + + val transformer = IrModuleToJsTransformer( + context, + mainArguments, + fullJs = generateFullJs, + dceJs = generateDceJs, + multiModule = multiModule, + relativeRequirePath = relativeRequirePath, + moduleToName = moduleToName, + ) + + irFactory.stageController = object : StageController(999) {} + + return transformer.generateModule(allModules) +} + +fun lowerPreservingIcData(module: IrModuleFragment, context: JsIrBackendContext, controller: WholeWorldStageController) { + // Lower all the things + controller.currentStage = 0 + + pirLowerings.forEachIndexed { i, lowering -> + controller.currentStage = i + 1 + when (lowering) { + is DeclarationLowering -> + lowering.declarationTransformer(context).lower(module) + is BodyLowering -> + lowering.bodyLowering(context).lower(module) + else -> TODO("what about other lowerings?") + } + } + + controller.currentStage = pirLowerings.size + 1 +} + +private fun prepareIr( + project: Project, + mainModule: MainModule, + analyzer: AbstractAnalyzerWithCompilerReport, + configuration: CompilerConfiguration, + allDependencies: KotlinLibraryResolveResult, + friendDependencies: List, + exportedDeclarations: Set = emptySet(), + dceRuntimeDiagnostic: RuntimeDiagnostic? = null, + es6mode: Boolean = false, + propertyLazyInitialization: Boolean, + irFactory: PersistentIrFactory, + useGlobalSignatures: Boolean, + useStdlibCache: Boolean, + icCache: Map, +): PreparedIr { + val cacheProvider: LoweringsCacheProvider? = when { + useStdlibCache -> object : LoweringsCacheProvider { + override fun cacheByPath(path: String): SerializedIcData? { + return icCache[path] + } + } + useGlobalSignatures -> EmptyLoweringsCacheProvider + else -> null + } + + val (moduleFragment: IrModuleFragment, dependencyModules, irBuiltIns, symbolTable, deserializer, moduleToName, loweredIrLoaded) = + loadIr(project, mainModule, analyzer, configuration, allDependencies, friendDependencies, irFactory, cacheProvider) + + val moduleDescriptor = moduleFragment.descriptor + + val allModules = when (mainModule) { + is MainModule.SourceFiles -> dependencyModules + listOf(moduleFragment) + is MainModule.Klib -> dependencyModules + } + + val context = JsIrBackendContext( + moduleDescriptor, + irBuiltIns, + symbolTable, + allModules.first(), + exportedDeclarations, + configuration, + es6mode = es6mode, + dceRuntimeDiagnostic = dceRuntimeDiagnostic, + propertyLazyInitialization = propertyLazyInitialization, + mapping = deserializer.mapping, + ) + + // Load declarations referenced during `context` initialization + val irProviders = listOf(deserializer) + ExternalDependenciesGenerator(symbolTable, irProviders).generateUnboundSymbolsAsDependencies() + + deserializer.postProcess() + symbolTable.noUnboundLeft("Unbound symbols at the end of linker") + + deserializer.loadIcIr { moveBodilessDeclarationsToSeparatePlace(context, it) } + + return PreparedIr(context, deserializer, allModules, moduleToName, loweredIrLoaded) +} + +data class PreparedIr( + val context: JsIrBackendContext, + val linker: JsIrLinker, + val allModules: List, + val moduleToName: Map, + val loweredIrLoaded: Set, +) \ No newline at end of file diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/transformers/irToJs/IrElementToJsExpressionTransformer.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/transformers/irToJs/IrElementToJsExpressionTransformer.kt index 9890b348be5..cc51d37902c 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/transformers/irToJs/IrElementToJsExpressionTransformer.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/transformers/irToJs/IrElementToJsExpressionTransformer.kt @@ -11,10 +11,7 @@ import org.jetbrains.kotlin.ir.backend.js.utils.JsGenerationContext import org.jetbrains.kotlin.ir.backend.js.utils.Namer import org.jetbrains.kotlin.ir.backend.js.utils.emptyScope import org.jetbrains.kotlin.ir.backend.js.utils.getJsNameOrKotlinName -import org.jetbrains.kotlin.ir.declarations.IrClass -import org.jetbrains.kotlin.ir.declarations.IrConstructor -import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction -import org.jetbrains.kotlin.ir.declarations.IrValueDeclaration +import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.expressions.* import org.jetbrains.kotlin.ir.types.isUnit import org.jetbrains.kotlin.ir.util.isEffectivelyExternal @@ -324,7 +321,7 @@ class IrElementToJsExpressionTransformer : BaseIrElementToJsNodeTransformer this === p.dispatchReceiverParameter is IrClass -> this === p.thisReceiver else -> false diff --git a/compiler/ir/ir.tree.persistent/src/org/jetbrains/kotlin/ir/declarations/persistent/PersistentIrFactory.kt b/compiler/ir/ir.tree.persistent/src/org/jetbrains/kotlin/ir/declarations/persistent/PersistentIrFactory.kt index bfa4868f44f..5567072506c 100644 --- a/compiler/ir/ir.tree.persistent/src/org/jetbrains/kotlin/ir/declarations/persistent/PersistentIrFactory.kt +++ b/compiler/ir/ir.tree.persistent/src/org/jetbrains/kotlin/ir/declarations/persistent/PersistentIrFactory.kt @@ -9,11 +9,13 @@ import org.jetbrains.kotlin.descriptors.* import org.jetbrains.kotlin.ir.IrStatement import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.expressions.IrBlockBody +import org.jetbrains.kotlin.ir.expressions.IrBody import org.jetbrains.kotlin.ir.expressions.IrExpression import org.jetbrains.kotlin.ir.expressions.IrExpressionBody import org.jetbrains.kotlin.ir.expressions.persistent.PersistentIrBlockBody import org.jetbrains.kotlin.ir.expressions.persistent.PersistentIrExpressionBody import org.jetbrains.kotlin.ir.symbols.* +import org.jetbrains.kotlin.ir.symbols.impl.IrPublicSymbolBase import org.jetbrains.kotlin.ir.types.IrType import org.jetbrains.kotlin.ir.util.IdSignature import org.jetbrains.kotlin.name.Name @@ -24,8 +26,24 @@ class PersistentIrFactory : IrFactory { override var stageController = StageController() + val allDeclarations = mutableListOf() + + val allBodies = mutableListOf() + + private fun IrDeclaration.register() { + allDeclarations += this + } + + fun declarationSignature(declaration: IrDeclaration): IdSignature? { + return (declaration as? PersistentIrDeclarationBase<*>)?.signature ?: declaration.symbol.signature + } + @Suppress("UNUSED_PARAMETER") - fun currentSignature(declaration: IrDeclaration): IdSignature? = null + internal fun currentSignature(declaration: IrDeclaration): IdSignature? { + val parentSig = stageController.currentDeclaration?.let { declarationSignature(it) } ?: return null + + return stageController.createSignature(parentSig) + } override fun createAnonymousInitializer( startOffset: Int, @@ -34,7 +52,7 @@ class PersistentIrFactory : IrFactory { symbol: IrAnonymousInitializerSymbol, isStatic: Boolean, ): IrAnonymousInitializer = - PersistentIrAnonymousInitializer(startOffset, endOffset, origin, symbol, isStatic, this) + PersistentIrAnonymousInitializer(startOffset, endOffset, origin, symbol, isStatic, this).also { it.register() } override fun createClass( startOffset: Int, @@ -58,7 +76,7 @@ class PersistentIrFactory : IrFactory { startOffset, endOffset, origin, symbol, name, kind, visibility, modality, isCompanion, isInner, isData, isExternal, isInline, isExpect, isFun, source, this - ) + ).also { it.register() } override fun createConstructor( startOffset: Int, @@ -77,7 +95,7 @@ class PersistentIrFactory : IrFactory { PersistentIrConstructor( startOffset, endOffset, origin, symbol, name, visibility, returnType, isInline, isExternal, isPrimary, isExpect, containerSource, this - ) + ).also { it.register() } override fun createEnumEntry( startOffset: Int, @@ -86,14 +104,14 @@ class PersistentIrFactory : IrFactory { symbol: IrEnumEntrySymbol, name: Name, ): IrEnumEntry = - PersistentIrEnumEntry(startOffset, endOffset, origin, symbol, name, this) + PersistentIrEnumEntry(startOffset, endOffset, origin, symbol, name, this).also { it.register() } override fun createErrorDeclaration( startOffset: Int, endOffset: Int, descriptor: DeclarationDescriptor?, ): IrErrorDeclaration = - PersistentIrErrorDeclaration(startOffset, endOffset, descriptor, this) + PersistentIrErrorDeclaration(startOffset, endOffset, descriptor, this).also { it.register() } override fun createField( startOffset: Int, @@ -107,7 +125,7 @@ class PersistentIrFactory : IrFactory { isExternal: Boolean, isStatic: Boolean, ): IrField = - PersistentIrField(startOffset, endOffset, origin, symbol, name, type, visibility, isFinal, isExternal, isStatic, this) + PersistentIrField(startOffset, endOffset, origin, symbol, name, type, visibility, isFinal, isExternal, isStatic, this).also { it.register() } override fun createFunction( startOffset: Int, @@ -132,7 +150,7 @@ class PersistentIrFactory : IrFactory { startOffset, endOffset, origin, symbol, name, visibility, modality, returnType, isInline, isExternal, isTailrec, isSuspend, isOperator, isInfix, isExpect, isFakeOverride, containerSource, this - ) + ).also { it.register() } override fun createFakeOverrideFunction( startOffset: Int, @@ -153,7 +171,7 @@ class PersistentIrFactory : IrFactory { PersistentIrFakeOverrideFunction( startOffset, endOffset, origin, name, visibility, modality, returnType, isInline, isExternal, isTailrec, isSuspend, isOperator, isInfix, isExpect, this - ) + ).also { it.register() } override fun createLocalDelegatedProperty( startOffset: Int, @@ -166,7 +184,7 @@ class PersistentIrFactory : IrFactory { ): IrLocalDelegatedProperty = PersistentIrLocalDelegatedProperty( startOffset, endOffset, origin, symbol, name, type, isVar, this - ) + ).also { it.register() } override fun createProperty( startOffset: Int, @@ -189,7 +207,7 @@ class PersistentIrFactory : IrFactory { startOffset, endOffset, origin, symbol, name, visibility, modality, isVar, isConst, isLateinit, isDelegated, isExternal, isExpect, isFakeOverride, containerSource, this - ) + ).also { it.register() } override fun createFakeOverrideProperty( startOffset: Int, @@ -209,7 +227,7 @@ class PersistentIrFactory : IrFactory { startOffset, endOffset, origin, name, visibility, modality, isVar, isConst, isLateinit, isDelegated, isExternal, isExpect, this - ) + ).also { it.register() } override fun createTypeAlias( startOffset: Int, @@ -221,7 +239,7 @@ class PersistentIrFactory : IrFactory { isActual: Boolean, origin: IrDeclarationOrigin, ): IrTypeAlias = - PersistentIrTypeAlias(startOffset, endOffset, symbol, name, visibility, expandedType, isActual, origin, this) + PersistentIrTypeAlias(startOffset, endOffset, symbol, name, visibility, expandedType, isActual, origin, this).also { it.register() } override fun createTypeParameter( startOffset: Int, @@ -233,7 +251,7 @@ class PersistentIrFactory : IrFactory { isReified: Boolean, variance: Variance, ): IrTypeParameter = - PersistentIrTypeParameter(startOffset, endOffset, origin, symbol, name, index, isReified, variance, this) + PersistentIrTypeParameter(startOffset, endOffset, origin, symbol, name, index, isReified, variance, this).also { it.register() } override fun createValueParameter( startOffset: Int, @@ -251,44 +269,56 @@ class PersistentIrFactory : IrFactory { ): IrValueParameter = PersistentIrValueParameter( startOffset, endOffset, origin, symbol, name, index, type, varargElementType, isCrossinline, isNoinline, isHidden, isAssignable, this - ) + ).also { it.register() } override fun createExpressionBody( startOffset: Int, endOffset: Int, initializer: IrExpressionBody.() -> Unit, ): IrExpressionBody = - PersistentIrExpressionBody(startOffset, endOffset, this, initializer) + PersistentIrExpressionBody(startOffset, endOffset, this, initializer).also { + allBodies += it + } override fun createExpressionBody( startOffset: Int, endOffset: Int, expression: IrExpression, ): IrExpressionBody = - PersistentIrExpressionBody(startOffset, endOffset, expression, this) + PersistentIrExpressionBody(startOffset, endOffset, expression, this).also { + allBodies += it + } override fun createExpressionBody( expression: IrExpression, ): IrExpressionBody = - PersistentIrExpressionBody(expression, this) + PersistentIrExpressionBody(expression, this).also { + allBodies += it + } override fun createBlockBody( startOffset: Int, endOffset: Int, ): IrBlockBody = - PersistentIrBlockBody(startOffset, endOffset, this) + PersistentIrBlockBody(startOffset, endOffset, this).also { + allBodies += it + } override fun createBlockBody( startOffset: Int, endOffset: Int, statements: List, ): IrBlockBody = - PersistentIrBlockBody(startOffset, endOffset, statements, this) + PersistentIrBlockBody(startOffset, endOffset, statements, this).also { + allBodies += it + } override fun createBlockBody( startOffset: Int, endOffset: Int, initializer: IrBlockBody.() -> Unit, ): IrBlockBody = - PersistentIrBlockBody(startOffset, endOffset, this, initializer) + PersistentIrBlockBody(startOffset, endOffset, this, initializer).also { + allBodies += it + } } diff --git a/compiler/ir/serialization.js/build.gradle.kts b/compiler/ir/serialization.js/build.gradle.kts index c5758522a60..936cf777359 100644 --- a/compiler/ir/serialization.js/build.gradle.kts +++ b/compiler/ir/serialization.js/build.gradle.kts @@ -8,6 +8,8 @@ dependencies { api(project(":compiler:ir.serialization.common")) api(project(":js:js.frontend")) implementation(project(":compiler:ir.backend.common")) + compile(project(":compiler:ir.tree.persistent")) + compileOnly(project(":kotlin-reflect-api")) compileOnly(intellijCoreDep()) { includeJars("intellij-core") } } diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/JsMapping.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/JsMapping.kt similarity index 100% rename from compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/JsMapping.kt rename to compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/JsMapping.kt diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcFileDeserializer.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcFileDeserializer.kt new file mode 100644 index 00000000000..e0c0be9130e --- /dev/null +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcFileDeserializer.kt @@ -0,0 +1,313 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.js.ic + +import org.jetbrains.kotlin.backend.common.overrides.DefaultFakeOverrideClassFilter +import org.jetbrains.kotlin.backend.common.serialization.* +import org.jetbrains.kotlin.backend.common.serialization.encodings.BinarySymbolData +import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.ir.backend.js.JsMappingState +import org.jetbrains.kotlin.ir.backend.js.JsStatementOrigins +import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker +import org.jetbrains.kotlin.ir.declarations.IrDeclaration +import org.jetbrains.kotlin.ir.declarations.IrFile +import org.jetbrains.kotlin.ir.expressions.IrStatementOriginImpl +import org.jetbrains.kotlin.ir.serialization.CarrierDeserializer +import org.jetbrains.kotlin.ir.symbols.IrSymbol +import org.jetbrains.kotlin.ir.util.IdSignature +import org.jetbrains.kotlin.library.SerializedIrFile +import org.jetbrains.kotlin.library.impl.DeclarationId +import org.jetbrains.kotlin.library.impl.DeclarationIrTableMemoryReader +import org.jetbrains.kotlin.library.impl.IrArrayMemoryReader +import org.jetbrains.kotlin.library.impl.IrLongArrayMemoryReader +import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite +import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile as ProtoIrFile + +class IcFileDeserializer( + val linker: JsIrLinker, + file: IrFile, + originalFileReader: IrLibraryFile, + fileProto: org.jetbrains.kotlin.backend.common.serialization.proto.IrFile, + deserializeBodies: Boolean, + allowErrorNodes: Boolean, + deserializeInlineFunctions: Boolean, + val moduleDeserializer: IrModuleDeserializer, + useGlobalSignatures: Boolean, + val handleNoModuleDeserializerFound: (IdSignature, ModuleDescriptor, Collection) -> IrModuleDeserializer, + val originalEnqueue: IdSignature.(IcFileDeserializer) -> Unit, + val icFileData: SerializedIcDataForFile, + val mappingState: JsMappingState, + val publicSignatureToIcFileDeserializer: MutableMap, + val enqueue: IdSignature.(IcFileDeserializer) -> Unit, +) { + + val originalSymbolDeserializer = + IrSymbolDeserializer( + linker.symbolTable, + originalFileReader, + file.symbol, + fileProto.actualList, + { idSig -> + idSig.enqueue(this) + if (idSig.hasTopLevel) { + idSig.topLevelSignature().originalEnqueue(this) + } + }, + linker::handleExpectActualMapping, + useGlobalSignatures = useGlobalSignatures, + enqueueAllDeclarations = true, + deserializePublicSymbol = ::deserializeOriginalPublicSymbol, + ) + + private val originalDeclarationDeserializer = IrDeclarationDeserializer( + linker.builtIns, + linker.symbolTable, + linker.symbolTable.irFactory, + originalFileReader, + file, + allowErrorNodes, + deserializeInlineFunctions, + deserializeBodies, + originalSymbolDeserializer, + linker.fakeOverrideBuilder.platformSpecificClassFilter, + linker.fakeOverrideBuilder, + allowRedeclaration = true, + ) + + private fun deserializeOriginalPublicSymbol(idSig: IdSignature, symbolKind: BinarySymbolData.SymbolKind): IrSymbol { + assert(idSig.isPublic) + + val topLevelSig = idSig.topLevelSignature() + + if (idSig in originalFileDeserializer.reversedSignatureIndex) { + topLevelSig.originalEnqueue(this) + idSig.enqueue(this) + linker.modulesWithReachableTopLevels.add(moduleDeserializer) + + return originalFileDeserializer.symbolDeserializer.deserializeIrSymbol(idSig, symbolKind).also { + linker.deserializedSymbols.add(it) + } + } else { + + val actualModuleDeserializer = + moduleDeserializer.findModuleDeserializerForTopLevelId(topLevelSig) ?: handleNoModuleDeserializerFound( + idSig, + moduleDeserializer.moduleDescriptor, + moduleDeserializer.moduleDependencies + ) + + return actualModuleDeserializer.deserializeIrSymbol(idSig, symbolKind) + } + } + + val originalFileDeserializer = IrFileDeserializer(file, originalFileReader, fileProto, originalSymbolDeserializer, originalDeclarationDeserializer) + + val originalVisited = HashSet() + + val originalSignatureQueue = ArrayDeque() // Top-level signatures to be deserialized from original KLIB + + // Returns whether this file should be queued for deserialization + fun enqueueForDeserialization(idSig: IdSignature): Boolean { + if (originalVisited.add(idSig)) { + originalSignatureQueue.addLast(idSig) + return originalSignatureQueue.size == 1 + } + + return false + } + + fun deserializePendingSignatures() { + while (!originalSignatureQueue.isEmpty()) { + val signature = originalSignatureQueue.removeFirst() + deserializeAnyDeclaration(signature) + } + } + + // Explicitly exported declarations (e.g. top-level initializers) must be deserialized before all other declarations. + // Thus we schedule their deserialization in deserializer's constructor. + val explicitlyExportedToCompiler: Collection = fileProto.explicitlyExportedToCompilerList.map { + val symbolData = originalSymbolDeserializer.parseSymbolData(it) + originalSymbolDeserializer.deserializeIdSignature(symbolData.signatureId) + } + + fun allOriginalDeclarationSignatures(): Collection = originalFileDeserializer.reversedSignatureIndex.keys + + // IC data processing starts here + + private val icFileReader = FileReaderFromSerializedIrFile(icFileData.file) + + val symbolDeserializer = IrSymbolDeserializer( + linker.symbolTable, + icFileReader, + file.symbol, + emptyList(), + { idSig -> enqueueLocalTopLevelDeclaration(idSig) }, + { _, s -> s }, + enqueueAllDeclarations = true, + useGlobalSignatures = true, + deserializedSymbols = originalFileDeserializer.symbolDeserializer.deserializedSymbols, + ::deserializePublicSymbol + ) + + private val declarationDeserializer = IrDeclarationDeserializer( + linker.builtIns, + linker.symbolTable, + linker.symbolTable.irFactory, + icFileReader, + file, + allowErrorNodes = true, + deserializeInlineFunctions = true, + deserializeBodies = true, + symbolDeserializer, + DefaultFakeOverrideClassFilter, + linker.fakeOverrideBuilder, + skipMutableState = true, + additionalStatementOriginIndex = additionalStatementOriginIndex, + allowErrorStatementOrigins = true, + allowRedeclaration = true, + ) + + private val protoFile: ProtoIrFile by lazy { ProtoIrFile.parseFrom(icFileData.file.fileData.codedInputStream, ExtensionRegistryLite.newInstance()) } + + private val carrierDeserializer by lazy { CarrierDeserializer(declarationDeserializer, icFileData.carriers) } + + val reversedSignatureIndex: Map by lazy { protoFile.declarationIdList.map { symbolDeserializer.deserializeIdSignature(it) to it }.toMap() } + + val visited = HashSet() + + val mappingsDeserializer by lazy { + mappingState.mappingsDeserializer(icFileData.mappings, { code -> + val symbolData = symbolDeserializer.parseSymbolData(code) + symbolDeserializer.deserializeIdSignature(symbolData.signatureId) + }) { + deserializeIrSymbol(it) + } + } + + fun init() { + reversedSignatureIndex.keys.forEach { + publicSignatureToIcFileDeserializer[it] = this + } + } + + private val containerSigToOrder by lazy { + mutableMapOf().also { map -> + val containerIds = IrLongArrayMemoryReader(icFileData.order.containerSignatures).array + val declarationIds = IrArrayMemoryReader(icFileData.order.declarationSignatures) + + containerIds.forEachIndexed { index, id -> + val symbolData = symbolDeserializer.parseSymbolData(id) + val containerSig = symbolDeserializer.deserializeIdSignature(symbolData.signatureId) + + map[containerSig] = declarationIds.tableItemBytes(index) + } + } + } + + fun loadClassOrder(classSignature: IdSignature): List? { + val bytes = containerSigToOrder[classSignature] ?: return null + + return IrLongArrayMemoryReader(bytes).array.map(::deserializeIrSymbol) + } + + + private fun deserializePublicSymbol(idSig: IdSignature, kind: BinarySymbolData.SymbolKind) : IrSymbol { + // TODO: reference lowered declarations cross-module + val topLevelSig = idSig.topLevelSignature() + val actualModuleDeserializer = + moduleDeserializer.findModuleDeserializerForTopLevelId(topLevelSig) ?: + handleNoModuleDeserializerFound( + idSig, + moduleDeserializer.moduleDescriptor, + moduleDeserializer.moduleDependencies + ) + + return actualModuleDeserializer.deserializeIrSymbol(idSig, kind) + } + + private fun enqueueLocalTopLevelDeclaration(idSig: IdSignature) { + // We only care about declarations from IC cache. They all are in the map. + val deser = publicSignatureToIcFileDeserializer[idSig] ?: return + idSig.enqueue(deser) + } + + fun deserializeDeclaration(idSig: IdSignature): IrDeclaration? { + cachedDeclaration(idSig)?.let { return it } + + val idSigIndex = reversedSignatureIndex[idSig] ?: return null + val declarationStream = icFileReader.irDeclaration(idSigIndex).codedInputStream + val declarationProto = org.jetbrains.kotlin.backend.common.serialization.proto.IrDeclaration.parseFrom(declarationStream, ExtensionRegistryLite.newInstance()) + return declarationDeserializer.deserializeDeclaration(declarationProto) + } + + // Return declaration iff it was already deserialized + private fun cachedDeclaration(idSig: IdSignature): IrDeclaration? { + val symbol = symbolDeserializer.deserializedSymbols[idSig] // Same map is used for both symbol deserializers + + if (symbol != null && symbol.isBound) return symbol.owner as? IrDeclaration + + return null + } + + fun deserializeAnyDeclaration(idSig: IdSignature): IrDeclaration? { + if (idSig is IdSignature.FileSignature) return null // TODO: is it needed + + cachedDeclaration(idSig)?.let { return it } + + // TODO fast path? + val maybeTopLevel = if (!idSig.isLocal || idSig.hasTopLevel) idSig.topLevelSignature() else idSig + + if (maybeTopLevel in originalFileDeserializer.reversedSignatureIndex.keys) { + originalFileDeserializer.deserializeFileImplicitDataIfFirstUse() + originalFileDeserializer.deserializeDeclaration(maybeTopLevel) + + // At this point the declaration should've been deserialized + return cachedDeclaration(idSig) // Will be null in case of fake overrides + } else if (maybeTopLevel in reversedSignatureIndex) { + return deserializeDeclaration(maybeTopLevel) + } + + // TODO: error? + return null + } + + fun deserializeIrSymbol(code: Long): IrSymbol { + return symbolDeserializer.deserializeIrSymbol(code) + } + + fun deserializeIrSymbol(idSig: IdSignature, symbolKind: BinarySymbolData.SymbolKind): IrSymbol { + idSig.enqueue(this) + return symbolDeserializer.deserializeIrSymbol(idSig, symbolKind) + } + + fun injectCarriers(declaration: IrDeclaration, signature: IdSignature) { + carrierDeserializer.injectCarriers(declaration, signature) + } + + companion object { + private val additionalStatementOrigins = JsStatementOrigins::class.nestedClasses.toList() + private val additionalStatementOriginIndex = + additionalStatementOrigins.mapNotNull { it.objectInstance as? IrStatementOriginImpl }.associateBy { it.debugName } + } +} + +private class FileReaderFromSerializedIrFile(val irFile: SerializedIrFile) : IrLibraryFile() { + private val declarationReader = DeclarationIrTableMemoryReader(irFile.declarations) + private val typeReader = IrArrayMemoryReader(irFile.types) + private val signatureReader = IrArrayMemoryReader(irFile.signatures) + private val stringReader = IrArrayMemoryReader(irFile.strings) + private val bodyReader = IrArrayMemoryReader(irFile.bodies) + + override fun irDeclaration(index: Int): ByteArray = declarationReader.tableItemBytes(DeclarationId(index)) + + override fun type(index: Int): ByteArray = typeReader.tableItemBytes(index) + + override fun signature(index: Int): ByteArray = signatureReader.tableItemBytes(index) + + override fun string(index: Int): ByteArray = stringReader.tableItemBytes(index) + + override fun body(index: Int): ByteArray = bodyReader.tableItemBytes(index) +} \ No newline at end of file diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcModuleDeserializer.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcModuleDeserializer.kt new file mode 100644 index 00000000000..2f52cb3d19d --- /dev/null +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcModuleDeserializer.kt @@ -0,0 +1,271 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.js.ic + +import org.jetbrains.kotlin.backend.common.serialization.* +import org.jetbrains.kotlin.backend.common.serialization.encodings.BinarySymbolData +import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.ir.backend.js.JsMapping +import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.declarations.impl.IrModuleFragmentImpl +import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrFactory +import org.jetbrains.kotlin.ir.symbols.IrFileSymbol +import org.jetbrains.kotlin.ir.symbols.IrPropertySymbol +import org.jetbrains.kotlin.ir.symbols.IrSimpleFunctionSymbol +import org.jetbrains.kotlin.ir.symbols.IrSymbol +import org.jetbrains.kotlin.ir.util.IdSignature +import org.jetbrains.kotlin.ir.util.isEffectivelyExternal +import org.jetbrains.kotlin.library.IrLibrary +import org.jetbrains.kotlin.library.impl.IrLongArrayMemoryReader +import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite + +import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile as ProtoFile + +class IcModuleDeserializer( + val irFactory: PersistentIrFactory, + val mapping: JsMapping, + val linker: JsIrLinker, + val icData: SerializedIcData, + moduleDescriptor: ModuleDescriptor, + override val klib: IrLibrary, + override val strategy: DeserializationStrategy, + private val containsErrorCode: Boolean = false, + private val useGlobalSignatures: Boolean = false, +) : IrModuleDeserializer(moduleDescriptor) { + + private val fileToDeserializerMap = mutableMapOf() + + internal val moduleReversedFileIndex = mutableMapOf() + internal val icModuleReversedFileIndex = mutableMapOf() + + override val moduleDependencies by lazy { + moduleDescriptor.allDependencyModules.filter { it != moduleDescriptor }.map { linker.resolveModuleDeserializer(it, null) } + } + + override fun fileDeserializers(): Collection { + return fileToDeserializerMap.values + } + + override fun init(delegate: IrModuleDeserializer) { + val fileCount = klib.fileCount() + + val files = ArrayList(fileCount) + + for (i in 0 until fileCount) { + val fileStream = klib.file(i).codedInputStream + val fileProto = ProtoFile.parseFrom(fileStream, ExtensionRegistryLite.newInstance()) + files.add(deserializeIrFile(fileProto, i, delegate, containsErrorCode)) + } + + moduleFragment.files.addAll(files) + + fileToDeserializerMap.values.forEach { it.symbolDeserializer.deserializeExpectActualMapping() } + } + + private fun IrSymbolDeserializer.deserializeExpectActualMapping() { + actuals.forEach { + val expectSymbol = parseSymbolData(it.expectSymbol) + val actualSymbol = parseSymbolData(it.actualSymbol) + + val expect = deserializeIdSignature(expectSymbol.signatureId) + val actual = deserializeIdSignature(actualSymbol.signatureId) + + assert(linker.expectUniqIdToActualUniqId[expect] == null) { + "Expect signature $expect is already actualized by ${linker.expectUniqIdToActualUniqId[expect]}, while we try to record $actual" + } + linker.expectUniqIdToActualUniqId[expect] = actual + // Non-null only for topLevel declarations. + findModuleDeserializerForTopLevelId(actual)?.let { md -> linker.topLevelActualUniqItToDeserializer[actual] = md } + } + } + + override fun referenceSimpleFunctionByLocalSignature(file: IrFile, idSignature: IdSignature): IrSimpleFunctionSymbol = + fileToDeserializerMap[file]?.symbolDeserializer?.referenceSimpleFunctionByLocalSignature(idSignature) + ?: error("No deserializer for file $file in module ${moduleDescriptor.name}") + + override fun referencePropertyByLocalSignature(file: IrFile, idSignature: IdSignature): IrPropertySymbol = + fileToDeserializerMap[file]?.symbolDeserializer?.referencePropertyByLocalSignature(idSignature) + ?: error("No deserializer for file $file in module ${moduleDescriptor.name}") + + // TODO: fix to topLevel checker + override fun contains(idSig: IdSignature): Boolean = idSig in moduleReversedFileIndex || idSig in icModuleReversedFileIndex + + override fun deserializeIrSymbol(idSig: IdSignature, symbolKind: BinarySymbolData.SymbolKind): IrSymbol { + assert(idSig.isPublic) + + if (idSig in icModuleReversedFileIndex) { + val icDeserializer = icModuleReversedFileIndex[idSig]!! + return icDeserializer.deserializeIrSymbol(idSig, symbolKind) + } + + val topLevelSignature = idSig.topLevelSignature() + val icDeserializer = moduleReversedFileIndex[topLevelSignature] + ?: error("No file for $topLevelSignature (@ $idSig) in module $moduleDescriptor") + + topLevelSignature.originalEnqueue(icDeserializer) + idSig.enqueue(icDeserializer) + linker.modulesWithReachableTopLevels.add(this) + + return icDeserializer.originalFileDeserializer.symbolDeserializer.deserializeIrSymbol(idSig, symbolKind).also { + linker.deserializedSymbols.add(it) + } + } + + override val moduleFragment: IrModuleFragment = IrModuleFragmentImpl(moduleDescriptor, linker.builtIns, emptyList()) + + private val pathToIcFileData = icData.files.associateBy { + it.file.path + } + + private val publicSignatureToIcFileDeserializer = mutableMapOf() + + private fun deserializeIrFile( + fileProto: ProtoFile, + fileIndex: Int, + moduleDeserializer: IrModuleDeserializer, + allowErrorNodes: Boolean + ): IrFile { + + val fileReader = IrLibraryFileFromKlib(moduleDeserializer.klib, fileIndex) + val file = fileReader.createFile(moduleFragment, fileProto) + + val icFileData = pathToIcFileData[file.path]!! + + val icDeserializer = IcFileDeserializer( + linker, + file, + fileReader, + fileProto, + strategy.needBodies, + allowErrorNodes, + strategy.inlineBodies, + moduleDeserializer, + useGlobalSignatures, + linker::handleNoModuleDeserializerFound, + { fileDeserializer -> originalEnqueue(fileDeserializer) }, + icFileData, + mapping.state, + publicSignatureToIcFileDeserializer, + { fileDeserializer -> enqueue(fileDeserializer) }, + ) + + icDeserializers += icDeserializer + + icDeserializer.explicitlyExportedToCompiler.forEach { it.topLevelSignature().originalEnqueue(icDeserializer) } + + fileToDeserializerMap[file] = icDeserializer.originalFileDeserializer + + val topLevelDeclarations = icDeserializer.originalFileDeserializer.reversedSignatureIndex.keys + topLevelDeclarations.forEach { + moduleReversedFileIndex.putIfAbsent(it, icDeserializer) // TODO Why not simple put? + } + + if (strategy.theWholeWorld) { + icDeserializer.allOriginalDeclarationSignatures().forEach { it.originalEnqueue(icDeserializer) } + } + if (strategy.theWholeWorld || strategy.explicitlyExported) { + linker.modulesWithReachableTopLevels.add(this) + } + + return file + } + + override fun addModuleReachableTopLevel(idSig: IdSignature) { + val fileLocalDeserializationState = moduleReversedFileIndex[idSig] ?: error("No file found for key $idSig") + idSig.originalEnqueue(fileLocalDeserializationState) + } + + override fun deserializeReachableDeclarations() { + while (!originalFileQueue.isEmpty()) { + originalFileQueue.removeFirst().deserializePendingSignatures() + } + } + + val originalFileQueue = ArrayDeque() + + fun IdSignature.originalEnqueue(fileDeserializer: IcFileDeserializer) { + if (fileDeserializer.enqueueForDeserialization(this)) { + originalFileQueue.addLast(fileDeserializer) + } + } + + val fileQueue = ArrayDeque() + val signatureQueue = ArrayDeque() + + val icDeserializers = mutableListOf() + val classToDeclarationSymbols = mutableMapOf>() + + fun IdSignature.enqueue(icDeserializer: IcFileDeserializer) { + if (this !in icDeserializer.visited) { + fileQueue.addLast(icDeserializer) + signatureQueue.addLast(this) + icDeserializer.visited += this + } + } + + override fun postProcess() { + icDeserializers.forEach { icDeserializer -> + if (!icDeserializer.visited.isEmpty()) { + val file = icDeserializer.originalFileDeserializer.file + icDeserializer.init() + icDeserializer.reversedSignatureIndex.keys.forEach { + if (it in icModuleReversedFileIndex) error("Duplicate signature $it in both ${icModuleReversedFileIndex[it]!!.originalFileDeserializer.file.path} and in ${file.path}") + + icModuleReversedFileIndex[it] = icDeserializer + } + } + } + + while (signatureQueue.isNotEmpty()) { + val icFileDeserializer = fileQueue.removeFirst() + val signature = signatureQueue.removeFirst() + + val declaration = icFileDeserializer.deserializeDeclaration(signature) ?: continue + + icFileDeserializer.injectCarriers(declaration, signature) + + icFileDeserializer.mappingsDeserializer(signature, declaration) + + // Make sure all members are loaded + if (declaration is IrClass) { + icFileDeserializer.loadClassOrder(signature)?.let { + classToDeclarationSymbols[declaration] = it + } + } + } + + irFactory.stageController.withStage(1000) { + + for (icDeserializer in icDeserializers) { + val fd = icDeserializer.originalFileDeserializer + val order = icDeserializer.icFileData.order + + fd.file.declarations.retainAll { it.isEffectivelyExternal() } + + IrLongArrayMemoryReader(order.topLevelSignatures).array.forEach { + val symbolData = icDeserializer.symbolDeserializer.parseSymbolData(it) + val idSig = icDeserializer.symbolDeserializer.deserializeIdSignature(symbolData.signatureId) + + // Don't create unbound symbols for top-level declarations we don't need. + if (idSig in icDeserializer.visited) { + val declaration = icDeserializer.deserializeIrSymbol(idSig, symbolData.kind).owner as IrDeclaration + fd.file.declarations += declaration + } + } + } + + for ((klass, declarations) in classToDeclarationSymbols.entries) { + irFactory.stageController.unrestrictDeclarationListsAccess { + klass.declarations.clear() + for (ds in declarations) { + klass.declarations += ds.owner as IrDeclaration + } + } + } + } + } +} \ No newline at end of file diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcSerializer.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcSerializer.kt new file mode 100644 index 00000000000..288d453d9e5 --- /dev/null +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcSerializer.kt @@ -0,0 +1,211 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.js.ic + +import org.jetbrains.kotlin.backend.common.serialization.DeclarationTable +import org.jetbrains.kotlin.backend.common.serialization.IdSignatureClashTracker +import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureSerializer +import org.jetbrains.kotlin.backend.common.serialization.signature.PublicIdSignatureComputer +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.backend.js.JsMapping +import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsGlobalDeclarationTable +import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrFileSerializer +import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrBodyBase +import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrDeclarationBase +import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrFactory +import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns +import org.jetbrains.kotlin.ir.expressions.IrExpressionBody +import org.jetbrains.kotlin.ir.serialization.serializeCarriers +import org.jetbrains.kotlin.ir.symbols.IrSymbol +import org.jetbrains.kotlin.ir.util.* +import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid +import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid +import org.jetbrains.kotlin.ir.visitors.acceptVoid +import org.jetbrains.kotlin.library.impl.IrMemoryArrayWriter +import org.jetbrains.kotlin.library.impl.IrMemoryLongArrayWriter + +class IcSerializer( + irBuiltIns: IrBuiltIns, + val mappings: JsMapping, + val irFactory: PersistentIrFactory, + val linker: JsIrLinker, + val module: IrModuleFragment +) { + + private val globalDeclarationTable = JsGlobalDeclarationTable(irBuiltIns) + + fun serializeDeclarations(moduleDeclarations: Iterable): SerializedIcData { + + // TODO serialize body carriers and new bodies as well + val moduleDeserializer = linker.moduleDeserializer(module.descriptor) + + val fileToDeserializer = moduleDeserializer.fileDeserializers().associateBy { it.file } + + val filteredDeclarations = moduleDeclarations.filter { + when { + it.fileOrNull.let { it == null || fileToDeserializer[it] == null } -> false + it is IrFakeOverrideFunction -> it.isBound + it is IrFakeOverrideProperty -> it.isBound + else -> (it.parent as? IrFakeOverrideFunction)?.isBound ?: (it.parent as? IrFakeOverrideProperty)?.isBound ?: true + } + } + + val filteredBodies = irFactory.allBodies.groupBy { + (it as? PersistentIrBodyBase<*>)?.let { + if (it.hasContainer) { + it.container.fileOrNull + } else null + } + } + + val dataToSerialize = filteredDeclarations.groupBy { + // TODO don't move declarations or effects outside the original file + // TODO Or invent a different mechanism for that + + it.file + } + + val icData = mutableListOf() + + for (file in fileToDeserializer.keys) { + val fileDeclarations = dataToSerialize[file] ?: emptyList() + val bodies = filteredBodies[file] ?: emptyList() + + val fileDeserializer = fileToDeserializer[file]!! + + val symbolToSignature = fileDeserializer.symbolDeserializer.deserializedSymbols.entries.associate { (idSig, symbol) -> symbol to idSig }.toMutableMap() + + val icDeclarationTable = IcDeclarationTable(globalDeclarationTable, irFactory, 1000000, 1000000, symbolToSignature) + val fileSerializer = JsIrFileSerializer( + linker.messageLogger, + icDeclarationTable, + mutableMapOf(), + skipExpects = true, + icMode = true, + allowNullTypes = true, + allowErrorStatementOrigins = true + ) + + bodies.forEach { body -> + if (body is IrExpressionBody) { + fileSerializer.serializeIrExpressionBody(body.expression) + } else { + fileSerializer.serializeIrStatementBody(body) + } + } + + // Only save newly created declarations + val newDeclarations = fileDeclarations.filter { d -> + d is PersistentIrDeclarationBase<*> && (d.createdOn > 0 || /*d.isFakeOverride ||*/ (d is IrValueParameter || d is IrTypeParameter) && (d.parent as IrDeclaration).isFakeOverride) + } + + val serializedCarriers = fileSerializer.serializeCarriers( + fileDeclarations, + bodies, + ) { declaration -> + icDeclarationTable.signatureByDeclaration(declaration) + } + + val serializedMappings = mappings.state.serializeMappings(fileDeclarations) { symbol -> + fileSerializer.serializeIrSymbol(symbol) + } + + val order = storeOrder(file) { symbol -> + fileSerializer.serializeIrSymbol(symbol) + } + + val serializedIrFile = fileSerializer.serializeDeclarationsForIC(file, newDeclarations) + + icData += SerializedIcDataForFile( + serializedIrFile, + serializedCarriers, + serializedMappings, + order, + ) + } + + return SerializedIcData(icData) + } + + // Returns precomputed signatures for the newly created declarations. Delegates to the default table otherwise. + class IcDeclarationTable( + globalDeclarationTable: JsGlobalDeclarationTable, + val irFactory: PersistentIrFactory, + newLocalIndex: Long, + newScopeIndex: Int, + val existingMappings: MutableMap + ) : DeclarationTable(globalDeclarationTable) { + + override val signaturer: IdSignatureSerializer = IdSignatureSerializerWithForIC(globalDeclarationTable.publicIdSignatureComputer, this, newLocalIndex, newScopeIndex) + + override fun signatureByDeclaration(declaration: IrDeclaration): IdSignature { + return existingMappings.getOrPut(declaration.symbol) { + irFactory.declarationSignature(declaration) ?: super.signatureByDeclaration(declaration) + } + } + } +} + +class IdSignatureSerializerWithForIC( + publicSignatureBuilder: PublicIdSignatureComputer, + table: DeclarationTable, + localIndexOffset: Long = 0, + scopeIndexOffset: Int = 0, +) : IdSignatureSerializer( + publicSignatureBuilder, + table +) { + init { + localIndex = localIndexOffset + scopeIndex = scopeIndexOffset + } + + override fun IrDeclaration.createFileLocalSignature(parentSignature: IdSignature, localIndex: Long): IdSignature { + if (this is IrTypeParameter) { + return IdSignature.GlobalFileLocalSignature(parentSignature, 1000_000_000_000L + index, fileOrNull?.path ?: "") + } + return IdSignature.GlobalFileLocalSignature(parentSignature, localIndex, fileOrNull?.path ?: "") + } + + override fun IrDeclaration.createScopeLocalSignature(scopeIndex: Int, description: String): IdSignature { + return IdSignature.GlobalScopeLocalDeclaration(scopeIndex, description, fileOrNull?.path ?: "") + } +} + +fun storeOrder(file: IrFile, idSigToLong: (IrSymbol) -> Long): SerializedOrder { + val topLevelSignatures = mutableListOf() + val containerSignatures = mutableListOf() + val declarationSignatures = mutableListOf() + + fun IrDeclaration.idSigIndex(): Long = idSigToLong(symbol) + + file.declarations.forEach { d -> + topLevelSignatures += d.idSigIndex() + d.acceptVoid(object : IrElementVisitorVoid { + override fun visitElement(element: IrElement) { + element.acceptChildrenVoid(this) + } + + override fun visitClass(declaration: IrClass) { + // First element is the container signature + containerSignatures += declaration.idSigIndex() + val declarationIds = declaration.declarations.map { it.idSigIndex() } + + declarationSignatures += IrMemoryLongArrayWriter(declarationIds).writeIntoMemory() + + super.visitClass(declaration) + } + }) + } + + return SerializedOrder( + IrMemoryLongArrayWriter(topLevelSignatures).writeIntoMemory(), + IrMemoryLongArrayWriter(containerSignatures).writeIntoMemory(), + IrMemoryArrayWriter(declarationSignatures).writeIntoMemory(), + ) +} \ No newline at end of file diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcSymbolTable.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcSymbolTable.kt new file mode 100644 index 00000000000..c8cea73a674 --- /dev/null +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcSymbolTable.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.js.ic + +import org.jetbrains.kotlin.ir.declarations.IrFactory +import org.jetbrains.kotlin.ir.declarations.IrTypeParameter +import org.jetbrains.kotlin.ir.symbols.IrFieldSymbol +import org.jetbrains.kotlin.ir.symbols.IrTypeParameterSymbol +import org.jetbrains.kotlin.ir.symbols.impl.IrFieldPublicSymbolImpl +import org.jetbrains.kotlin.ir.symbols.impl.IrTypeParameterPublicSymbolImpl +import org.jetbrains.kotlin.ir.util.IdSignature +import org.jetbrains.kotlin.ir.util.IdSignatureComposer +import org.jetbrains.kotlin.ir.util.NameProvider +import org.jetbrains.kotlin.ir.util.SymbolTable + +class IcSymbolTable( + signaturer: IdSignatureComposer, + irFactory: IrFactory, + nameProvider: NameProvider = NameProvider.DEFAULT, +) : SymbolTable( + signaturer, + irFactory, + nameProvider, +) { + override fun referenceFieldFromLinker(sig: IdSignature): IrFieldSymbol = + fieldSymbolTable.run { + fieldSymbolTable.referenced(sig) { IrFieldPublicSymbolImpl(sig) } + } + + override fun declareGlobalTypeParameter( + sig: IdSignature, + symbolFactory: () -> IrTypeParameterSymbol, + typeParameterFactory: (IrTypeParameterSymbol) -> IrTypeParameter + ): IrTypeParameter { + return globalTypeParameterSymbolTable.declare(sig, symbolFactory, typeParameterFactory) + } + + override fun referenceTypeParameterFromLinker(sig: IdSignature): IrTypeParameterSymbol { + return scopedTypeParameterSymbolTable.get(sig) ?: globalTypeParameterSymbolTable.referenced(sig) { + IrTypeParameterPublicSymbolImpl(sig) + } + } +} \ No newline at end of file diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/data.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/data.kt new file mode 100644 index 00000000000..fa861e2d9b0 --- /dev/null +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/data.kt @@ -0,0 +1,111 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.js.ic + +import org.jetbrains.kotlin.ir.backend.js.SerializedMapping +import org.jetbrains.kotlin.ir.backend.js.SerializedMappings +import org.jetbrains.kotlin.ir.serialization.SerializedCarriers +import org.jetbrains.kotlin.library.SerializedIrFile +import org.jetbrains.kotlin.library.impl.IrArrayMemoryReader +import org.jetbrains.kotlin.library.impl.IrMemoryArrayWriter +import org.jetbrains.kotlin.library.impl.toArray +import java.io.File +import java.nio.charset.Charset + +class SerializedIcData( + val files: Collection, +) + +class SerializedIcDataForFile( + val file: SerializedIrFile, + val carriers: SerializedCarriers, + val mappings: SerializedMappings, + val order: SerializedOrder, +) + +class SerializedOrder( + val topLevelSignatures: ByteArray, + val containerSignatures: ByteArray, + val declarationSignatures: ByteArray, +) + +fun SerializedIcData.writeTo(dir: File) { + if (!dir.exists()) error("Directory doesn't exist: ${dir.absolutePath}") + if (!dir.isDirectory) error("Not a directory: ${dir.absolutePath}") + + files.forEach { + val fqnPath = it.file.fqName + val fileId = it.file.path.hashCode().toString(Character.MAX_RADIX) + val irFileDirectory = "ic-$fqnPath.$fileId.file" + val fileDir = File(dir, irFileDirectory) + + // TODO: just rewrite? + if (!fileDir.exists()) { + if (!fileDir.mkdirs()) error("Failed to create output dir for file ${fileDir.absolutePath}") + } + + // .file + File(fileDir, "file.fileData").writeBytes(it.file.fileData) + File(fileDir, "file.path").writeBytes(it.file.path.toByteArray(Charsets.UTF_8)) + File(fileDir, "file.declarations").writeBytes(it.file.declarations) + File(fileDir, "file.types").writeBytes(it.file.types) + File(fileDir, "file.signatures").writeBytes(it.file.signatures) + File(fileDir, "file.strings").writeBytes(it.file.strings) + File(fileDir, "file.bodies").writeBytes(it.file.bodies) + // .carriers + File(fileDir, "carriers.signatures").writeBytes(it.carriers.signatures) + File(fileDir, "carriers.declarationCarriers").writeBytes(it.carriers.declarationCarriers) + File(fileDir, "carriers.bodyCarriers").writeBytes(it.carriers.bodyCarriers) + File(fileDir, "carriers.removedOn").writeBytes(it.carriers.removedOn) + // .mappings + File(fileDir, "mappings.keys").writeBytes(it.mappings.keyBytes()) + File(fileDir, "mappings.values").writeBytes(it.mappings.valueBytes()) + // .order + File(fileDir, "order.topLevelSignatures").writeBytes(it.order.topLevelSignatures) + File(fileDir, "order.containerSignatures").writeBytes(it.order.containerSignatures) + File(fileDir, "order.declarationSignatures").writeBytes(it.order.declarationSignatures) + } +} + +private fun SerializedMappings.keyBytes() = IrMemoryArrayWriter(mappings.map { it.keys }).writeIntoMemory() +private fun SerializedMappings.valueBytes() = IrMemoryArrayWriter(mappings.map { it.values }).writeIntoMemory() + +fun File.readIcData(): SerializedIcData { + if (!this.isDirectory) error("Directory doesn't exist: ${this.absolutePath}") + + return SerializedIcData(this.listFiles()!!.filter { it.isDirectory}.map { fileDir -> + val file = SerializedIrFile( + fileData = File(fileDir, "file.fileData").readBytes(), + fqName = fileDir.name.split('.').dropLast(2).joinToString(separator = "."), + path = File(fileDir, "file.path").readBytes().toString(Charsets.UTF_8), + types = File(fileDir, "file.types").readBytes(), + signatures = File(fileDir, "file.signatures").readBytes(), + strings = File(fileDir, "file.strings").readBytes(), + bodies = File(fileDir, "file.bodies").readBytes(), + declarations = File(fileDir, "file.declarations").readBytes() + ) + + val carriers = SerializedCarriers( + signatures = File(fileDir, "carriers.signatures").readBytes(), + declarationCarriers = File(fileDir, "carriers.declarationCarriers").readBytes(), + bodyCarriers = File(fileDir, "carriers.bodyCarriers").readBytes(), + removedOn = File(fileDir, "carriers.removedOn").readBytes(), + ) + + val mappingKeys = IrArrayMemoryReader(File(fileDir, "mappings.keys").readBytes()).toArray() + val mappingValues = IrArrayMemoryReader(File(fileDir, "mappings.values").readBytes()).toArray() + assert(mappingKeys.size == mappingValues.size) + val mappings = SerializedMappings(mappingKeys.zip(mappingValues).map { (k, v) -> SerializedMapping(k, v) }) + + val order = SerializedOrder( + topLevelSignatures = File(fileDir, "order.topLevelSignatures").readBytes(), + containerSignatures = File(fileDir, "order.containerSignatures").readBytes(), + declarationSignatures = File(fileDir, "order.declarationSignatures").readBytes(), + ) + + SerializedIcDataForFile(file, carriers, mappings, order) + }) +} \ No newline at end of file diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt index 3cba9f27d05..2f768f727d8 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt @@ -30,6 +30,8 @@ import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.ir.IrBuiltIns import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI +import org.jetbrains.kotlin.ir.backend.js.ic.IcSymbolTable +import org.jetbrains.kotlin.ir.backend.js.ic.SerializedIcData import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrModuleSerializer import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerDesc @@ -167,9 +169,9 @@ fun generateKLib( } val depsDescriptors = - ModulesStructure(project, MainModule.SourceFiles(files), analyzer, configuration, dependencies, friendDependencies) + ModulesStructure(project, MainModule.SourceFiles(files), analyzer, configuration, dependencies, friendDependencies, EmptyLoweringsCacheProvider) val allDependencies = depsDescriptors.allDependencies - val (psi2IrContext, hasErrors) = runAnalysisAndPreparePsi2Ir(depsDescriptors, irFactory, errorPolicy) + val (psi2IrContext, hasErrors) = runAnalysisAndPreparePsi2Ir(depsDescriptors, SymbolTable(IdSignatureDescriptor(JsManglerDesc), irFactory, errorPolicy) val irBuiltIns = psi2IrContext.irBuiltIns val expectDescriptorToSymbol = mutableMapOf() @@ -232,6 +234,7 @@ data class IrModuleInfo( val symbolTable: SymbolTable, val deserializer: JsIrLinker, val moduleFragmentToUniqueName: Map, + val loweredIrLoaded: Set = emptySet(), ) private fun sortDependencies(resolvedDependencies: List, mapping: Map): Collection { @@ -244,7 +247,14 @@ private fun sortDependencies(resolvedDependencies: List, }.reversed() } -@OptIn(ObsoleteDescriptorBasedAPI::class) +interface LoweringsCacheProvider { + fun cacheByPath(path: String): SerializedIcData? +} + +object EmptyLoweringsCacheProvider : LoweringsCacheProvider { + override fun cacheByPath(path: String): SerializedIcData? = null +} + fun loadIr( project: Project, mainModule: MainModule, @@ -253,16 +263,20 @@ fun loadIr( dependencies: Collection, friendDependencies: Collection, irFactory: IrFactory, - verifySignatures: Boolean + verifySignatures: Boolean, + loweringsCacheProvider: LoweringsCacheProvider? = null ): IrModuleInfo { - val depsDescriptors = ModulesStructure(project, mainModule, analyzer, configuration, dependencies, friendDependencies) + val depsDescriptors = ModulesStructure(project, mainModule, analyzer, configuration, dependencies, friendDependencies, loweringsCacheProvider ?: EmptyLoweringsCacheProvider) val errorPolicy = configuration.get(JSConfigurationKeys.ERROR_TOLERANCE_POLICY) ?: ErrorTolerancePolicy.DEFAULT val messageLogger = configuration.get(IrMessageLogger.IR_MESSAGE_LOGGER) ?: IrMessageLogger.None val allDependencies = depsDescriptors.allDependencies + val signaturer = IdSignatureDescriptor(JsManglerDesc) + val symbolTable = if (loweringsCacheProvider == null) SymbolTable(signaturer, irFactory) else IcSymbolTable(signaturer, irFactory) + when (mainModule) { is MainModule.SourceFiles -> { - val (psi2IrContext, _) = runAnalysisAndPreparePsi2Ir(depsDescriptors, irFactory, errorPolicy) + val (psi2IrContext, _) = runAnalysisAndPreparePsi2Ir(depsDescriptors, errorPolicy, symbolTable) val irBuiltIns = psi2IrContext.irBuiltIns val symbolTable = psi2IrContext.symbolTable val feContext = psi2IrContext.run { @@ -271,6 +285,16 @@ fun loadIr( val moduleFragmentToUniqueName = mutableMapOf() val irLinker = JsIrLinker(psi2IrContext.moduleDescriptor, messageLogger, irBuiltIns, symbolTable, feContext, null) + JsIrLinker( + psi2IrContext.moduleDescriptor, + messageLogger, + irBuiltIns, + symbolTable, + feContext, + null, + depsDescriptors.loweredIcData, + loweringsCacheProvider != null + ) val deserializedModuleFragments = sortDependencies(allDependencies, depsDescriptors.descriptors).map { klib -> irLinker.deserializeIrModuleHeader( depsDescriptors.getModuleDescriptor(klib), @@ -301,7 +325,8 @@ fun loadIr( (irBuiltIns as IrBuiltInsOverDescriptors).knownBuiltins.forEach { it.acceptVoid(mangleChecker) } } - return IrModuleInfo(moduleFragment, deserializedModuleFragments, irBuiltIns, symbolTable, irLinker, moduleFragmentToUniqueName) + return IrModuleInfo(moduleFragment, deserializedModuleFragments, irBuiltIns, symbolTable, irLinker, moduleFragmentToUniqueName, + depsDescriptors.modulesWithCaches(deserializedModuleFragments)) } is MainModule.Klib -> { val mainModuleLib = depsDescriptors.allDependencies.find { it.library.libraryFile.canonicalPath == mainModule.libPath }?.library @@ -313,8 +338,32 @@ fun loadIr( val typeTranslator = TypeTranslatorImpl(symbolTable, depsDescriptors.compilerConfiguration.languageVersionSettings, moduleDescriptor) val irBuiltIns = IrBuiltInsOverDescriptors(moduleDescriptor.builtIns, typeTranslator, symbolTable) + + val loweredIcData = if (loweringsCacheProvider == null) emptyMap() else { + val result = mutableMapOf() + + for (lib in depsDescriptors.moduleDependencies.keys) { + val path = lib.libraryFile.absolutePath + val icData = loweringsCacheProvider.cacheByPath(path) + if (icData != null) { + result[depsDescriptors.getModuleDescriptor(lib)] = icData + } + } + + result + } + val irLinker = - JsIrLinker(null, messageLogger, irBuiltIns, symbolTable, null, null) + JsIrLinker( + null, + messageLogger, + irBuiltIns, + symbolTable, + null, + null, + loweredIcData, + loweringsCacheProvider != null + ) val moduleFragmentToUniqueName = mutableMapOf() @@ -338,22 +387,22 @@ fun loadIr( ExternalDependenciesGenerator(symbolTable, listOf(irLinker)).generateUnboundSymbolsAsDependencies() irLinker.postProcess() - return IrModuleInfo(moduleFragment, deserializedModuleFragments, irBuiltIns, symbolTable, irLinker, moduleFragmentToUniqueName) + return IrModuleInfo(moduleFragment, deserializedModuleFragments, irBuiltIns, symbolTable, irLinker, moduleFragmentToUniqueName, + depsDescriptors.modulesWithCaches(deserializedModuleFragments)) } } } private fun runAnalysisAndPreparePsi2Ir( depsDescriptors: ModulesStructure, - irFactory: IrFactory, - errorIgnorancePolicy: ErrorTolerancePolicy + errorIgnorancePolicy: ErrorTolerancePolicy, + symbolTable: SymbolTable, ): Pair { val analysisResult = depsDescriptors.runAnalysis(errorIgnorancePolicy) val psi2Ir = Psi2IrTranslator( depsDescriptors.compilerConfiguration.languageVersionSettings, Psi2IrConfiguration(errorIgnorancePolicy.allowErrors) ) - val symbolTable = SymbolTable(IdSignatureDescriptor(JsManglerDesc), irFactory) return psi2Ir.createGeneratorContext( analysisResult.moduleDescriptor, analysisResult.bindingContext, @@ -426,7 +475,8 @@ private class ModulesStructure( private val analyzer: AbstractAnalyzerWithCompilerReport, val compilerConfiguration: CompilerConfiguration, dependencies: Collection, - friendDependenciesPaths: Collection + friendDependenciesPaths: Collection, + private val loweringsCacheProvider: LoweringsCacheProvider ) { val allDependencies = jsResolveLibraries( dependencies, @@ -497,6 +547,8 @@ private class ModulesStructure( // TODO: these are roughly equivalent to KlibResolvedModuleDescriptorsFactoryImpl. Refactor me. val descriptors = mutableMapOf() + val loweredIcData = mutableMapOf() + fun getModuleDescriptor(current: KotlinLibrary): ModuleDescriptorImpl = descriptors.getOrPut(current) { val isBuiltIns = current.unresolvedDependencies.isEmpty() @@ -513,9 +565,18 @@ private class ModulesStructure( val dependencies = moduleDependencies.getValue(current).map { getModuleDescriptor(it) } md.setDependencies(listOf(md) + dependencies) + + loweringsCacheProvider.cacheByPath(current.libraryFile.absolutePath)?.let { icData -> + loweredIcData[md] = icData + } + md } + fun modulesWithCaches(allModules: Iterable): Set { + return allModules.filter { it.descriptor in loweredIcData }.toSet() + } + val builtInModuleDescriptor = if (builtInsDep != null) getModuleDescriptor(builtInsDep.library) diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/IrIcModuleDeserializerWithBuiltIns.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/IrIcModuleDeserializerWithBuiltIns.kt new file mode 100644 index 00000000000..3ecbbd35ed8 --- /dev/null +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/IrIcModuleDeserializerWithBuiltIns.kt @@ -0,0 +1,66 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir + +import org.jetbrains.kotlin.backend.common.serialization.IrModuleDeserializer +import org.jetbrains.kotlin.backend.common.serialization.IrModuleDeserializerWithBuiltIns +import org.jetbrains.kotlin.backend.common.serialization.encodings.BinarySymbolData +import org.jetbrains.kotlin.backend.common.serialization.knownBuiltins +import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction +import org.jetbrains.kotlin.ir.declarations.IrSymbolOwner +import org.jetbrains.kotlin.ir.declarations.IrTypeParametersContainer +import org.jetbrains.kotlin.ir.descriptors.IrAbstractFunctionFactory +import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns +import org.jetbrains.kotlin.ir.symbols.IrSymbol +import org.jetbrains.kotlin.ir.util.IdSignature + +class IrIcModuleDeserializerWithBuiltIns( + builtIns: IrBuiltIns, + functionFactory: IrAbstractFunctionFactory, + delegate: IrModuleDeserializer, +) : IrModuleDeserializerWithBuiltIns(builtIns, functionFactory, delegate) { + + override fun additionalBuiltIns(builtIns: IrBuiltIns): Map { + val result = mutableMapOf() + + builtIns.knownBuiltins.forEach { + val symbol = (it as IrSymbolOwner).symbol + val declaration = symbol.owner + if (declaration is IrSimpleFunction) { + declaration.typeParameters.forEachIndexed { i, tp -> + result[IdSignature.GlobalFileLocalSignature(symbol.signature!!, 1000_000_000_000L + i, "")] = tp.symbol + } + } + } + + return result + } + + override fun checkIsFunctionInterface(idSig: IdSignature): Boolean { + if (idSig is IdSignature.GlobalFileLocalSignature) return checkIsFunctionInterface(idSig.container) + return super.checkIsFunctionInterface(idSig) + } + + override fun contains(idSig: IdSignature): Boolean { + return super.contains(idSig) || idSig is IdSignature.GlobalFileLocalSignature && checkIsFunctionInterface(idSig.container) + } + + override fun resolveFunctionalInterface(idSig: IdSignature, symbolKind: BinarySymbolData.SymbolKind): IrSymbol { + if (idSig is IdSignature.GlobalFileLocalSignature) { + val containerSymbolKind = when (idSig.container.asPublic()!!.nameSegments.size) { + 1 -> BinarySymbolData.SymbolKind.CLASS_SYMBOL + 3 -> BinarySymbolData.SymbolKind.FUNCTION_SYMBOL + else -> error("Cannot infer symbolKind") + } + + val declaration = resolveFunctionalInterface(idSig.container, containerSymbolKind).owner as IrTypeParametersContainer + + return declaration.typeParameters[(idSig.id - 1000_000_000_000L).toInt()].symbol + } + + return super.resolveFunctionalInterface(idSig, symbolKind) + } +} \ No newline at end of file diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsDeclarationTable.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsDeclarationTable.kt index e6338eb3a78..d63e74f3783 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsDeclarationTable.kt +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsDeclarationTable.kt @@ -14,7 +14,7 @@ import org.jetbrains.kotlin.ir.declarations.IrTypeParameter import org.jetbrains.kotlin.ir.util.IdSignature import org.jetbrains.kotlin.ir.util.render -class JsUniqIdClashTracker : IdSignatureClashTracker { +class JsUniqIdClashTracker() : IdSignatureClashTracker { private val committedIdSignatures = mutableMapOf() override fun commit(declaration: IrDeclaration, signature: IdSignature) { @@ -34,8 +34,8 @@ class JsUniqIdClashTracker : IdSignatureClashTracker { } } -class JsGlobalDeclarationTable(builtIns: IrBuiltIns) : - GlobalDeclarationTable(JsManglerIr, JsUniqIdClashTracker()) { +class JsGlobalDeclarationTable(builtIns: IrBuiltIns, tracker: IdSignatureClashTracker = JsUniqIdClashTracker()) : + GlobalDeclarationTable(JsManglerIr, tracker) { init { loadKnownBuiltins(builtIns) } diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsIrFileSerializer.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsIrFileSerializer.kt index 158bcb0683e..07cbf934cab 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsIrFileSerializer.kt +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsIrFileSerializer.kt @@ -24,6 +24,8 @@ class JsIrFileSerializer( skipExpects: Boolean, bodiesOnlyForInlines: Boolean = false, icMode: Boolean = false, + allowNullTypes: Boolean = false, + allowErrorStatementOrigins: Boolean = false, ) : IrFileSerializer( messageLogger, declarationTable, @@ -32,6 +34,8 @@ class JsIrFileSerializer( bodiesOnlyForInlines = bodiesOnlyForInlines, skipExpects = skipExpects, skipMutableState = icMode, + allowNullTypes = allowNullTypes, + allowErrorStatementOrigins = allowErrorStatementOrigins, ) { companion object { private val JS_EXPORT_FQN = FqName("kotlin.js.JsExport") diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsIrLinker.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsIrLinker.kt index 3cdeeba92d1..463b9da3fe7 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsIrLinker.kt +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsIrLinker.kt @@ -7,10 +7,18 @@ package org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir import org.jetbrains.kotlin.backend.common.overrides.FakeOverrideBuilder import org.jetbrains.kotlin.backend.common.serialization.* +import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureSerializer import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.ir.backend.js.JsMapping +import org.jetbrains.kotlin.ir.backend.js.ic.IcModuleDeserializer +import org.jetbrains.kotlin.ir.backend.js.ic.IdSignatureSerializerWithForIC +import org.jetbrains.kotlin.ir.backend.js.ic.SerializedIcData import org.jetbrains.kotlin.ir.IrBuiltIns import org.jetbrains.kotlin.ir.builders.TranslationPluginContext import org.jetbrains.kotlin.ir.declarations.IrModuleFragment +import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrFactory +import org.jetbrains.kotlin.ir.descriptors.IrAbstractFunctionFactory +import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns import org.jetbrains.kotlin.ir.types.IrTypeSystemContextImpl import org.jetbrains.kotlin.ir.util.IrMessageLogger import org.jetbrains.kotlin.ir.util.ReferenceSymbolTable @@ -24,10 +32,15 @@ import org.jetbrains.kotlin.library.containsErrorCode class JsIrLinker( private val currentModule: ModuleDescriptor?, messageLogger: IrMessageLogger, builtIns: IrBuiltIns, symbolTable: SymbolTable, override val translationPluginContext: TranslationPluginContext?, - private val icData: ICData? = null + private val icData: ICData? = null, + private val loweredIcData: Map = emptyMap(), + private val useGlobalSignatures: Boolean = false, ) : KotlinIrLinker(currentModule, messageLogger, builtIns, symbolTable, emptyList()) { - override val fakeOverrideBuilder = FakeOverrideBuilder(this, symbolTable, JsManglerIr, IrTypeSystemContextImpl(builtIns)) + override val fakeOverrideBuilder = FakeOverrideBuilder(this, symbolTable, JsManglerIr, IrTypeSystemContextImpl(builtIns), + signatureSerializerFactory = { publicSignatureBuilder, table -> + if (useGlobalSignatures) IdSignatureSerializerWithForIC(publicSignatureBuilder, table) else IdSignatureSerializer(publicSignatureBuilder, table) + }) override fun isBuiltInModule(moduleDescriptor: ModuleDescriptor): Boolean = moduleDescriptor === moduleDescriptor.builtIns.builtInsModule @@ -35,13 +48,43 @@ class JsIrLinker( private val IrLibrary.libContainsErrorCode: Boolean get() = this is KotlinLibrary && this.containsErrorCode - override fun createModuleDeserializer(moduleDescriptor: ModuleDescriptor, klib: KotlinLibrary?, strategy: DeserializationStrategy): IrModuleDeserializer = klib?.let { lib -> - JsModuleDeserializer(moduleDescriptor, lib, strategy, lib.versions.abiVersion ?: KotlinAbiVersion.CURRENT, lib.libContainsErrorCode) - } ?: error("Expecting kotlin library") + override fun createModuleDeserializer(moduleDescriptor: ModuleDescriptor, klib: IrLibrary?, strategy: DeserializationStrategy): IrModuleDeserializer { + require(klib != null) { "Expecting kotlin library" } + loweredIcData[moduleDescriptor]?.let { loweredIcData -> + return IcModuleDeserializer( + symbolTable.irFactory as PersistentIrFactory, + mapping, + this, + loweredIcData, + moduleDescriptor, + klib, + strategy, + containsErrorCode = klib.libContainsErrorCode, + useGlobalSignatures = useGlobalSignatures + ) + } + return klib?.let { lib -> + JsModuleDeserializer(moduleDescriptor, lib, strategy, lib.versions.abiVersion ?: KotlinAbiVersion.CURRENT, lib.libContainsErrorCode) + } ?: error("Expecting kotlin library") + } + val mapping: JsMapping by lazy { JsMapping(symbolTable.irFactory) } - private inner class JsModuleDeserializer(moduleDescriptor: ModuleDescriptor, klib: IrLibrary, strategy: DeserializationStrategy, libraryAbiVersion: KotlinAbiVersion, allowErrorCode: Boolean) : - BasicIrModuleDeserializer(this, moduleDescriptor, klib, strategy, libraryAbiVersion, allowErrorCode) + private inner class JsModuleDeserializer(moduleDescriptor: ModuleDescriptor, klib: IrLibrary, strategy: DeserializationStrategy, allowErrorCode: Boolean) : + BasicIrModuleDeserializer(this, moduleDescriptor, klib, strategy, libraryAbiVersion, allowErrorCode, useGlobalSignatures) + + override fun maybeWrapWithBuiltInAndInit( + moduleDescriptor: ModuleDescriptor, + moduleDeserializer: IrModuleDeserializer + ): IrModuleDeserializer { + return if (isBuiltInModule(moduleDescriptor)) { + if (useGlobalSignatures) { + IrIcModuleDeserializerWithBuiltIns(builtIns, functionalInterfaceFactory, moduleDeserializer) + } else { + IrModuleDeserializerWithBuiltIns(builtIns, functionalInterfaceFactory, moduleDeserializer) + } + } else moduleDeserializer + } override fun createCurrentModuleDeserializer(moduleFragment: IrModuleFragment, dependencies: Collection): IrModuleDeserializer { val currentModuleDeserializer = super.createCurrentModuleDeserializer(moduleFragment, dependencies) @@ -58,6 +101,20 @@ class JsIrLinker( .map { it.moduleFragment } .filter { it.descriptor !== currentModule } + + fun moduleDeserializer(moduleDescriptor: ModuleDescriptor): IrModuleDeserializer { + return deserializersForModules[moduleDescriptor] ?: error("Deserializer for $moduleDescriptor not found") + } + + fun loadIcIr(preprocess: (IrModuleFragment) -> Unit) { + deserializersForModules.values.forEach { + if (it.moduleDescriptor in loweredIcData) { + preprocess(it.moduleFragment) + } + it.postProcess() + } + } + class JsFePluginContext( override val moduleDescriptor: ModuleDescriptor, override val symbolTable: ReferenceSymbolTable, diff --git a/js/js.tests/test/org/jetbrains/kotlin/js/test/BasicIrBoxTest.kt b/js/js.tests/test/org/jetbrains/kotlin/js/test/BasicIrBoxTest.kt index a358fc872d2..5105d5e5941 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/js/test/BasicIrBoxTest.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/js/test/BasicIrBoxTest.kt @@ -10,6 +10,8 @@ import org.jetbrains.kotlin.backend.common.phaser.PhaseConfig import org.jetbrains.kotlin.backend.common.phaser.toPhaseMap import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport import org.jetbrains.kotlin.ir.backend.js.* +import org.jetbrains.kotlin.ir.backend.js.ic.SerializedIcData +import org.jetbrains.kotlin.ir.backend.js.ic.prepareSingleLibraryIcCache import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrFactory import org.jetbrains.kotlin.js.config.JSConfigurationKeys @@ -29,6 +31,13 @@ private val fullRuntimeKlib: String = System.getProperty("kotlin.js.full.stdlib. private val defaultRuntimeKlib = System.getProperty("kotlin.js.reduced.stdlib.path") private val kotlinTestKLib = System.getProperty("kotlin.js.kotlin.test.path") +// TODO Cache on FS (requires bootstrap) +private val predefinedKlibHasIcCache = mutableMapOf( + File(fullRuntimeKlib).absolutePath to null, + File(kotlinTestKLib).absolutePath to null, + File(defaultRuntimeKlib).absolutePath to null +) + abstract class BasicIrBoxTest( pathToTestDir: String, testGroupOutputDirPrefix: String, @@ -49,11 +58,15 @@ abstract class BasicIrBoxTest( private fun getBoolean(s: String, default: Boolean) = System.getProperty(s)?.let { parseBoolean(it) } ?: default - private val lowerPerModule: Boolean = getBoolean("kotlin.js.ir.lowerPerModule") + private val runIcMode: Boolean = getBoolean("kotlin.js.ir.icMode") + + private val lowerPerModule: Boolean = runIcMode || getBoolean("kotlin.js.ir.lowerPerModule") + + private val klibMainModule: Boolean = false || getBoolean("kotlin.js.ir.klibMainModule") override val skipRegularMode: Boolean = getBoolean("kotlin.js.ir.skipRegularMode") - override val runIrDce: Boolean = !lowerPerModule && getBoolean("kotlin.js.ir.dce", true) + override val runIrDce: Boolean = getBoolean("kotlin.js.ir.dce", true) override val runIrPir: Boolean = !lowerPerModule && getBoolean("kotlin.js.ir.pir", true) @@ -108,10 +121,31 @@ abstract class BasicIrBoxTest( val allKlibPaths = (runtimeKlibs + transitiveLibraries.map { compilationCache[it] ?: error("Can't find compiled module for dependency $it") - }).map { File(it).absolutePath } + }).map { File(it).absolutePath }.toMutableList() + + val klibPath = outputFile.absolutePath.replace("_v5.js", "/") + + if (isMainModule && klibMainModule) { + val resolvedLibraries = jsResolveLibraries(allKlibPaths, emptyList(), messageCollectorLogger(MessageCollector.NONE)) + + generateKLib( + project = config.project, + files = filesToCompile, + analyzer = AnalyzerWithCompilerReport(config.configuration), + configuration = config.configuration, + allDependencies = resolvedLibraries, + friendDependencies = emptyList(), + irFactory = IrFactoryImpl, + outputKlibPath = klibPath, + nopack = true, + null + ) + + allKlibPaths += File(klibPath).absolutePath + } val actualOutputFile = outputFile.absolutePath.let { - if (!isMainModule) it.replace("_v5.js", "/") else it + if (!isMainModule) klibPath else it } if (isMainModule) { @@ -134,11 +168,42 @@ abstract class BasicIrBoxTest( PhaseConfig(jsPhases) } + val mainModule = if (!klibMainModule) { + MainModule.SourceFiles(filesToCompile) + } else { + val mainLib = resolvedLibraries.getFullList().find { it.libraryFile.absolutePath == File(klibPath).absolutePath }!! + MainModule.Klib(mainLib) + } + if (!skipRegularMode) { + val icCache: Map = if (!runIcMode) emptyMap() else { + val map = mutableMapOf() + for (klibPath in allKlibPaths) { + val icData = predefinedKlibHasIcCache[klibPath] ?: prepareSingleLibraryIcCache( + project = project, + analyzer = AnalyzerWithCompilerReport(config.configuration), + configuration = config.configuration, + library = resolvedLibraries.getFullList() + .single { it.libraryFile.absolutePath == File(klibPath).absolutePath }, + dependencies = resolvedLibraries.filterRoots { it.library.libraryFile.absolutePath == File(klibPath).absolutePath }, + icCache = map + ) + + if (klibPath in predefinedKlibHasIcCache) { + predefinedKlibHasIcCache[klibPath] = icData + } + + map[klibPath] = icData + } + + map + } + + val irFactory = if (lowerPerModule) PersistentIrFactory() else IrFactoryImpl val compiledModule = compile( project = config.project, - mainModule = MainModule.SourceFiles(filesToCompile), + mainModule = mainModule, analyzer = AnalyzerWithCompilerReport(config.configuration), configuration = config.configuration, phaseConfig = phaseConfig, @@ -155,7 +220,9 @@ abstract class BasicIrBoxTest( lowerPerModule = lowerPerModule, safeExternalBoolean = safeExternalBoolean, safeExternalBooleanDiagnostic = safeExternalBooleanDiagnostic, - verifySignatures = !skipMangleVerification + verifySignatures = !skipMangleVerification, + useStdlibCache = runIcMode, + icCache = icCache ) compiledModule.outputs!!.writeTo(outputFile, config) @@ -172,7 +239,7 @@ abstract class BasicIrBoxTest( if (runIrPir && !skipDceDriven) { compile( project = config.project, - mainModule = MainModule.SourceFiles(filesToCompile), + mainModule = mainModule, analyzer = AnalyzerWithCompilerReport(config.configuration), configuration = config.configuration, phaseConfig = phaseConfig,