[JS IR] run IC box tests

2021-09-29 18:52:47 +03:00
parent b1b88a0d11
commit 546ce501cb
7 changed files with 433 additions and 233 deletions
@@ -165,14 +165,6 @@ fun lowerPreservingIcData(module: IrModuleFragment, context: JsIrBackendContext,
    controller.currentStage = pirLowerings.size + 1
 }

-@Suppress("UNUSED_PARAMETER")
-fun generateJsFromAst(
-    mainModule: String,
-    caches: Map<String, ModuleCache>
-): CompilerResult {
-    TODO(">> EP to generate Js from BinaryAST with root $mainModule")
-}
-
 fun generateJsCode(
    context: JsIrBackendContext,
    moduleFragment: IrModuleFragment,
@@ -0,0 +1,118 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.ir.backend.js
+
+import org.jetbrains.kotlin.backend.common.phaser.PhaseConfig
+import org.jetbrains.kotlin.backend.common.phaser.invokeToplevel
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI
+import org.jetbrains.kotlin.ir.backend.js.ic.ModuleCache
+import org.jetbrains.kotlin.ir.backend.js.ic.PersistentCacheConsumer
+import org.jetbrains.kotlin.ir.backend.js.lower.generateJsTests
+import org.jetbrains.kotlin.ir.backend.js.lower.moveBodilessDeclarationsToSeparatePlace
+import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
+import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformer
+import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.generateWrappedModuleBody
+import org.jetbrains.kotlin.ir.backend.js.utils.serialization.JsIrAstDeserializer
+import org.jetbrains.kotlin.ir.declarations.IrFactory
+import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
+import org.jetbrains.kotlin.ir.descriptors.IrBuiltInsOverDescriptors
+import org.jetbrains.kotlin.ir.util.ExternalDependenciesGenerator
+import org.jetbrains.kotlin.ir.util.noUnboundLeft
+import org.jetbrains.kotlin.js.config.RuntimeDiagnostic
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.serialization.js.ModuleKind
+import java.io.ByteArrayInputStream
+
+@OptIn(ObsoleteDescriptorBasedAPI::class)
+fun compileWithIC(
+    module: IrModuleFragment,
+    configuration: CompilerConfiguration,
+    deserializer: JsIrLinker,
+    dependencies: Collection<IrModuleFragment>,
+    mainArguments: List<String>? = null,
+    exportedDeclarations: Set<FqName> = emptySet(),
+    generateFullJs: Boolean = true,
+    generateDceJs: Boolean = false,
+    dceDriven: Boolean = false,
+    dceRuntimeDiagnostic: RuntimeDiagnostic? = null,
+    es6mode: Boolean = false,
+    multiModule: Boolean = false,
+    relativeRequirePath: Boolean = false,
+    propertyLazyInitialization: Boolean = false,
+    verifySignatures: Boolean = true,
+    baseClassIntoMetadata: Boolean = false,
+    lowerPerModule: Boolean = false,
+    safeExternalBoolean: Boolean = false,
+    safeExternalBooleanDiagnostic: RuntimeDiagnostic? = null,
+    filesToLower: Set<String>?,
+    cacheConsumer: PersistentCacheConsumer,
+) {
+
+    val mainModule = module
+    val allModules = dependencies
+    val moduleDescriptor = module.descriptor
+    val irBuiltIns = module.irBuiltins
+    val symbolTable = (irBuiltIns as IrBuiltInsOverDescriptors).symbolTable
+
+    val context = JsIrBackendContext(
+        moduleDescriptor,
+        irBuiltIns,
+        symbolTable,
+        module,
+        exportedDeclarations,
+        configuration,
+        es6mode = es6mode,
+        dceRuntimeDiagnostic = dceRuntimeDiagnostic,
+        propertyLazyInitialization = propertyLazyInitialization,
+        baseClassIntoMetadata = baseClassIntoMetadata,
+        safeExternalBoolean = safeExternalBoolean,
+        safeExternalBooleanDiagnostic = safeExternalBooleanDiagnostic
+    )
+
+    // Load declarations referenced during `context` initialization
+    val irProviders = listOf(deserializer)
+    ExternalDependenciesGenerator(symbolTable, irProviders).generateUnboundSymbolsAsDependencies()
+
+    deserializer.postProcess()
+    symbolTable.noUnboundLeft("Unbound symbols at the end of linker")
+
+    allModules.forEach { module ->
+        moveBodilessDeclarationsToSeparatePlace(context, module)
+    }
+
+    generateJsTests(context, mainModule)
+
+    jsPhases.invokeToplevel(PhaseConfig(jsPhases), context, allModules)
+
+    val transformer = IrModuleToJsTransformer(
+        context,
+        mainArguments,
+        fullJs = generateFullJs,
+        dceJs = generateDceJs,
+        multiModule = multiModule,
+        relativeRequirePath = relativeRequirePath,
+    )
+
+    val dirtyFiles = filesToLower?.let { dirties ->
+        module.files.filter { it.fileEntry.name in dirties }
+    } ?: module.files
+
+    val ast = transformer.generateBinaryAst(dirtyFiles)
+
+    ast.entries.forEach { (path, bytes) -> cacheConsumer.commitBinaryAst(path, bytes) }
+}
+
+
+@Suppress("UNUSED_PARAMETER")
+fun generateJsFromAst(
+    mainModule: String,
+    caches: Map<String, ModuleCache>
+): CompilerResult {
+    val deserializer = JsIrAstDeserializer()
+    val fragments = caches.values.map { it.asts.values.mapNotNull { it.ast?.let { deserializer.deserialize(ByteArrayInputStream(it))} } }
+    return CompilerResult(generateWrappedModuleBody("JS_TESTS", ModuleKind.PLAIN, fragments), null)
+}
@@ -17,15 +17,12 @@ import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI
-import org.jetbrains.kotlin.ir.backend.js.JsFactories
-import org.jetbrains.kotlin.ir.backend.js.jsResolveLibraries
+import org.jetbrains.kotlin.ir.backend.js.*
 import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
 import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerDesc
-import org.jetbrains.kotlin.ir.backend.js.moduleName
-import org.jetbrains.kotlin.ir.backend.js.toResolverLogger
+import org.jetbrains.kotlin.ir.declarations.IrFactory
 import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
 import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
-import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrFactory
 import org.jetbrains.kotlin.ir.descriptors.IrBuiltInsOverDescriptors
 import org.jetbrains.kotlin.ir.util.*
 import org.jetbrains.kotlin.ir.visitors.acceptVoid
@@ -34,6 +31,7 @@ import org.jetbrains.kotlin.konan.properties.propertyList
 import org.jetbrains.kotlin.library.KLIB_PROPERTY_DEPENDS
 import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.library.unresolvedDependencies
+import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
 import org.jetbrains.kotlin.psi2ir.generators.TypeTranslatorImpl
 import org.jetbrains.kotlin.storage.LockBasedStorageManager
@@ -145,7 +143,9 @@ private fun KotlinLibrary.filesAndSigReaders(): List<Pair<String, IdSignatureDes

 private fun buildCacheForModule(
    libraryPath: String,
+    configuration: CompilerConfiguration,
    irModule: IrModuleFragment,
+    deserializer: JsIrLinker,
    dependencies: Collection<IrModuleFragment>,
    dirtyFiles: Collection<String>,
    cleanInlineHashes: Map<IdSignature, Hash>,
@@ -191,7 +191,7 @@ private fun buildCacheForModule(
    }

    // TODO: actual way of building a cache could change in future
-    buildCacheForModuleFiles(irModule, dependencies, dirtyFiles, cacheConsumer)
+    buildCacheForModuleFiles(irModule, dependencies, deserializer, configuration, dirtyFiles, cacheConsumer)

    cacheConsumer.commitLibraryPath(libraryPath)
 }
@@ -229,10 +229,14 @@ private fun loadModules(
 }

@OptIn(ObsoleteDescriptorBasedAPI::class)
-private fun createLinker(configuration: CompilerConfiguration, loadedModules: Map<ModuleDescriptor, KotlinLibrary>): JsIrLinker {
+private fun createLinker(
+    configuration: CompilerConfiguration,
+    loadedModules: Map<ModuleDescriptor, KotlinLibrary>,
+    irFactory: IrFactory
+): JsIrLinker {
    val logger = configuration[IrMessageLogger.IR_MESSAGE_LOGGER] ?: IrMessageLogger.None
    val signaturer = IdSignatureDescriptor(JsManglerDesc)
-    val symbolTable = SymbolTable(signaturer, PersistentIrFactory())
+    val symbolTable = SymbolTable(signaturer, irFactory)
    val moduleDescriptor = loadedModules.keys.last()
    val typeTranslator = TypeTranslatorImpl(symbolTable, configuration.languageVersionSettings, moduleDescriptor)
    val irBuiltIns = IrBuiltInsOverDescriptors(moduleDescriptor.builtIns, typeTranslator, symbolTable)
@@ -289,7 +293,8 @@ fun actualizeCacheForModule(
    cachePath: String,
    compilerConfiguration: CompilerConfiguration,
    dependencies: Collection<ModulePath>,
-    icCachePaths: Collection<String>
+    icCachePaths: Collection<String>,
+    irFactory: IrFactory,
 ): Boolean {
    val icCacheMap: Map<ModulePath, String> = loadCacheInfo(icCachePaths).also {
        it[moduleName.toCanonicalPath()] = cachePath
@@ -312,7 +317,7 @@ fun actualizeCacheForModule(
    val currentModule = libraries[moduleName.toCanonicalPath()] ?: error("No loaded library found for path $moduleName")
    val persistentCacheConsumer = createCacheConsumer(cachePath)

-    return actualizeCacheForModule(currentModule, compilerConfiguration, dependencyGraph, persistentCacheProviders, persistentCacheConsumer)
+    return actualizeCacheForModule(currentModule, compilerConfiguration, dependencyGraph, persistentCacheProviders, persistentCacheConsumer, irFactory)
 }


@@ -321,7 +326,8 @@ private fun actualizeCacheForModule(
    configuration: CompilerConfiguration,
    dependencyGraph: Map<KotlinLibrary, Collection<KotlinLibrary>>,
    persistentCacheProviders: Map<KotlinLibrary, PersistentCacheProvider>,
-    persistentCacheConsumer: PersistentCacheConsumer
+    persistentCacheConsumer: PersistentCacheConsumer,
+    irFactory: IrFactory,
 ): Boolean {
    // 1. Invalidate
    val dependencies = dependencyGraph[library]!!
@@ -375,7 +381,7 @@ private fun actualizeCacheForModule(

    val loadedModules = loadModules(configuration.languageVersionSettings, dependencyGraph)

-    val jsIrLinker = createLinker(configuration, loadedModules)
+    val jsIrLinker = createLinker(configuration, loadedModules, irFactory)

    val irModules = ArrayList<Pair<IrModuleFragment, KotlinLibrary>>(loadedModules.size)

@@ -408,7 +414,9 @@ private fun actualizeCacheForModule(

    buildCacheForModule(
        library.libraryFile.canonicalPath,
+        configuration,
        currentIrModule,
+        jsIrLinker,
        irModules.map { it.first },
        dirtySet,
        sigHashes,
@@ -424,11 +432,12 @@ fun rebuildCacheForDirtyFiles(
    configuration: CompilerConfiguration,
    dependencyGraph: Map<KotlinLibrary, Collection<KotlinLibrary>>,
    dirtyFiles: Collection<String>?,
-    cacheConsumer: PersistentCacheConsumer
+    cacheConsumer: PersistentCacheConsumer,
+    irFactory: IrFactory,
 ) {
    val loadedModules = loadModules(configuration.languageVersionSettings, dependencyGraph)

-    val jsIrLinker = createLinker(configuration, loadedModules)
+    val jsIrLinker = createLinker(configuration, loadedModules, irFactory)

    val irModules = ArrayList<Pair<IrModuleFragment, KotlinLibrary>>(loadedModules.size)

@@ -453,16 +462,28 @@ fun rebuildCacheForDirtyFiles(

    val currentIrModule = irModules.find { it.second == library }?.first!!

-    buildCacheForModuleFiles(currentIrModule, irModules.map { it.first }, dirtyFiles, cacheConsumer)
+    buildCacheForModuleFiles(currentIrModule, irModules.map { it.first }, jsIrLinker, configuration, dirtyFiles, cacheConsumer)
 }

@Suppress("UNUSED_PARAMETER")
 private fun buildCacheForModuleFiles(
    currentModule: IrModuleFragment,
    dependencies: Collection<IrModuleFragment>,
+    deserializer: JsIrLinker,
+    configuration: CompilerConfiguration,
    dirtyFiles: Collection<String>?, // if null consider the whole module dirty
    cacheConsumer: PersistentCacheConsumer
 ) {
+    compileWithIC(
+        currentModule,
+        configuration = configuration,
+        deserializer = deserializer,
+        dependencies = dependencies,
+        exportedDeclarations = setOf(FqName("box")),
+        filesToLower = dirtyFiles?.toSet(),
+        cacheConsumer = cacheConsumer,
+    )
+
 //    println("creating caches for module ${currentModule.name}")
 //    println("Store them into $cacheConsumer")
 //    val dirtyS = if (dirtyFiles == null) "[ALL]" else dirtyFiles.joinToString(",", "[", "]") { it }
@@ -50,6 +50,18 @@ class IrModuleToJsTransformer(
    private val mainModuleName = backendContext.configuration[CommonConfigurationKeys.MODULE_NAME]!!
    private val moduleKind = backendContext.configuration[JSConfigurationKeys.MODULE_KIND]!!

+    fun generateBinaryAst(files: Iterable<IrFile>): Map<String, ByteArray> {
+        val exportModelGenerator = ExportModelGenerator(backendContext, generateNamespacesForPackages = true)
+
+        val exportData = files.associate { file ->
+            file to exportModelGenerator.generateExport(file)
+        }
+
+        files.forEach { StaticMembersLowering(backendContext).lower(it) }
+
+        return generateBinaryAst(files, exportData)
+    }
+
    fun generateModule(modules: Iterable<IrModuleFragment>): CompilerResult {
        val exportModelGenerator = ExportModelGenerator(backendContext, generateNamespacesForPackages = true)

@@ -65,21 +77,42 @@ class IrModuleToJsTransformer(
            module.files.forEach { StaticMembersLowering(backendContext).lower(it) }
        }

-        val jsCode = if (fullJs) generateWrappedModuleBody(mainModuleName, modules, generateProgramFragments(modules, exportData)) else null
+        val jsCode = if (fullJs) generateWrappedModuleBody(mainModuleName, moduleKind, generateProgramFragments(modules, exportData)) else null

        val dceJsCode = if (dceJs) {
            eliminateDeadDeclarations(modules, backendContext, removeUnusedAssociatedObjects)

-            generateWrappedModuleBody(mainModuleName, modules, generateProgramFragments(modules, exportData))
+            generateWrappedModuleBody(mainModuleName, moduleKind, generateProgramFragments(modules, exportData))
        } else null

        return CompilerResult(jsCode, dceJsCode, dts)
    }

+    private fun generateBinaryAst(
+        files: Iterable<IrFile>,
+        exportData: Map<IrFile, List<ExportedDeclaration>>,
+    ): Map<String, ByteArray> {
+
+        val serializer = JsIrAstSerializer()
+
+        val result = mutableMapOf<String, ByteArray>()
+        files.forEach { f ->
+            val exports = exportData[f]!! // TODO
+            val fragment = generateProgramFragment(f, exports)
+            val output = ByteArrayOutputStream()
+            serializer.serialize(fragment, output)
+            val binaryAst = output.toByteArray()
+            result[f.fileEntry.name] = binaryAst
+        }
+
+        return result
+    }
+
+
    private fun generateProgramFragments(
        modules: Iterable<IrModuleFragment>,
        exportData: Map<IrModuleFragment, Map<IrFile, List<ExportedDeclaration>>>,
-    ): Map<IrFile, JsIrProgramFragment> {
+    ): List<List<JsIrProgramFragment>> {

        val fragments = mutableMapOf<IrFile, JsIrProgramFragment>()
        modules.forEach { m ->
@@ -89,79 +122,26 @@ class IrModuleToJsTransformer(
            }
        }

-        // TODO remove serialization -> deserialization work
-        val serialized = mutableListOf<Pair<IrFile, ByteArray>>()
+//        // TODO remove serialization -> deserialization work
+//        val serialized = mutableListOf<Pair<IrFile, ByteArray>>()
+//
+//        val serializer = JsIrAstSerializer()
+//        fragments.entries.forEach { (file, fragment) ->
+//            val output = ByteArrayOutputStream()
+//            serializer.serialize(fragment, output)
+//            val binaryAst = output.toByteArray()
+//            serialized += file to binaryAst
+//        }
+//
+//        val restoredMap = mutableMapOf<IrFile, JsIrProgramFragment>()
+//
+//        val deserializer = JsIrAstDeserializer()
+//
+//        serialized.forEach { (file, binaryAst) ->
+//            restoredMap[file] = deserializer.deserialize(ByteArrayInputStream(binaryAst))
+//        }

-        val serializer = JsIrAstSerializer()
-        fragments.entries.forEach { (file, fragment) ->
-            val output = ByteArrayOutputStream()
-            serializer.serialize(fragment, output)
-            val binaryAst = output.toByteArray()
-            serialized += file to binaryAst
-        }
-
-        val restoredMap = mutableMapOf<IrFile, JsIrProgramFragment>()
-
-        val deserializer = JsIrAstDeserializer()
-
-        serialized.forEach { (file, binaryAst) ->
-            restoredMap[file] = deserializer.deserialize(ByteArrayInputStream(binaryAst))
-        }
-
-        return restoredMap
-    }
-
-    private fun generateWrappedModuleBody(
-        moduleName: String,
-        modules: Iterable<IrModuleFragment>,
-        fragments: Map<IrFile, JsIrProgramFragment>,
-    ): CompilationOutputs {
-        val program = Merger(
-            moduleName,
-            moduleKind,
-            modules.map { it.files.map { fragments[it]!! } },
-            generateScriptModule,
-            generateRegionComments
-        ).merge()
-
-        program.resolveTemporaryNames()
-
-        val jsCode = TextOutputImpl()
-
-        val configuration = backendContext.configuration
-        val sourceMapPrefix = configuration.get(JSConfigurationKeys.SOURCE_MAP_PREFIX, "")
-        val sourceMapsEnabled = configuration.getBoolean(JSConfigurationKeys.SOURCE_MAP)
-
-        val sourceMapBuilder = SourceMap3Builder(null, jsCode, sourceMapPrefix)
-        val sourceMapBuilderConsumer =
-            if (sourceMapsEnabled) {
-                val sourceRoots = configuration.get(JSConfigurationKeys.SOURCE_MAP_SOURCE_ROOTS, emptyList<String>()).map(::File)
-                val generateRelativePathsInSourceMap = sourceMapPrefix.isEmpty() && sourceRoots.isEmpty()
-                val outputDir = if (generateRelativePathsInSourceMap) configuration.get(JSConfigurationKeys.OUTPUT_DIR) else null
-
-                val pathResolver = SourceFilePathResolver(sourceRoots, outputDir)
-
-                val sourceMapContentEmbedding =
-                    configuration.get(JSConfigurationKeys.SOURCE_MAP_EMBED_SOURCES, SourceMapSourceEmbedding.INLINING)
-
-                SourceMapBuilderConsumer(
-                    File("."),
-                    sourceMapBuilder,
-                    pathResolver,
-                    sourceMapContentEmbedding == SourceMapSourceEmbedding.ALWAYS,
-                    sourceMapContentEmbedding != SourceMapSourceEmbedding.NEVER
-                )
-            } else {
-                NoOpSourceLocationConsumer
-            }
-
-        program.accept(JsToStringGenerationVisitor(jsCode, sourceMapBuilderConsumer))
-
-        return CompilationOutputs(
-            jsCode.toString(),
-            program,
-            if (sourceMapsEnabled) sourceMapBuilder.build() else null
-        )
+        return modules.map { it.files.map { fragments[it]!! } }
    }

    private val generateFilePaths = backendContext.configuration.getBoolean(JSConfigurationKeys.GENERATE_COMMENTS_WITH_FILE_PATH)
@@ -333,4 +313,29 @@ class IrModuleToJsTransformer(
        val importedJsModules = (declarationLevelJsModules + packageLevelJsModules).distinctBy { it.key }
        return Pair(importStatements, importedJsModules)
    }
+}
+
+fun generateWrappedModuleBody(
+    moduleName: String,
+    moduleKind: ModuleKind,
+    fragments: List<List<JsIrProgramFragment>>
+): CompilationOutputs {
+    val program = Merger(
+        moduleName,
+        moduleKind,
+        fragments,
+        false,
+        true
+    ).merge()
+
+    program.resolveTemporaryNames()
+
+    val jsCode = TextOutputImpl()
+
+    program.accept(JsToStringGenerationVisitor(jsCode, NoOpSourceLocationConsumer))
+
+    return CompilationOutputs(
+        jsCode.toString(),
+        null
+    )
 }
@@ -44,7 +44,7 @@ import org.jetbrains.kotlin.types.checker.KotlinTypeChecker
 class IrBuiltInsOverDescriptors(
    val builtIns: KotlinBuiltIns,
    private val typeTranslator: TypeTranslator,
-    private val symbolTable: SymbolTable
+    val symbolTable: SymbolTable
 ) : IrBuiltIns() {
    override val languageVersionSettings = typeTranslator.languageVersionSettings