FIR JS: Prepare JS/KLib infrastructure for FIR integration

rearrange declarations, abstract things that should be done differently with FIR, etc.
2022-06-07 17:05:54 +02:00
parent 126c8eafa3
commit 8bf108dc17
20 changed files with 479 additions and 193 deletions
@@ -1,32 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.ir.backend.js
-
-import org.jetbrains.kotlin.config.LanguageVersionSettings
-import org.jetbrains.kotlin.context.ModuleContext
-import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
-import org.jetbrains.kotlin.incremental.components.LookupTracker
-import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider
-import org.jetbrains.kotlin.js.analyze.AbstractTopDownAnalyzerFacadeForJS
-import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
-
-// TODO: put it in separated module `frontend.js`
-object TopDownAnalyzerFacadeForJSIR : AbstractTopDownAnalyzerFacadeForJS() {
-    override fun loadIncrementalCacheMetadata(
-        incrementalData: IncrementalDataProvider,
-        moduleContext: ModuleContext,
-        lookupTracker: LookupTracker,
-        languageVersionSettings: LanguageVersionSettings
-    ): PackageFragmentProvider {
-        return JsFactories.DefaultDeserializedDescriptorFactory.createCachedPackageFragmentProvider(
-            incrementalData.compiledPackageParts.values.map { it.metadata },
-            moduleContext.storageManager,
-            moduleContext.module,
-            CompilerDeserializationConfiguration(languageVersionSettings),
-            lookupTracker
-        )
-    }
-}
@@ -8,6 +8,10 @@ package org.jetbrains.kotlin.ir.backend.js
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.vfs.VfsUtilCore
 import com.intellij.psi.PsiElement
+import org.jetbrains.kotlin.KtIoFileSourceFile
+import org.jetbrains.kotlin.KtPsiSourceFile
+import org.jetbrains.kotlin.KtSourceFile
+import org.jetbrains.kotlin.KtVirtualFileSourceFile
 import org.jetbrains.kotlin.analyzer.AbstractAnalyzerWithCompilerReport
 import org.jetbrains.kotlin.analyzer.AnalysisResult
 import org.jetbrains.kotlin.analyzer.CompilationErrorException
@@ -19,9 +23,7 @@ import org.jetbrains.kotlin.backend.common.serialization.*
 import org.jetbrains.kotlin.backend.common.serialization.linkerissues.checkNoUnboundSymbols
 import org.jetbrains.kotlin.backend.common.serialization.mangle.ManglerChecker
 import org.jetbrains.kotlin.backend.common.serialization.mangle.descriptor.Ir2DescriptorManglerAdapter
-import org.jetbrains.kotlin.backend.common.serialization.metadata.DynamicTypeDeserializer
-import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataIncrementalSerializer
-import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion
+import org.jetbrains.kotlin.backend.common.serialization.metadata.*
 import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDescriptor
 import org.jetbrains.kotlin.builtins.KotlinBuiltIns
 import org.jetbrains.kotlin.config.*
@@ -30,6 +32,7 @@ import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider
 import org.jetbrains.kotlin.ir.IrBuiltIns
 import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI
 import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.*
@@ -41,6 +44,7 @@ import org.jetbrains.kotlin.ir.linkage.IrDeserializer
 import org.jetbrains.kotlin.ir.symbols.IrSymbol
 import org.jetbrains.kotlin.ir.util.*
 import org.jetbrains.kotlin.ir.visitors.acceptVoid
+import org.jetbrains.kotlin.js.analyze.AbstractTopDownAnalyzerFacadeForJS
 import org.jetbrains.kotlin.js.analyzer.JsAnalysisResult
 import org.jetbrains.kotlin.js.config.ErrorTolerancePolicy
 import org.jetbrains.kotlin.js.config.JSConfigurationKeys
@@ -205,42 +209,24 @@ fun generateIrForKlibSerialization(

 fun generateKLib(
    depsDescriptors: ModulesStructure,
-    irFactory: IrFactory,
    outputKlibPath: String,
    nopack: Boolean,
-    verifySignatures: Boolean = true,
    abiVersion: KotlinAbiVersion = KotlinAbiVersion.CURRENT,
-    jsOutputName: String?
+    jsOutputName: String?,
+    icData: List<KotlinFileSerializedData>,
+    expectDescriptorToSymbol: MutableMap<DeclarationDescriptor, IrSymbol>,
+    moduleFragment: IrModuleFragment,
+    serializeSingleFile: (KtSourceFile) -> ProtoBuf.PackageFragment
 ) {
-    val project = depsDescriptors.project
-    val files = (depsDescriptors.mainModule as MainModule.SourceFiles).files
+    val files = (depsDescriptors.mainModule as MainModule.SourceFiles).files.map(::KtPsiSourceFile)
    val configuration = depsDescriptors.compilerConfiguration
    val allDependencies = depsDescriptors.allDependencies.map { it.library }
    val messageLogger = configuration.irMessageLogger

-    val icData = mutableListOf<KotlinFileSerializedData>()
-    val expectDescriptorToSymbol = mutableMapOf<DeclarationDescriptor, IrSymbol>()
-
-    val moduleFragment = generateIrForKlibSerialization(
-        project,
-        files,
-        configuration,
-        depsDescriptors.jsFrontEndResult.jsAnalysisResult,
-        sortDependencies(depsDescriptors.descriptors),
-        icData,
-        expectDescriptorToSymbol,
-        irFactory,
-        verifySignatures
-    ) {
-        depsDescriptors.getModuleDescriptor(it)
-    }
-
    serializeModuleIntoKlib(
        configuration[CommonConfigurationKeys.MODULE_NAME]!!,
-        project,
        configuration,
        messageLogger,
-        depsDescriptors.jsFrontEndResult.bindingContext,
        files,
        outputKlibPath,
        allDependencies,
@@ -251,7 +237,8 @@ fun generateKLib(
        perFile = false,
        depsDescriptors.jsFrontEndResult.hasErrors,
        abiVersion,
-        jsOutputName
+        jsOutputName,
+        serializeSingleFile
    )
 }

@@ -484,22 +471,6 @@ fun getIrModuleInfoForSourceFiles(
    )
 }

-fun prepareAnalyzedSourceModule(
-    project: Project,
-    files: List<KtFile>,
-    configuration: CompilerConfiguration,
-    dependencies: List<String>,
-    friendDependencies: List<String>,
-    analyzer: AbstractAnalyzerWithCompilerReport,
-    errorPolicy: ErrorTolerancePolicy = configuration.get(JSConfigurationKeys.ERROR_TOLERANCE_POLICY) ?: ErrorTolerancePolicy.DEFAULT,
-): ModulesStructure {
-    val mainModule = MainModule.SourceFiles(files)
-    val sourceModule = ModulesStructure(project, mainModule, configuration, dependencies, friendDependencies)
-    return sourceModule.apply {
-        runAnalysis(errorPolicy, analyzer)
-    }
-}
-
 private fun preparePsi2Ir(
    depsDescriptors: ModulesStructure,
    errorIgnorancePolicy: ErrorTolerancePolicy,
@@ -633,12 +604,12 @@ class ModulesStructure(

    lateinit var jsFrontEndResult: JsFrontEndResult

-    fun runAnalysis(errorPolicy: ErrorTolerancePolicy, analyzer: AbstractAnalyzerWithCompilerReport) {
+    fun runAnalysis(errorPolicy: ErrorTolerancePolicy, analyzer: AbstractAnalyzerWithCompilerReport, analyzerFacade: AbstractTopDownAnalyzerFacadeForJS) {
        require(mainModule is MainModule.SourceFiles)
        val files = mainModule.files

        analyzer.analyzeAndReport(files) {
-            TopDownAnalyzerFacadeForJSIR.analyzeFiles(
+            analyzerFacade.analyzeFiles(
                files,
                project,
                compilerConfiguration,
@@ -665,7 +636,7 @@ class ModulesStructure(
            else hasErrors = true
        }

-        hasErrors = TopDownAnalyzerFacadeForJSIR.checkForErrors(files, analysisResult.bindingContext, errorPolicy) || hasErrors
+        hasErrors = analyzerFacade.checkForErrors(files, analysisResult.bindingContext, errorPolicy) || hasErrors

        jsFrontEndResult = JsFrontEndResult(analysisResult, hasErrors)
    }
@@ -712,11 +683,9 @@ private fun getDescriptorForElement(

 fun serializeModuleIntoKlib(
    moduleName: String,
-    project: Project,
    configuration: CompilerConfiguration,
    messageLogger: IrMessageLogger,
-    bindingContext: BindingContext,
-    files: List<KtFile>,
+    files: List<KtSourceFile>,
    klibPath: String,
    dependencies: List<KotlinLibrary>,
    moduleFragment: IrModuleFragment,
@@ -727,6 +696,7 @@ fun serializeModuleIntoKlib(
    containsErrorCode: Boolean = false,
    abiVersion: KotlinAbiVersion,
    jsOutputName: String?,
+    serializeSingleFile: (KtSourceFile) -> ProtoBuf.PackageFragment
 ) {
    assert(files.size == moduleFragment.files.size)

@@ -748,7 +718,6 @@ fun serializeModuleIntoKlib(
        ).serializedIrModule(moduleFragment)

    val moduleDescriptor = moduleFragment.descriptor
-    val metadataSerializer = KlibMetadataIncrementalSerializer(configuration, project, containsErrorCode)

    val incrementalResultsConsumer = configuration.get(JSConfigurationKeys.INCREMENTAL_RESULTS_CONSUMER)
    val empty = ByteArray(0)
@@ -764,33 +733,38 @@ fun serializeModuleIntoKlib(

    val additionalFiles = mutableListOf<KotlinFileSerializedData>()

-    for ((ktFile, binaryFile) in files.zip(serializedIr.files)) {
-        assert(ktFile.virtualFilePath == binaryFile.path) {
+    for ((ktSourceFile, binaryFile) in files.zip(serializedIr.files)) {
+        assert(ktSourceFile.path == binaryFile.path) {
            """The Kt and Ir files are put in different order
-                Kt: ${ktFile.virtualFilePath}
+                Kt: ${ktSourceFile.path}
                Ir: ${binaryFile.path}
            """.trimMargin()
        }
-        val packageFragment = metadataSerializer.serializeScope(ktFile, bindingContext, moduleDescriptor)
+        val packageFragment = serializeSingleFile(ktSourceFile)
        val compiledKotlinFile = KotlinFileSerializedData(packageFragment.toByteArray(), binaryFile)

        additionalFiles += compiledKotlinFile
-        processCompiledFileData(VfsUtilCore.virtualToIoFile(ktFile.virtualFile), compiledKotlinFile)
+        val ioFile = ktSourceFile.toIoFileOrNull()
+        assert(ioFile != null) {
+            "No file found for source ${ktSourceFile.path}"
+        }
+        processCompiledFileData(ioFile!!, compiledKotlinFile)
    }

    val compiledKotlinFiles = (cleanFiles + additionalFiles)

-    val header = metadataSerializer.serializeHeader(
-        moduleDescriptor,
+    val header = serializeKlibHeader(
+        configuration.languageVersionSettings, moduleDescriptor,
        compiledKotlinFiles.map { it.irData.fqName }.distinct().sorted(),
        emptyList()
    ).toByteArray()
+
    incrementalResultsConsumer?.run {
        processHeader(header)
    }

    val serializedMetadata =
-        metadataSerializer.serializedMetadata(
+        makeSerializedKlibMetadata(
            compiledKotlinFiles.groupBy { it.irData.fqName }
                .map { (fqn, data) -> fqn to data.sortedBy { it.irData.path }.map { it.metadata } }.toMap(),
            header
@@ -832,7 +806,7 @@ fun serializeModuleIntoKlib(

 const val KLIB_PROPERTY_JS_OUTPUT_NAME = "jsOutputName"

-private fun KlibMetadataIncrementalSerializer.serializeScope(
+fun KlibMetadataIncrementalSerializer.serializeScope(
    ktFile: KtFile,
    bindingContext: BindingContext,
    moduleDescriptor: ModuleDescriptor
@@ -841,6 +815,16 @@ private fun KlibMetadataIncrementalSerializer.serializeScope(
    return serializePackageFragment(moduleDescriptor, memberScope, ktFile.packageFqName)
 }

+fun KlibMetadataIncrementalSerializer.serializeScope(
+    ktSourceFile: KtSourceFile,
+    bindingContext: BindingContext,
+    moduleDescriptor: ModuleDescriptor
+): ProtoBuf.PackageFragment {
+    val ktFile = (ktSourceFile as KtPsiSourceFile).psiFile as KtFile
+    val memberScope = ktFile.declarations.map { getDescriptorForElement(bindingContext, it) }
+    return serializePackageFragment(moduleDescriptor, memberScope, ktFile.packageFqName)
+}
+
 private fun compareMetadataAndGoToNextICRoundIfNeeded(
    analysisResult: AnalysisResult,
    config: CompilerConfiguration,
@@ -861,7 +845,7 @@ private fun compareMetadataAndGoToNextICRoundIfNeeded(
    if (nextRoundChecker.shouldGoToNextRound()) throw IncrementalNextRoundException()
 }

-private fun KlibMetadataIncrementalSerializer(configuration: CompilerConfiguration, project: Project, allowErrors: Boolean) =
+fun KlibMetadataIncrementalSerializer(configuration: CompilerConfiguration, project: Project, allowErrors: Boolean) =
    KlibMetadataIncrementalSerializer(
        languageVersionSettings = configuration.languageVersionSettings,
        metadataVersion = configuration.metadataVersion,
@@ -876,3 +860,49 @@ private fun Map<IrModuleFragment, KotlinLibrary>.getUniqueNameForEachFragment():
        klib.jsOutputName?.let { moduleFragment to it }
    }.toMap()
 }
+
+fun KtSourceFile.toIoFileOrNull(): File? = when (this) {
+    is KtIoFileSourceFile -> file
+    is KtVirtualFileSourceFile -> VfsUtilCore.virtualToIoFile(virtualFile)
+    is KtPsiSourceFile -> VfsUtilCore.virtualToIoFile(psiFile.virtualFile)
+    else -> path?.let(::File)
+}
+
+fun IncrementalDataProvider.getSerializedData(newSources: List<KtSourceFile>): List<KotlinFileSerializedData> {
+    val nonCompiledSources = newSources.associateBy { it.toIoFileOrNull()!! }
+    val compiledIrFiles = serializedIrFiles
+    val compiledMetaFiles = compiledPackageParts
+
+    assert(compiledIrFiles.size == compiledMetaFiles.size)
+
+    val storage = mutableListOf<KotlinFileSerializedData>()
+
+    for (f in compiledIrFiles.keys) {
+        if (f in nonCompiledSources) continue
+
+        val irData = compiledIrFiles[f] ?: error("No Ir Data found for file $f")
+        val metaFile = compiledMetaFiles[f] ?: error("No Meta Data found for file $f")
+        val irFile = with(irData) {
+            SerializedIrFile(
+                fileData,
+                String(fqn),
+                f.path.replace('\\', '/'),
+                types,
+                signatures,
+                strings,
+                bodies,
+                declarations,
+                debugInfo
+            )
+        }
+        storage.add(KotlinFileSerializedData(metaFile.metadata, irFile))
+    }
+    return storage
+}
+
+@JvmName("getSerializedDataByPsiFiles")
+fun IncrementalDataProvider.getSerializedData(newSources: List<KtFile>): List<KotlinFileSerializedData> =
+    getSerializedData(newSources.map(::KtPsiSourceFile))
+
+val CompilerConfiguration.incrementalDataProvider : IncrementalDataProvider?
+    get() = get(JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER)