[JS IR] initial support for KLIB incremental compilation with K2

2023-04-06 12:17:36 +02:00
parent a9683bd4a8
commit c839fdca3e
15 changed files with 1362 additions and 138 deletions
@@ -17,6 +17,7 @@ import org.jetbrains.kotlin.fir.analysis.checkers.FirPlatformDiagnosticSuppresso
 import org.jetbrains.kotlin.fir.analysis.js.checkers.FirJsPlatformDiagnosticSuppressor
 import org.jetbrains.kotlin.fir.checkers.registerJsCheckers
 import org.jetbrains.kotlin.fir.deserialization.ModuleDataProvider
+import org.jetbrains.kotlin.fir.deserialization.SingleModuleDataProvider
 import org.jetbrains.kotlin.fir.extensions.FirExtensionRegistrar
 import org.jetbrains.kotlin.fir.java.FirProjectSessionProvider
 import org.jetbrains.kotlin.fir.resolve.calls.ConeCallConflictResolverFactory
@@ -35,6 +36,7 @@ object FirJsSessionFactory : FirAbstractSessionFactory() {
        extensionRegistrars: List<FirExtensionRegistrar>,
        languageVersionSettings: LanguageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
        lookupTracker: LookupTracker?,
+        icData: KlibIcData? = null,
        registerExtraComponents: ((FirSession) -> Unit) = {},
        init: FirSessionConfigurator.() -> Unit
    ): FirSession {
@@ -52,10 +54,18 @@ object FirJsSessionFactory : FirAbstractSessionFactory() {
            },
            registerExtraCheckers = { it.registerJsCheckers() },
            createKotlinScopeProvider = { FirKotlinScopeProvider { _, declaredMemberScope, _, _, _ -> declaredMemberScope } },
-            createProviders = { _, _, symbolProvider, generatedSymbolsProvider, syntheticFunctionInterfaceProvider, dependencies ->
+            createProviders = { session, kotlinScopeProvider, symbolProvider, generatedSymbolsProvider, syntheticFunctionInterfaceProvider, dependencies ->
                listOfNotNull(
                    symbolProvider,
                    generatedSymbolsProvider,
+                    icData?.let {
+                        KlibIcCacheBasedSymbolProvider(
+                            session,
+                            SingleModuleDataProvider(moduleData),
+                            kotlinScopeProvider,
+                            it,
+                        )
+                    },
                    syntheticFunctionInterfaceProvider,
                    *dependencies.toTypedArray(),
                )
@@ -5,6 +5,7 @@

 package org.jetbrains.kotlin.fir.session

+import org.jetbrains.kotlin.fir.FirModuleData
 import org.jetbrains.kotlin.fir.FirSession
 import org.jetbrains.kotlin.fir.declarations.FirDeclarationOrigin
 import org.jetbrains.kotlin.fir.deserialization.*
@@ -20,6 +21,7 @@ import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
+import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedContainerSource
 import org.jetbrains.kotlin.serialization.deserialization.getClassId
 import org.jetbrains.kotlin.utils.SmartList
 import java.nio.file.Paths
@@ -30,14 +32,17 @@ class KlibBasedSymbolProvider(
    kotlinScopeProvider: FirKotlinScopeProvider,
    private val resolvedLibraries: Collection<KotlinLibrary>,
    defaultDeserializationOrigin: FirDeclarationOrigin = FirDeclarationOrigin.Library
-) : AbstractFirDeserializedSymbolProvider(
-    session, moduleDataProvider, kotlinScopeProvider, defaultDeserializationOrigin, KlibMetadataSerializerProtocol
+) : MetadataLibraryBasedSymbolProvider<KotlinLibrary>(
+    session,
+    moduleDataProvider,
+    kotlinScopeProvider,
+    defaultDeserializationOrigin
 ) {
    private val moduleHeaders by lazy {
        resolvedLibraries.associate { it to parseModuleHeader(it.moduleHeaderData) }
    }

-    private val fragmentNamesInLibraries: Map<String, List<KotlinLibrary>> by lazy {
+    override val fragmentNamesInLibraries: Map<String, List<KotlinLibrary>> by lazy {
        buildMap<String, SmartList<KotlinLibrary>> {
            for ((library, header) in moduleHeaders) {
                for (fragmentName in header.packageFragmentNameList) {
@@ -48,7 +53,7 @@ class KlibBasedSymbolProvider(
        }
    }

-    private val knownPackagesInLibraries: Set<FqName> by lazy {
+    override val knownPackagesInLibraries: Set<FqName> by lazy {
        buildSet<FqName> {
            for ((_, header) in moduleHeaders) {
                for (fragmentName in header.packageFragmentNameList) {
@@ -62,124 +67,12 @@ class KlibBasedSymbolProvider(
        }
    }

-    private val annotationDeserializer = KlibBasedAnnotationDeserializer(session)
-    private val constDeserializer = FirConstDeserializer(session, KlibMetadataSerializerProtocol)
-    private val deserializationConfiguration = CompilerDeserializationConfiguration(session.languageVersionSettings)
-    private val cachedFragments = mutableMapOf<KotlinLibrary, MutableMap<Pair<String, String>, ProtoBuf.PackageFragment>>()
-
-    private fun getPackageFragment(
-        resolvedLibrary: KotlinLibrary, packageStringName: String, packageMetadataPart: String
-    ): ProtoBuf.PackageFragment {
-        return cachedFragments.getOrPut(resolvedLibrary) {
-            mutableMapOf()
-        }.getOrPut(packageStringName to packageMetadataPart) {
-            parsePackageFragment(resolvedLibrary.packageMetadata(packageStringName, packageMetadataPart))
-        }
+    override fun moduleData(library: KotlinLibrary): FirModuleData? {
+        val libraryPath = Paths.get(library.libraryFile.path)
+        return moduleDataProvider.getModuleData(libraryPath)
    }

-    override fun computePackagePartsInfos(packageFqName: FqName): List<PackagePartsCacheData> {
-        val packageStringName = if (packageFqName.isRoot) "" else packageFqName.asString()
-
-        val librariesWithFragment = fragmentNamesInLibraries[packageStringName] ?: return emptyList()
-
-        return librariesWithFragment.flatMap { resolvedLibrary ->
-            resolvedLibrary.packageMetadataParts(packageStringName).mapNotNull {
-                val fragment = getPackageFragment(resolvedLibrary, packageStringName, it)
-
-                val libraryPath = Paths.get(resolvedLibrary.libraryFile.path)
-                val moduleData = moduleDataProvider.getModuleData(libraryPath) ?: return@mapNotNull null
-                val packageProto = fragment.`package`
-
-                val nameResolver = NameResolverImpl(
-                    fragment.strings,
-                    fragment.qualifiedNames,
-                )
-
-                PackagePartsCacheData(
-                    packageProto,
-                    FirDeserializationContext.createForPackage(
-                        packageFqName, packageProto, nameResolver, moduleData,
-                        annotationDeserializer,
-                        constDeserializer,
-                        createDeserializedContainerSource(resolvedLibrary, packageFqName),
-                    ),
-                )
-            }
-        }
-    }
-
-    override fun computePackageSetWithNonClassDeclarations(): Set<String> = fragmentNamesInLibraries.keys
-
-    override fun knownTopLevelClassesInPackage(packageFqName: FqName): Set<String> =
-        buildSet {
-            forEachFragmentInPackage(packageFqName) { _, fragment, nameResolver ->
-                for (classNameId in fragment.getExtension(KlibMetadataProtoBuf.className).orEmpty()) {
-                    add(nameResolver.getClassId(classNameId).shortClassName.asString())
-                }
-            }
-        }
-
-    @OptIn(SymbolInternals::class)
-    override fun extractClassMetadata(classId: ClassId, parentContext: FirDeserializationContext?): ClassMetadataFindResult? {
-        forEachFragmentInPackage(classId.packageFqName) { resolvedLibrary, fragment, nameResolver ->
-            val finder = KlibMetadataClassDataFinder(fragment, nameResolver)
-            val classProto = finder.findClassData(classId)?.classProto ?: return@forEachFragmentInPackage
-
-            val libraryPath = Paths.get(resolvedLibrary.libraryFile.path)
-            val moduleData = moduleDataProvider.getModuleData(libraryPath) ?: return null
-
-            return ClassMetadataFindResult.NoMetadata { symbol ->
-                val source = createDeserializedContainerSource(resolvedLibrary,
-
-                    classId.packageFqName
-                )
-
-                deserializeClassToSymbol(
-                    classId,
-                    classProto,
-                    symbol,
-                    nameResolver,
-                    session,
-                    moduleData,
-                    annotationDeserializer,
-                    kotlinScopeProvider,
-                    KlibMetadataSerializerProtocol,
-                    parentContext,
-                    source,
-                    origin = defaultDeserializationOrigin,
-                    deserializeNestedClass = this::getClass,
-                )
-                symbol.fir.isNewPlaceForBodyGeneration = isNewPlaceForBodyGeneration(classProto)
-            }
-        }
-
-        return null
-    }
-
-    private inline fun forEachFragmentInPackage(
-        packageFqName: FqName,
-        f: (KotlinLibrary, ProtoBuf.PackageFragment, NameResolver) -> Unit
-    ) {
-        val packageStringName = packageFqName.asString()
-
-        val librariesWithFragment = fragmentNamesInLibraries[packageStringName] ?: return
-
-        for (resolvedLibrary in librariesWithFragment) {
-            for (packageMetadataPart in resolvedLibrary.packageMetadataParts(packageStringName)) {
-
-                val fragment = getPackageFragment(resolvedLibrary, packageStringName, packageMetadataPart)
-
-                val nameResolver = NameResolverImpl(
-                    fragment.strings,
-                    fragment.qualifiedNames,
-                )
-
-                f(resolvedLibrary, fragment, nameResolver)
-            }
-        }
-    }
-
-    private fun createDeserializedContainerSource(
+    override fun createDeserializedContainerSource(
        resolvedLibrary: KotlinLibrary,
        packageFqName: FqName
    ) = KlibDeserializedContainerSource(
@@ -188,14 +81,4 @@ class KlibBasedSymbolProvider(
        deserializationConfiguration,
        packageFqName
    )
-
-    override fun isNewPlaceForBodyGeneration(classProto: ProtoBuf.Class) = false
-
-    override fun getPackage(fqName: FqName): FqName? {
-        return if (fqName in knownPackagesInLibraries) {
-            fqName
-        } else {
-            null
-        }
-    }
 }
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.fir.session
+
+
+import org.jetbrains.kotlin.fir.FirModuleData
+import org.jetbrains.kotlin.fir.FirSession
+import org.jetbrains.kotlin.fir.declarations.FirDeclarationOrigin
+import org.jetbrains.kotlin.fir.deserialization.SingleModuleDataProvider
+import org.jetbrains.kotlin.fir.scopes.FirKotlinScopeProvider
+import org.jetbrains.kotlin.library.metadata.KlibDeserializedContainerSource
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedContainerSource
+import org.jetbrains.kotlin.utils.SmartList
+
+class KlibIcCacheBasedSymbolProvider(
+    session: FirSession,
+    moduleDataProvider: SingleModuleDataProvider,
+    kotlinScopeProvider: FirKotlinScopeProvider,
+    private val icData: KlibIcData,
+    defaultDeserializationOrigin: FirDeclarationOrigin = FirDeclarationOrigin.Library
+) : MetadataLibraryBasedSymbolProvider<KlibIcData>(
+    session,
+    moduleDataProvider,
+    kotlinScopeProvider,
+    defaultDeserializationOrigin
+) {
+    override fun moduleData(library: KlibIcData): FirModuleData {
+        return moduleDataProvider.allModuleData.single()
+    }
+
+    override val fragmentNamesInLibraries: Map<String, List<KlibIcData>> by lazy {
+        buildMap<String, SmartList<KlibIcData>> {
+            for (fragmentName in icData.packageFragmentNameList) {
+                getOrPut(fragmentName) { SmartList() }
+                    .add(icData)
+            }
+        }
+    }
+
+    override val knownPackagesInLibraries: Set<FqName> by lazy {
+        buildSet<FqName> {
+            for (fragmentName in icData.packageFragmentNameList) {
+                var curPackage = FqName(fragmentName)
+                while (!curPackage.isRoot) {
+                    add(curPackage)
+                    curPackage = curPackage.parent()
+                }
+            }
+        }
+    }
+
+    override fun createDeserializedContainerSource(resolvedLibrary: KlibIcData, packageFqName: FqName): DeserializedContainerSource {
+        return KlibDeserializedContainerSource(false, "Package '$packageFqName'", false)
+    }
+}
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.fir.session
+
+import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider
+import org.jetbrains.kotlin.library.MetadataLibrary
+import org.jetbrains.kotlin.library.metadata.KlibMetadataProtoBuf
+import org.jetbrains.kotlin.library.metadata.parsePackageFragment
+
+class KlibIcData(incrementalData: IncrementalDataProvider) : MetadataLibrary {
+
+    private val parts: Map<String, Map<String, ByteArray>> by lazy {
+        val result = mutableMapOf<String, MutableMap<String, ByteArray>>()
+
+        incrementalData.compiledPackageParts.entries.forEach { (f, tv) ->
+            val proto = parsePackageFragment(tv.metadata)
+            val fqName = proto.getExtension(KlibMetadataProtoBuf.fqName)
+            result.getOrPut(fqName, ::mutableMapOf).put(f.name, tv.metadata)
+        }
+
+        result
+    }
+
+    val packageFragmentNameList: Collection<String>
+        get() = parts.keys
+
+    override val moduleHeaderData: ByteArray
+        get() = error("moduleHeaderData is not implemented")
+
+    override fun packageMetadataParts(fqName: String): Set<String> {
+        return parts[fqName]?.keys ?: emptySet()
+    }
+
+    override fun packageMetadata(fqName: String, partName: String): ByteArray {
+        return parts[fqName]?.get(partName) ?: error("Metadata not found for package $fqName part $partName")
+    }
+}
@@ -0,0 +1,177 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.fir.session
+
+import org.jetbrains.kotlin.fir.FirModuleData
+import org.jetbrains.kotlin.fir.FirSession
+import org.jetbrains.kotlin.fir.caches.FirCache
+import org.jetbrains.kotlin.fir.caches.firCachesFactory
+import org.jetbrains.kotlin.fir.caches.getValue
+import org.jetbrains.kotlin.fir.declarations.FirDeclarationOrigin
+import org.jetbrains.kotlin.fir.deserialization.*
+import org.jetbrains.kotlin.fir.isNewPlaceForBodyGeneration
+import org.jetbrains.kotlin.fir.languageVersionSettings
+import org.jetbrains.kotlin.fir.scopes.FirKotlinScopeProvider
+import org.jetbrains.kotlin.fir.symbols.SymbolInternals
+import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.library.MetadataLibrary
+import org.jetbrains.kotlin.library.metadata.*
+import org.jetbrains.kotlin.library.metadata.resolver.KotlinResolvedLibrary
+import org.jetbrains.kotlin.metadata.ProtoBuf
+import org.jetbrains.kotlin.metadata.deserialization.NameResolver
+import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl
+import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
+import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedContainerSource
+import org.jetbrains.kotlin.serialization.deserialization.getClassId
+import org.jetbrains.kotlin.utils.SmartList
+import java.nio.file.Paths
+
+abstract class MetadataLibraryBasedSymbolProvider<L : MetadataLibrary>(
+    session: FirSession,
+    moduleDataProvider: ModuleDataProvider,
+    kotlinScopeProvider: FirKotlinScopeProvider,
+    defaultDeserializationOrigin: FirDeclarationOrigin = FirDeclarationOrigin.Library
+) : AbstractFirDeserializedSymbolProvider(
+    session, moduleDataProvider, kotlinScopeProvider, defaultDeserializationOrigin, KlibMetadataSerializerProtocol
+) {
+    protected abstract fun moduleData(library: L): FirModuleData?
+
+    protected abstract val fragmentNamesInLibraries: Map<String, List<L>>
+
+    protected abstract val knownPackagesInLibraries: Set<FqName>
+
+    private val annotationDeserializer = KlibBasedAnnotationDeserializer(session)
+    private val constDeserializer = FirConstDeserializer(session, KlibMetadataSerializerProtocol)
+    protected val deserializationConfiguration = CompilerDeserializationConfiguration(session.languageVersionSettings)
+    private val cachedFragments = mutableMapOf<L, MutableMap<Pair<String, String>, ProtoBuf.PackageFragment>>()
+
+    private fun getPackageFragment(
+        resolvedLibrary: L, packageStringName: String, packageMetadataPart: String
+    ): ProtoBuf.PackageFragment {
+        return cachedFragments.getOrPut(resolvedLibrary) {
+            mutableMapOf()
+        }.getOrPut(packageStringName to packageMetadataPart) {
+            parsePackageFragment(resolvedLibrary.packageMetadata(packageStringName, packageMetadataPart))
+        }
+    }
+
+    override fun computePackagePartsInfos(packageFqName: FqName): List<PackagePartsCacheData> {
+        val packageStringName = if (packageFqName.isRoot) "" else packageFqName.asString()
+
+        val librariesWithFragment = fragmentNamesInLibraries[packageStringName] ?: return emptyList()
+
+        return librariesWithFragment.flatMap { resolvedLibrary ->
+            resolvedLibrary.packageMetadataParts(packageStringName).mapNotNull {
+                val fragment = getPackageFragment(resolvedLibrary, packageStringName, it)
+
+                val moduleData = moduleData(resolvedLibrary) ?: return@mapNotNull null
+                val packageProto = fragment.`package`
+
+                val nameResolver = NameResolverImpl(
+                    fragment.strings,
+                    fragment.qualifiedNames,
+                )
+
+                PackagePartsCacheData(
+                    packageProto,
+                    FirDeserializationContext.createForPackage(
+                        packageFqName, packageProto, nameResolver, moduleData,
+                        annotationDeserializer,
+                        constDeserializer,
+                        createDeserializedContainerSource(resolvedLibrary, packageFqName),
+                    ),
+                )
+            }
+        }
+    }
+
+    override fun computePackageSetWithNonClassDeclarations(): Set<String> = fragmentNamesInLibraries.keys
+
+    override fun knownTopLevelClassesInPackage(packageFqName: FqName): Set<String> =
+        buildSet {
+            forEachFragmentInPackage(packageFqName) { _, fragment, nameResolver ->
+                for (classNameId in fragment.getExtension(KlibMetadataProtoBuf.className).orEmpty()) {
+                    add(nameResolver.getClassId(classNameId).shortClassName.asString())
+                }
+            }
+        }
+
+    @OptIn(SymbolInternals::class)
+    override fun extractClassMetadata(classId: ClassId, parentContext: FirDeserializationContext?): ClassMetadataFindResult? {
+        forEachFragmentInPackage(classId.packageFqName) { resolvedLibrary, fragment, nameResolver ->
+            val finder = KlibMetadataClassDataFinder(fragment, nameResolver)
+            val classProto = finder.findClassData(classId)?.classProto ?: return@forEachFragmentInPackage
+
+            val moduleData = moduleData(resolvedLibrary) ?: return null
+
+            return ClassMetadataFindResult.NoMetadata { symbol ->
+                val source = createDeserializedContainerSource(
+                    resolvedLibrary,
+                    classId.packageFqName
+                )
+
+                deserializeClassToSymbol(
+                    classId,
+                    classProto,
+                    symbol,
+                    nameResolver,
+                    session,
+                    moduleData,
+                    annotationDeserializer,
+                    kotlinScopeProvider,
+                    KlibMetadataSerializerProtocol,
+                    parentContext,
+                    source,
+                    origin = defaultDeserializationOrigin,
+                    deserializeNestedClass = this::getClass,
+                )
+                symbol.fir.isNewPlaceForBodyGeneration = isNewPlaceForBodyGeneration(classProto)
+            }
+        }
+
+        return null
+    }
+
+    private inline fun forEachFragmentInPackage(
+        packageFqName: FqName,
+        f: (L, ProtoBuf.PackageFragment, NameResolver) -> Unit
+    ) {
+        val packageStringName = packageFqName.asString()
+
+        val librariesWithFragment = fragmentNamesInLibraries[packageStringName] ?: return
+
+        for (resolvedLibrary in librariesWithFragment) {
+            for (packageMetadataPart in resolvedLibrary.packageMetadataParts(packageStringName)) {
+
+                val fragment = getPackageFragment(resolvedLibrary, packageStringName, packageMetadataPart)
+
+                val nameResolver = NameResolverImpl(
+                    fragment.strings,
+                    fragment.qualifiedNames,
+                )
+
+                f(resolvedLibrary, fragment, nameResolver)
+            }
+        }
+    }
+
+    protected abstract fun createDeserializedContainerSource(
+        resolvedLibrary: L,
+        packageFqName: FqName
+    ): DeserializedContainerSource?
+
+    override fun isNewPlaceForBodyGeneration(classProto: ProtoBuf.Class) = false
+
+    override fun getPackage(fqName: FqName): FqName? {
+        return if (fqName in knownPackagesInLibraries) {
+            fqName
+        } else {
+            null
+        }
+    }
+}