[klib] Add an option to write out header klibs

The header klib is supposed to only contain the public abi of the module similar to jvm-abi-gen. It is intended to be used as a dependency for other klib compilations instead of the full klib for compilation avoidance. ^KT-60807
2023-05-09 14:29:21 +02:00
parent 829e0675f4
commit 0a612e4268
90 changed files with 1374 additions and 76 deletions
@@ -27,6 +27,7 @@ import org.jetbrains.kotlin.library.impl.IrMemoryStringWriter
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.types.Variance
 import org.jetbrains.kotlin.utils.filterIsInstanceAnd
+import org.jetbrains.kotlin.utils.addToStdlib.applyIf
 import java.io.File
 import org.jetbrains.kotlin.backend.common.serialization.proto.AccessorIdSignature as ProtoAccessorIdSignature
 import org.jetbrains.kotlin.backend.common.serialization.proto.CommonIdSignature as ProtoCommonIdSignature
@@ -118,6 +119,7 @@ open class IrFileSerializer(
    private val languageVersionSettings: LanguageVersionSettings,
    private val bodiesOnlyForInlines: Boolean = false,
    private val normalizeAbsolutePaths: Boolean = false,
+    private val skipPrivateApi: Boolean = false,
    private val sourceBaseDirs: Collection<String>
 ) {
    private val loopIndex = hashMapOf<IrLoop, Int>()
@@ -183,7 +185,11 @@ open class IrFileSerializer(
    private fun serializeIrStatementOrigin(origin: IrStatementOrigin): Int =
        serializeString((origin as? IrStatementOriginImpl)?.debugName ?: error("Unable to serialize origin ${origin.javaClass.name}"))

-    private fun serializeCoordinates(start: Int, end: Int): Long = BinaryCoordinates.encode(start, end)
+    private fun serializeCoordinates(start: Int, end: Int): Long = if (skipPrivateApi) {
+        0L
+    } else {
+        BinaryCoordinates.encode(start, end)
+    }

    /* ------- Strings ---------------------------------------------------------- */

@@ -1340,9 +1346,9 @@ open class IrFileSerializer(

 // ---------- Top level ------------------------------------------------------

-    private fun serializeFileEntry(entry: IrFileEntry): ProtoFileEntry = ProtoFileEntry.newBuilder()
+    private fun serializeFileEntry(entry: IrFileEntry, includeLineStartOffsets: Boolean = true): ProtoFileEntry = ProtoFileEntry.newBuilder()
        .setName(entry.matchAndNormalizeFilePath())
-        .addAllLineStartOffset(entry.lineStartOffsets.asIterable())
+        .applyIf(includeLineStartOffsets) { addAllLineStartOffset(entry.lineStartOffsets.asIterable()) }
        .build()

    open fun backendSpecificExplicitRoot(node: IrAnnotationContainer): Boolean = false
@@ -1351,12 +1357,18 @@ open class IrFileSerializer(
    open fun backendSpecificSerializeAllMembers(irClass: IrClass) = false
    open fun backendSpecificMetadata(irFile: IrFile): FileBackendSpecificMetadata? = null

+    private fun skipIfPrivate(declaration: IrDeclaration) =
+        skipPrivateApi && (declaration as? IrDeclarationWithVisibility)?.visibility?.isPublicAPI != true
+
    open fun memberNeedsSerialization(member: IrDeclaration): Boolean {
        val parent = member.parent
        require(parent is IrClass)
        if (backendSpecificSerializeAllMembers(parent)) return true
        if (bodiesOnlyForInlines && member is IrAnonymousInitializer && parent.visibility != DescriptorVisibilities.LOCAL)
            return false
+        if (skipIfPrivate(member)) {
+            return false
+        }

        return (!member.isFakeOverride)
    }
@@ -1414,11 +1426,15 @@ open class IrFileSerializer(
        val topLevelDeclarations = mutableListOf<SerializedDeclaration>()

        val proto = ProtoFile.newBuilder()
-            .setFileEntry(serializeFileEntry(file.fileEntry))
+            .setFileEntry(serializeFileEntry(file.fileEntry, includeLineStartOffsets = !skipPrivateApi))
            .addAllFqName(serializeFqName(file.packageFqName.asString()))
            .addAllAnnotation(serializeAnnotations(file.annotations))

        file.declarations.forEach {
+            if (skipIfPrivate(it)) {
+                // Skip the declaration if producing header klib and the declaration is not public.
+                return@forEach
+            }
            if (it.descriptor.isExpectMember && !it.descriptor.isSerializableExpectClass) {
                // Skip the declaration unless it is `expect annotation class` marked with `OptionalExpectation`
                // without the corresponding `actual` counterpart for the current leaf target.
@@ -1441,7 +1457,7 @@ open class IrFileSerializer(

        // Make sure that all top level properties are initialized on library's load.
        file.declarations
-            .filterIsInstanceAnd<IrProperty> { it.backingField?.initializer != null && keepOrderOfProperties(it) }
+            .filterIsInstanceAnd<IrProperty> { it.backingField?.initializer != null && keepOrderOfProperties(it) && !skipIfPrivate(it) }
            .forEach {
                val fieldSymbol = it.backingField?.symbol ?: error("Not found ID ${it.render()}")
                proto.addExplicitlyExportedToCompiler(serializeIrSymbol(fieldSymbol))
@@ -27,8 +27,9 @@ class KlibMetadataMonolithicSerializer(
    exportKDoc: Boolean,
    skipExpects: Boolean,
    includeOnlyModuleContent: Boolean = false,
-    allowErrorTypes: Boolean = false
-) : KlibMetadataSerializer(languageVersionSettings, metadataVersion, project, exportKDoc, skipExpects, includeOnlyModuleContent, allowErrorTypes) {
+    allowErrorTypes: Boolean = false,
+    produceHeaderKlib: Boolean = false,
+) : KlibMetadataSerializer(languageVersionSettings, metadataVersion, project, exportKDoc, skipExpects, includeOnlyModuleContent, allowErrorTypes, produceHeaderKlib) {

    private fun serializePackageFragment(fqName: FqName, module: ModuleDescriptor): List<ProtoBuf.PackageFragment> {

@@ -18,7 +18,7 @@ import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
-import org.jetbrains.kotlin.resolve.descriptorUtil.fqNameSafe
+import org.jetbrains.kotlin.resolve.descriptorUtil.*
 import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter
 import org.jetbrains.kotlin.serialization.ApproximatingStringTable
 import org.jetbrains.kotlin.serialization.DescriptorSerializer
@@ -36,7 +36,8 @@ abstract class KlibMetadataSerializer(
    val exportKDoc: Boolean = false,
    val skipExpects: Boolean = false,
    val includeOnlyModuleContent: Boolean = false,
-    private val allowErrorTypes: Boolean
+    private val allowErrorTypes: Boolean,
+    val produceHeaderKlib: Boolean = false,
 ) {

    lateinit var serializerContext: SerializerContext
@@ -54,7 +55,8 @@ abstract class KlibMetadataSerializer(
            metadataVersion,
            ApproximatingStringTable(),
            allowErrorTypes,
-            exportKDoc
+            exportKDoc,
+            produceHeaderKlib
        )
        return SerializerContext(
            extension,
@@ -94,8 +96,8 @@ abstract class KlibMetadataSerializer(
    // TODO: we filter out expects with present actuals.
    // This is done because deserialized member scope doesn't give us actuals
    // when it has a choice
-    private fun List<DeclarationDescriptor>.filterOutExpectsWithActuals(): List<DeclarationDescriptor> {
-        val actualClassIds = this.filter{ !it.isExpectMember }.map { ClassId.topLevel(it.fqNameSafe) }
+    private fun Sequence<DeclarationDescriptor>.filterOutExpectsWithActuals(): Sequence<DeclarationDescriptor> {
+        val actualClassIds = this.filter { !it.isExpectMember }.map { ClassId.topLevel(it.fqNameSafe) }
        return this.filterNot {
            // TODO: this only filters classes for now.
            // Need to do the same for functions etc
@@ -103,12 +105,20 @@ abstract class KlibMetadataSerializer(
        }
    }

-    protected fun List<DeclarationDescriptor>.filterOutExpects(): List<DeclarationDescriptor> =
+    private fun Sequence<DeclarationDescriptor>.filterOutExpects(): Sequence<DeclarationDescriptor> =
        if (skipExpects)
            this.filterNot { it.isExpectMember && !it.isSerializableExpectClass }
        else
            this.filterOutExpectsWithActuals()

+    private fun Sequence<DeclarationDescriptor>.filterPrivate(): Sequence<DeclarationDescriptor> =
+        if (produceHeaderKlib) {
+            // We keep all interfaces since publicly accessible classes can inherit from private interfaces.
+            this.filter {
+                it is ClassDescriptor && it.kind.isInterface || it is DeclarationDescriptorWithVisibility && it.effectiveVisibility().publicApi
+            }
+        } else this
+
    private fun serializeClasses(packageName: FqName,
                                 //builder: ProtoBuf.PackageFragment.Builder,
                                 descriptors: Collection<DeclarationDescriptor>): List<Pair<ProtoBuf.Class, Int>> {
@@ -131,8 +141,8 @@ abstract class KlibMetadataSerializer(
        allTopLevelDescriptors: List<DeclarationDescriptor>
    ): List<ProtoBuf.PackageFragment> {

-        val classifierDescriptors = allClassifierDescriptors.filterOutExpects()
-        val topLevelDescriptors = allTopLevelDescriptors.filterOutExpects()
+        val classifierDescriptors = allClassifierDescriptors.asSequence().filterOutExpects().filterPrivate().toList()
+        val topLevelDescriptors = allTopLevelDescriptors.asSequence().filterOutExpects().filterPrivate().toList()

        if (TOP_LEVEL_CLASS_DECLARATION_COUNT_PER_FILE == null &&
            TOP_LEVEL_DECLARATION_COUNT_PER_FILE == null) {
@@ -5,7 +5,6 @@

 package org.jetbrains.kotlin.backend.common.serialization.metadata

-import org.jetbrains.kotlin.config.LanguageFeature
 import org.jetbrains.kotlin.config.LanguageVersionSettings
 import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.library.metadata.KlibMetadataProtoBuf
@@ -15,8 +14,10 @@ import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
 import org.jetbrains.kotlin.metadata.serialization.MutableVersionRequirementTable
 import org.jetbrains.kotlin.psi.KtDeclaration
 import org.jetbrains.kotlin.psi.KtPrimaryConstructor
+import org.jetbrains.kotlin.resolve.DescriptorUtils
 import org.jetbrains.kotlin.resolve.source.getPsi
 import org.jetbrains.kotlin.serialization.DescriptorSerializer
+import org.jetbrains.kotlin.serialization.DescriptorSerializer.Companion.sort
 import org.jetbrains.kotlin.serialization.KotlinSerializerExtensionBase
 import org.jetbrains.kotlin.serialization.StringTableImpl
 import org.jetbrains.kotlin.serialization.deserialization.DYNAMIC_TYPE_DESERIALIZER_ID
@@ -28,9 +29,22 @@ class KlibMetadataSerializerExtension(
    override val metadataVersion: BinaryVersion,
    override val stringTable: StringTableImpl,
    private val allowErrorTypes: Boolean,
-    private val exportKDoc: Boolean
+    private val exportKDoc: Boolean,
+    private val produceHeaderKlib: Boolean
 ) : KotlinSerializerExtensionBase(KlibMetadataSerializerProtocol) {
    override fun shouldUseTypeTable(): Boolean = true
+    override val customClassMembersProducer: ClassMembersProducer?
+        get() = if (produceHeaderKlib)
+            object : ClassMembersProducer {
+                override fun getCallableMembers(classDescriptor: ClassDescriptor) =
+                    sort(
+                        DescriptorUtils.getAllDescriptors(classDescriptor.defaultType.memberScope)
+                            .filterIsInstance<CallableMemberDescriptor>()
+                            .filter { it.kind != CallableMemberDescriptor.Kind.FAKE_OVERRIDE }
+                            .filter { it.visibility.isPublicAPI }
+                    )
+            }
+        else super.customClassMembersProducer

    private fun descriptorFileId(descriptor: DeclarationDescriptorWithSource): Int? {
        val fileName = descriptor.source.containingFile.name ?: return null