[KLIB tool] Introduce new command: dump-ir-signatures

^KT-62341
2023-10-03 10:58:12 +02:00
parent 447b0eddac
commit 4b2776e126
3 changed files with 249 additions and 2 deletions
@@ -0,0 +1,167 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.cli.klib
+
+import org.jetbrains.kotlin.backend.common.serialization.*
+import org.jetbrains.kotlin.backend.common.serialization.IrLibraryFileFromBytes.Companion.extensionRegistryLite
+import org.jetbrains.kotlin.backend.common.serialization.encodings.BinarySymbolData
+import org.jetbrains.kotlin.backend.common.serialization.proto.IrDeclaration.DeclaratorCase.*
+import org.jetbrains.kotlin.ir.util.IdSignature
+import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.library.impl.IrArrayMemoryReader
+import org.jetbrains.kotlin.metadata.ProtoBuf
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.utils.addToStdlib.ifTrue
+import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile as ProtoFile
+import org.jetbrains.kotlin.backend.common.serialization.proto.IrDeclaration as ProtoDeclaration
+import org.jetbrains.kotlin.backend.common.serialization.proto.IrDeclarationBase as ProtoDeclarationBase
+import org.jetbrains.kotlin.backend.common.serialization.proto.IrClass as ProtoClass
+import org.jetbrains.kotlin.backend.common.serialization.proto.IrTypeAlias as ProtoTypeAlias
+import org.jetbrains.kotlin.backend.common.serialization.proto.IrFunctionBase as ProtoFunctionBase
+import org.jetbrains.kotlin.backend.common.serialization.proto.IrProperty as ProtoProperty
+
+internal class IrSignaturesExtractor(private val library: KotlinLibrary) {
+    data class Signatures(
+            val declaredSignatures: Set<IdSignature>,
+            val importedSignatures: Set<IdSignature>
+    )
+
+    private val interner = IrInterningService()
+
+    private inner class IrSignatureExtractorFromFile(
+            private val fileIndex: Int,
+            private val allKnownSignatures: MutableSet<IdSignature>,
+            private val ownDeclarationSignatures: OwnDeclarationSignatures
+    ) {
+        private val fileProto = ProtoFile.parseFrom(library.file(fileIndex).codedInputStream, extensionRegistryLite)
+        private val fileReader = IrLibraryFileFromBytes(IrKlibBytesSource(library, fileIndex))
+
+        private val signatureDeserializer: IdSignatureDeserializer = run {
+            val packageFQN = fileReader.deserializeFqName(fileProto.fqNameList)
+            val fileName = if (fileProto.hasFileEntry() && fileProto.fileEntry.hasName()) fileProto.fileEntry.name else "<unknown>"
+
+            val fileSignature = IdSignature.FileSignature(
+                    id = Any(), // Just an unique object.
+                    fqName = FqName(packageFQN),
+                    fileName = fileName
+            )
+            IdSignatureDeserializer(fileReader, fileSignature, interner)
+        }
+
+        fun extract() {
+            collectAllKnownSignatures()
+            collectSignaturesFromDeclarations()
+        }
+
+        private fun collectAllKnownSignatures() {
+            val maxSignatureIndex = IrArrayMemoryReader(library.signatures(fileIndex)).entryCount() - 1 // Index of the latest signature in the current file.
+            (0..maxSignatureIndex).mapTo(allKnownSignatures, signatureDeserializer::deserializeIdSignature)
+        }
+
+        private fun collectSignaturesFromDeclarations() {
+            for (topLevelDeclarationIndex in fileProto.declarationIdList) {
+                extractSignature(declarationProto = fileReader.declaration(topLevelDeclarationIndex), isParentPrivate = false)
+            }
+        }
+
+        private fun extractSignature(declarationProto: ProtoDeclaration, isParentPrivate: Boolean) {
+            when (declarationProto.declaratorCase) {
+                IR_CLASS -> extractSignatureFromClass(declarationProto.irClass, isParentPrivate)
+                IR_CONSTRUCTOR -> extractSignatureFromFunction(declarationProto.irConstructor.base, isParentPrivate)
+                IR_FUNCTION -> extractSignatureFromFunction(declarationProto.irFunction.base, isParentPrivate)
+                IR_PROPERTY -> extractSignatureFromProperty(declarationProto.irProperty, isParentPrivate)
+                IR_TYPE_ALIAS -> extractSignatureFromTypeAlias(declarationProto.irTypeAlias, isParentPrivate)
+                IR_ENUM_ENTRY -> extractSignatureFromSymbol(declarationProto.irEnumEntry.base.symbol, isPrivate = isParentPrivate)
+                IR_ANONYMOUS_INIT -> extractSignatureFromPrivateDeclaration(declarationProto.irAnonymousInit.base)
+                IR_TYPE_PARAMETER -> extractSignatureFromPrivateDeclaration(declarationProto.irTypeParameter.base)
+                IR_FIELD -> extractSignatureFromPrivateDeclaration(declarationProto.irField.base)
+                IR_VARIABLE -> extractSignatureFromPrivateDeclaration(declarationProto.irVariable.base)
+                IR_VALUE_PARAMETER -> extractSignatureFromPrivateDeclaration(declarationProto.irValueParameter.base)
+                IR_LOCAL_DELEGATED_PROPERTY -> extractSignatureFromPrivateDeclaration(declarationProto.irLocalDelegatedProperty.base)
+                IR_ERROR_DECLARATION, DECLARATOR_NOT_SET, null -> Unit
+            }
+        }
+
+        private fun extractSignatureFromClass(classProto: ProtoClass, isParentPrivate: Boolean) {
+            val isPrivate = isParentPrivate || classProto.base.isPrivate()
+            extractSignatureFromSymbol(classProto.base.symbol, isPrivate)
+            classProto.declarationList.forEach { extractSignature(it, isPrivate) }
+        }
+
+        private fun extractSignatureFromFunction(functionProto: ProtoFunctionBase, isParentPrivate: Boolean) {
+            val isPrivate = isParentPrivate || functionProto.base.isPrivate()
+            extractSignatureFromSymbol(functionProto.base.symbol, isPrivate)
+        }
+
+        private fun extractSignatureFromProperty(propertyProto: ProtoProperty, isParentPrivate: Boolean) {
+            val isPrivate = isParentPrivate || propertyProto.base.isPrivate()
+            extractSignatureFromSymbol(propertyProto.base.symbol, isPrivate)
+            propertyProto.hasGetter().ifTrue { extractSignatureFromFunction(propertyProto.getter.base, isPrivate) }
+            propertyProto.hasSetter().ifTrue { extractSignatureFromFunction(propertyProto.setter.base, isPrivate) }
+            propertyProto.hasBackingField().ifTrue { extractSignatureFromPrivateDeclaration(propertyProto.backingField.base) }
+        }
+
+        private fun extractSignatureFromTypeAlias(typeAliasProto: ProtoTypeAlias, isParentPrivate: Boolean) {
+            val isPrivate = isParentPrivate || typeAliasProto.base.isPrivate()
+            extractSignatureFromSymbol(typeAliasProto.base.symbol, isPrivate)
+        }
+
+        private fun extractSignatureFromPrivateDeclaration(declarationProto: ProtoDeclarationBase) {
+            extractSignatureFromSymbol(declarationProto.symbol, isPrivate = true)
+        }
+
+        private fun extractSignatureFromSymbol(symbolId: Long, isPrivate: Boolean) {
+            val signatureId = BinarySymbolData.decode(symbolId).signatureId
+            val signature = signatureDeserializer.deserializeIdSignature(signatureId)
+            ownDeclarationSignatures[signature] = !isPrivate
+        }
+
+        private fun ProtoDeclarationBase.isPrivate(): Boolean =
+                when (IrFlags.VISIBILITY.get(flags.toInt())) {
+                    ProtoBuf.Visibility.PUBLIC,
+                    ProtoBuf.Visibility.PROTECTED,
+                    ProtoBuf.Visibility.INTERNAL -> false
+                    ProtoBuf.Visibility.PRIVATE,
+                    ProtoBuf.Visibility.PRIVATE_TO_THIS,
+                    ProtoBuf.Visibility.LOCAL,
+                    null -> true
+                }
+    }
+
+    fun extract(): Signatures {
+        val allKnownSignatures: MutableSet<IdSignature> = hashSetOf()
+        val ownDeclarationSignatures: OwnDeclarationSignatures = hashMapOf()
+
+        for (fileIndex in 0 until library.fileCount()) {
+            IrSignatureExtractorFromFile(fileIndex, allKnownSignatures, ownDeclarationSignatures).extract()
+        }
+
+        val outermostOwnDeclarationSignatures: MutableSet<IdSignature> = hashSetOf()
+        ownDeclarationSignatures.keys.forEach { signature ->
+            outermostOwnDeclarationSignatures += signature.getOutermostSignature()
+            if (signature is IdSignature.CompositeSignature && signature.container is IdSignature.FileSignature)
+                outermostOwnDeclarationSignatures += signature.inner
+        }
+
+        val importedSignatures = allKnownSignatures.filterTo(hashSetOf()) { signature ->
+            if (signature.isLocal || !signature.isPubliclyVisible)
+                false
+            else {
+                val outermostSignature = signature.getOutermostSignature()
+                outermostSignature !is IdSignature.FileSignature && outermostSignature !in outermostOwnDeclarationSignatures
+            }
+        }
+
+        return Signatures(
+                declaredSignatures = ownDeclarationSignatures.entries.mapNotNullTo(hashSetOf()) { (signature, isPublic) -> signature.takeIf { isPublic } },
+                importedSignatures = importedSignatures
+        )
+    }
+
+    private fun IdSignature.getOutermostSignature() = if (hasTopLevel) topLevelSignature() else this
+}
+
+private typealias OwnDeclarationSignatures = MutableMap<IdSignature, /* isPublic? */ Boolean>
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.cli.klib
+
+import org.jetbrains.kotlin.ir.util.IdSignature
+import org.jetbrains.kotlin.ir.util.IdSignatureRenderer
+import org.jetbrains.kotlin.ir.util.render
+import org.jetbrains.kotlin.library.KotlinIrSignatureVersion
+
+internal class IrSignaturesRenderer(private val output: Appendable, signatureVersion: KotlinIrSignatureVersion?) {
+    private val individualSignatureRenderer = when (signatureVersion) {
+        KotlinIrSignatureVersion.V1 -> IdSignatureRenderer.LEGACY
+        null, KotlinIrSignatureVersion.V2 -> IdSignatureRenderer.DEFAULT
+        else -> error("Unsupported signature version: ${signatureVersion.number}")
+    }
+
+    fun render(signatures: IrSignaturesExtractor.Signatures) {
+        header("Declared signatures: ${signatures.declaredSignatures.size}")
+        sortedSignatures(signatures.declaredSignatures)
+        blankLine()
+
+        header("Imported signatures: ${signatures.importedSignatures.size}")
+        sortedSignatures(signatures.importedSignatures)
+    }
+
+    private fun header(subject: String) {
+        output.append("// ").append(subject).append('\n')
+    }
+
+    private fun blankLine() {
+        output.append('\n')
+    }
+
+    private fun sortedSignatures(signatures: Set<IdSignature>) {
+        signatures.asSequence()
+                .map { signature -> signature.render(individualSignatureRenderer) }
+                .sorted()
+                .forEach { signatureText -> output.append(signatureText).append('\n') }
+    }
+}
@@ -63,7 +63,10 @@ fun printUsage() {
               remove                Remove the library from the local repository
               dump-ir               Dump the intermediate representation (IR) of all declarations in the library
                                       (to be used for debugging purposes only)
-               signatures            Dump IR signatures of all public declarations in the library
+               dump-ir-signatures    Dump IR signatures of all public declarations in the library and all public declarations consumed
+                                       by this library (as two separate lists). This command relies purely on the data in IR.
+               signatures            [DEPRECATED] Please, use "dump-ir-signatures" instead.
+                                       Dump IR signatures of all public declarations in the library
                                       Note that this command renders the signatures from the metadata. Signatures for certain
                                       declarations in the metadata and in IR may differ if compiler plugins (such as Compose)
                                       were applied during library compilation.
@@ -71,6 +74,9 @@ fun printUsage() {

            and the options are:
               -repository <path>    Work with the specified repository
+               -signature-version {${KotlinIrSignatureVersion.CURRENTLY_SUPPORTED_VERSIONS.joinToString("|") { it.number.toString() }}}
+                                     Render IR signatures of a specific version. By default, the most up-to-date signature version
+                                       supported in the library is used.
               -print-signatures {true|false}
                                     Print IR signature for every declaration (only for "contents" and "dump-ir" commands)
            """.trimIndent()
@@ -104,7 +110,18 @@ class Command(args: Array<String>) {

    val verb = args[0]
    val library = args[1]
-    val options = parseArgs(args.drop(2).toTypedArray())
+    val options: Map<String, List<String>> = parseArgs(args.drop(2).toTypedArray())
+}
+
+private fun Command.parseSignatureVersion(): KotlinIrSignatureVersion? {
+    val rawSignatureVersion = options["-signature-version"]?.last() ?: return null
+    val signatureVersion = rawSignatureVersion.toIntOrNull()?.let(::KotlinIrSignatureVersion)
+            ?: error("Invalid signature version: $rawSignatureVersion")
+
+    if (signatureVersion !in KotlinIrSignatureVersion.CURRENTLY_SUPPORTED_VERSIONS)
+        error("Unsupported signature version: ${signatureVersion.number}")
+
+    return signatureVersion
 }

 fun warn(text: String) {
@@ -264,12 +281,29 @@ class Library(val libraryNameOrPath: String, val requestedRepository: String?) {
    }

    fun signatures(output: Appendable) {
+        output.append("WARNING: \"signatures\" is deprecated. Please, use \"dump-ir-signatures\" instead.\n\n")
+
        val module = loadModule()
        val printer = SignaturePrinter(output, DefaultKlibSignatureRenderer())

        printer.print(module)
    }

+    fun dumpIrSignatures(output: Appendable, signatureVersion: KotlinIrSignatureVersion?) {
+        val library = libraryInCurrentDir(libraryNameOrPath)
+        signatureVersion?.checkSupportedInLibrary(library)
+
+        val signatures = IrSignaturesExtractor(library).extract()
+        IrSignaturesRenderer(output, signatureVersion).render(signatures)
+    }
+
+    private fun KotlinIrSignatureVersion.checkSupportedInLibrary(library: KotlinLibrary) {
+        val supportedSignatureVersions = library.versions.irSignatureVersions
+        if (this !in supportedSignatureVersions)
+            error("Signature version ${this.number} is not supported in library ${library.libraryFile}." +
+                    " Supported versions: ${supportedSignatureVersions.joinToString { it.number.toString() }}")
+    }
+
    private fun loadModule(): ModuleDescriptor {
        val storageManager = LockBasedStorageManager("klib")
        val library = libraryInRepoOrCurrentDir(repository, libraryNameOrPath)
@@ -314,10 +348,13 @@ fun main(args: Array<String>) {
    val repository = command.options["-repository"]?.last()
    val printSignatures = command.options["-print-signatures"]?.last()?.toBoolean() == true

+    val signatureVersion = command.parseSignatureVersion()
+
    val library = Library(command.library, repository)

    when (command.verb) {
        "dump-ir" -> library.ir(System.out, printSignatures)
+        "dump-ir-signatures" -> library.dumpIrSignatures(System.out, signatureVersion)
        "contents" -> library.contents(System.out, printSignatures)
        "signatures" -> library.signatures(System.out)
        "info" -> library.info()