JVM Metadata: Add a bit to mark anonymous objects in inline functions

Anonymous objects in the scope of an inline functions are copied to
all call-sites. This makes them part of the public ABI of a Kotlin
library.

We introduce a flag to mark all classes in the scope of an inline
function. When compiling with assertions enabled, we check that
this flag is set whenever we inline an anonymous object from another
module.

compiler/backend/src/org/jetbrains/kotlin/codegen/ClosureCodegen.java

@@ -34,6 +34,7 @@ import org.jetbrains.kotlin.psi.KtElement;
 import org.jetbrains.kotlin.resolve.DescriptorUtils;
 import org.jetbrains.kotlin.resolve.calls.model.ResolvedCall;
 import org.jetbrains.kotlin.resolve.descriptorUtil.DescriptorUtilsKt;
+import org.jetbrains.kotlin.resolve.inline.InlineUtil;
 import org.jetbrains.kotlin.resolve.jvm.diagnostics.JvmDeclarationOriginKt;
 import org.jetbrains.kotlin.resolve.scopes.MemberScope;
 import org.jetbrains.kotlin.resolve.scopes.receivers.TransientReceiver;
@@ -287,7 +288,9 @@ public class ClosureCodegen extends MemberCodegen<KtElement> {
         if (builder == null) return;
         ProtoBuf.Function functionProto = builder.build();
 
-        WriteAnnotationUtilKt.writeKotlinMetadata(v, state, KotlinClassHeader.Kind.SYNTHETIC_CLASS, 0, av -> {
+        boolean publicAbi = InlineUtil.isInPublicInlineScope(frontendFunDescriptor);
+
+        WriteAnnotationUtilKt.writeKotlinMetadata(v, state, KotlinClassHeader.Kind.SYNTHETIC_CLASS, publicAbi, 0, av -> {
             writeAnnotationData(av, serializer, functionProto);
             return Unit.INSTANCE;
         });

compiler/backend/src/org/jetbrains/kotlin/codegen/CodeFragmentCodegen.kt

@@ -79,7 +79,7 @@ class CodeFragmentCodegen private constructor(
     }
 
     override fun generateKotlinMetadataAnnotation() {
-        writeSyntheticClassMetadata(v, state)
+        writeSyntheticClassMetadata(v, state, false)
     }
 
     private fun genConstructor() {

compiler/backend/src/org/jetbrains/kotlin/codegen/ImplementationBodyCodegen.java

@@ -43,6 +43,7 @@ import org.jetbrains.kotlin.resolve.calls.callUtil.CallUtilKt;
 import org.jetbrains.kotlin.resolve.calls.model.ResolvedCall;
 import org.jetbrains.kotlin.resolve.calls.model.VariableAsFunctionResolvedCall;
 import org.jetbrains.kotlin.resolve.descriptorUtil.DescriptorUtilsKt;
+import org.jetbrains.kotlin.resolve.inline.InlineUtil;
 import org.jetbrains.kotlin.resolve.jvm.annotations.JvmAnnotationUtilKt;
 import org.jetbrains.kotlin.resolve.jvm.diagnostics.JvmDeclarationOrigin;
 import org.jetbrains.kotlin.resolve.jvm.diagnostics.JvmDeclarationOriginKt;
@@ -304,7 +305,9 @@ public class ImplementationBodyCodegen extends ClassBodyCodegen {
     protected void generateKotlinMetadataAnnotation() {
         ProtoBuf.Class classProto = serializer.classProto(descriptor).build();
 
-        WriteAnnotationUtilKt.writeKotlinMetadata(v, state, KotlinClassHeader.Kind.CLASS, 0, av -> {
+        boolean publicAbi = InlineUtil.isInPublicInlineScope(descriptor);
+
+        WriteAnnotationUtilKt.writeKotlinMetadata(v, state, KotlinClassHeader.Kind.CLASS, publicAbi, 0, av -> {
             writeAnnotationData(av, serializer, classProto);
             return Unit.INSTANCE;
         });

compiler/backend/src/org/jetbrains/kotlin/codegen/InterfaceImplBodyCodegen.kt

@@ -141,7 +141,7 @@ class InterfaceImplBodyCodegen(
     override fun generateKotlinMetadataAnnotation() {
         (v as InterfaceImplClassBuilder).stopCounting()
 
-        writeSyntheticClassMetadata(v, state)
+        writeSyntheticClassMetadata(v, state, false)
     }
 
     override fun done() {

compiler/backend/src/org/jetbrains/kotlin/codegen/MultifileClassCodegen.kt

@@ -391,7 +391,7 @@ class MultifileClassCodegenImpl(
             facadeClassType: Type,
             kotlinPackageFqName: FqName
         ) {
-            writeKotlinMetadata(classBuilder, state, KotlinClassHeader.Kind.MULTIFILE_CLASS, flags) { av ->
+            writeKotlinMetadata(classBuilder, state, KotlinClassHeader.Kind.MULTIFILE_CLASS, false, flags) { av ->
                 val arv = av.visitArray(JvmAnnotationNames.METADATA_DATA_FIELD_NAME)
                 for (internalName in partInternalNames) {
                     arv.visit(null, internalName)

compiler/backend/src/org/jetbrains/kotlin/codegen/MultifileClassPartCodegen.kt

@@ -96,7 +96,7 @@ class MultifileClassPartCodegen(
 
         val extraFlags = if (shouldGeneratePartHierarchy) JvmAnnotationNames.METADATA_MULTIFILE_PARTS_INHERIT_FLAG else 0
 
-        writeKotlinMetadata(v, state, KotlinClassHeader.Kind.MULTIFILE_CLASS_PART, extraFlags) { av ->
+        writeKotlinMetadata(v, state, KotlinClassHeader.Kind.MULTIFILE_CLASS_PART, false, extraFlags) { av ->
             DescriptorAsmUtil.writeAnnotationData(av, serializer, packageProto)
             av.visit(JvmAnnotationNames.METADATA_MULTIFILE_CLASS_NAME_FIELD_NAME, facadeClassType.internalName)
 

compiler/backend/src/org/jetbrains/kotlin/codegen/PackagePartCodegen.java

@@ -107,7 +107,7 @@ public class PackagePartCodegen extends MemberCodegen<KtFile> {
     protected void generateKotlinMetadataAnnotation() {
         Pair<DescriptorSerializer, ProtoBuf.Package> serializedPart = serializePackagePartMembers(this, packagePartType);
 
-        WriteAnnotationUtilKt.writeKotlinMetadata(v, state, KotlinClassHeader.Kind.FILE_FACADE, 0, av -> {
+        WriteAnnotationUtilKt.writeKotlinMetadata(v, state, KotlinClassHeader.Kind.FILE_FACADE, false, 0, av -> {
             writeAnnotationData(av, serializedPart.getFirst(), serializedPart.getSecond());
 
             FqName kotlinPackageFqName = element.getPackageFqName();

compiler/backend/src/org/jetbrains/kotlin/codegen/PropertyReferenceCodegen.kt

@@ -30,6 +30,7 @@ import org.jetbrains.kotlin.psi.KtElement
 import org.jetbrains.kotlin.resolve.PropertyImportedFromObject
 import org.jetbrains.kotlin.resolve.descriptorUtil.builtIns
 import org.jetbrains.kotlin.resolve.descriptorUtil.getSuperClassNotAny
+import org.jetbrains.kotlin.resolve.inline.InlineUtil
 import org.jetbrains.kotlin.resolve.isUnderlyingPropertyOfInlineClass
 import org.jetbrains.kotlin.resolve.jvm.AsmTypes.*
 import org.jetbrains.kotlin.resolve.jvm.diagnostics.JvmDeclarationOrigin
@@ -209,7 +210,7 @@ class PropertyReferenceCodegen(
     }
 
     override fun generateKotlinMetadataAnnotation() {
-        writeSyntheticClassMetadata(v, state)
+        writeSyntheticClassMetadata(v, state, InlineUtil.isInPublicInlineScope(classDescriptor))
     }
 
     fun putInstanceOnStack(receiverValue: StackValue?): StackValue {

compiler/backend/src/org/jetbrains/kotlin/codegen/SamWrapperCodegen.java

@@ -138,7 +138,7 @@ public class SamWrapperCodegen {
         );
         cv.visitSource(file.getName(), null);
 
-        WriteAnnotationUtilKt.writeSyntheticClassMetadata(cv, state);
+        WriteAnnotationUtilKt.writeSyntheticClassMetadata(cv, state, isInsideInline);
 
         generateInnerClassInformation(file, asmType, cv);
 

compiler/backend/src/org/jetbrains/kotlin/codegen/ScriptCodegen.kt

@@ -69,7 +69,7 @@ class ScriptCodegen private constructor(
     override fun generateKotlinMetadataAnnotation() {
         val serializer = DescriptorSerializer.create(scriptDescriptor, JvmSerializerExtension(v.serializationBindings, state), null)
         val classProto = serializer.classProto(scriptDescriptor).build()
-        writeKotlinMetadata(v, state, KotlinClassHeader.Kind.CLASS, JvmAnnotationNames.METADATA_SCRIPT_FLAG) { av ->
+        writeKotlinMetadata(v, state, KotlinClassHeader.Kind.CLASS, false, JvmAnnotationNames.METADATA_SCRIPT_FLAG) { av ->
             DescriptorAsmUtil.writeAnnotationData(av, serializer, classProto)
         }
     }

compiler/backend/src/org/jetbrains/kotlin/codegen/binding/CodegenAnnotatingVisitor.java

@@ -1012,7 +1012,15 @@ class CodegenAnnotatingVisitor extends KtVisitorVoid {
         ClassDescriptor classDescriptor = (ClassDescriptor) type.getConstructor().getDeclarationDescriptor();
         assert classDescriptor != null : "because it's enum";
 
-        WhenByEnumsMapping mapping = new WhenByEnumsMapping(classDescriptor, currentClassName, fieldNumber);
+        boolean isPublicAbi = false;
+        for (FunctionDescriptor descriptor : functionsStack) {
+            if (descriptor.isInline()) {
+                isPublicAbi = !DescriptorVisibilities.isPrivate(descriptor.getVisibility());
+                break;
+            }
+        }
+
+        WhenByEnumsMapping mapping = new WhenByEnumsMapping(classDescriptor, currentClassName, fieldNumber, isPublicAbi);
 
         for (ConstantValue<?> constant : switchCodegenProvider.getAllConstants(expression)) {
             if (constant instanceof NullValue) continue;

compiler/backend/src/org/jetbrains/kotlin/codegen/codegenUtil.kt

@@ -654,7 +654,7 @@ private fun generateLambdaForRunSuspend(
         visitEnd()
     }
 
-    writeSyntheticClassMetadata(lambdaBuilder, state)
+    writeSyntheticClassMetadata(lambdaBuilder, state, false)
 
     lambdaBuilder.done()
     return lambdaBuilder.thisName

compiler/backend/src/org/jetbrains/kotlin/codegen/coroutines/CoroutineCodegen.kt

@@ -32,6 +32,7 @@ import org.jetbrains.kotlin.resolve.DescriptorUtils
 import org.jetbrains.kotlin.resolve.descriptorUtil.builtIns
 import org.jetbrains.kotlin.resolve.descriptorUtil.module
 import org.jetbrains.kotlin.resolve.indexOrMinusOne
+import org.jetbrains.kotlin.resolve.inline.InlineUtil
 import org.jetbrains.kotlin.resolve.isInlineClassType
 import org.jetbrains.kotlin.resolve.jvm.AsmTypes
 import org.jetbrains.kotlin.resolve.jvm.diagnostics.ErrorsJvm
@@ -742,7 +743,8 @@ class CoroutineCodegenForNamedFunction private constructor(
     }
 
     override fun generateKotlinMetadataAnnotation() {
-        writeKotlinMetadata(v, state, KotlinClassHeader.Kind.SYNTHETIC_CLASS, 0) { av ->
+        val publicAbi = InlineUtil.isInPublicInlineScope(classDescriptor)
+        writeKotlinMetadata(v, state, KotlinClassHeader.Kind.SYNTHETIC_CLASS, publicAbi, 0) { av ->
             val serializer = DescriptorSerializer.createForLambda(JvmSerializerExtension(v.serializationBindings, state))
             val functionProto =
                 serializer.functionProto(

compiler/backend/src/org/jetbrains/kotlin/codegen/inline/AnonymousObjectTransformer.kt

@@ -118,6 +118,10 @@ class AnonymousObjectTransformer(
 
             override fun visitEnd() {}
         }, ClassReader.SKIP_FRAMES)
+        val header = metadataReader.createHeader()
+        assert(isSameModule || (header != null && isPublicAbi(header))) {
+            "Trying to inline an anonymous object which is not part of the public ABI: ${oldObjectType.className}"
+        }
 
         // When regenerating objects in inline lambdas, keep the old SMAP and don't remap the line numbers to
         // save time. The result is effectively the same anyway.
@@ -198,7 +202,6 @@ class AnonymousObjectTransformer(
             visitor.visitInnerClass(node.name, node.outerName, node.innerName, node.access)
         }
 
-        val header = metadataReader.createHeader()
         if (header != null) {
             writeTransformedMetadata(header, classBuilder)
         }
@@ -229,7 +232,9 @@ class AnonymousObjectTransformer(
     }
 
     private fun writeTransformedMetadata(header: KotlinClassHeader, classBuilder: ClassBuilder) {
-        writeKotlinMetadata(classBuilder, state, header.kind, header.extraInt) action@{ av ->
+        // The transformed anonymous object becomes part of the public ABI if it is inside of a public inline function.
+        val publicAbi = inliningContext.callSiteInfo.isInPublicInlineScope
+        writeKotlinMetadata(classBuilder, state, header.kind, publicAbi, header.extraInt and JvmAnnotationNames.METADATA_PUBLIC_ABI_FLAG.inv()) action@{ av ->
             val (newProto, newStringTable) = transformMetadata(header) ?: run {
                 val data = header.data
                 val strings = header.strings
@@ -242,6 +247,11 @@ class AnonymousObjectTransformer(
         }
     }
 
+    private fun isPublicAbi(header: KotlinClassHeader): Boolean =
+        // The public abi flag was only introduced in metadata version 1.6.0, before then we have to skip this check.
+        !header.metadataVersion.isAtLeast(1, 6, 0) ||
+                header.extraInt and JvmAnnotationNames.METADATA_PUBLIC_ABI_FLAG != 0
+
     private fun transformMetadata(header: KotlinClassHeader): Pair<MessageLite, JvmStringTable>? {
         val data = header.data ?: return null
         val strings = header.strings ?: return null
@@ -317,7 +327,7 @@ class AnonymousObjectTransformer(
             InlineCallSiteInfo(
                 transformationInfo.oldClassName,
                 Method(sourceNode.name, if (isConstructor) transformationInfo.newConstructorDescriptor else sourceNode.desc),
-                inliningContext.callSiteInfo.isInlineOrInsideInline,
+                inliningContext.callSiteInfo.inlineScopeVisibility,
                 inliningContext.callSiteInfo.file,
                 inliningContext.callSiteInfo.lineNumber
             ),

compiler/backend/src/org/jetbrains/kotlin/codegen/inline/PsiSourceCompilerForInline.kt

@@ -74,7 +74,7 @@ class PsiSourceCompilerForInline(
             return InlineCallSiteInfo(
                 parentCodegen.className,
                 signature.asmMethod,
-                context.functionDescriptor.isInlineOrInsideInline(),
+                context.functionDescriptor.getInlineCallSiteVisibility(),
                 callElement.containingFile,
                 CodegenUtil.getLineNumberForElement(callElement, false) ?: 0
             )
@@ -416,8 +416,11 @@ class PsiSourceCompilerForInline(
 }
 
 fun DeclarationDescriptor.isInlineOrInsideInline(): Boolean =
-    if (this is FunctionDescriptor && isInline) true
-    else containingDeclaration?.isInlineOrInsideInline() == true
+    getInlineCallSiteVisibility() != null
+
+fun DeclarationDescriptor.getInlineCallSiteVisibility(): DescriptorVisibility? =
+    if (this is FunctionDescriptor && isInline) visibility
+    else containingDeclaration?.getInlineCallSiteVisibility()
 
 fun getDeclarationLabels(lambdaOrFun: PsiElement?, descriptor: DeclarationDescriptor): Set<String> {
     val result = HashSet<String>()

compiler/backend/src/org/jetbrains/kotlin/codegen/inline/SourceCompilerForInline.kt

@@ -10,6 +10,8 @@ import org.jetbrains.kotlin.codegen.AsmUtil
 import org.jetbrains.kotlin.codegen.inline.coroutines.FOR_INLINE_SUFFIX
 import org.jetbrains.kotlin.codegen.state.GenerationState
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
+import org.jetbrains.kotlin.descriptors.DescriptorVisibilities
+import org.jetbrains.kotlin.descriptors.DescriptorVisibility
 import org.jetbrains.kotlin.incremental.components.LocationInfo
 import org.jetbrains.kotlin.incremental.components.Position
 import org.jetbrains.kotlin.incremental.components.ScopeKind
@@ -24,10 +26,16 @@ import org.jetbrains.org.objectweb.asm.tree.MethodNode
 class InlineCallSiteInfo(
     val ownerClassName: String,
     val method: Method,
-    val isInlineOrInsideInline: Boolean,
+    val inlineScopeVisibility: DescriptorVisibility?,
     val file: PsiFile?,
     val lineNumber: Int
-)
+) {
+    val isInlineOrInsideInline: Boolean
+        get() = inlineScopeVisibility != null
+
+    val isInPublicInlineScope: Boolean
+        get() = inlineScopeVisibility != null && !DescriptorVisibilities.isPrivate(inlineScopeVisibility)
+}
 
 interface SourceCompilerForInline {
     val state: GenerationState

compiler/backend/src/org/jetbrains/kotlin/codegen/when/MappingClassesForWhenByEnumCodegen.java

@@ -57,7 +57,8 @@ public class MappingClassesForWhenByEnumCodegen {
         generateFields(cb, mappings);
         generateInitialization(cb, mappings);
 
-        WriteAnnotationUtilKt.writeSyntheticClassMetadata(cb, state);
+        boolean publicAbi = mappings.stream().anyMatch(WhenByEnumsMapping::isPublicAbi);
+        WriteAnnotationUtilKt.writeSyntheticClassMetadata(cb, state, publicAbi);
 
         cb.done();
     }

compiler/backend/src/org/jetbrains/kotlin/codegen/when/WhenByEnumsMapping.java

@@ -32,16 +32,19 @@ public class WhenByEnumsMapping {
     private final String outerClassInternalNameForExpression;
     private final String mappingsClassInternalName;
     private final int fieldNumber;
+    private final boolean isPublicAbi;
 
     public WhenByEnumsMapping(
             @NotNull ClassDescriptor enumClassDescriptor,
             @NotNull String outerClassInternalNameForExpression,
-            int fieldNumber
+            int fieldNumber,
+            boolean isPublicAbi
     ) {
         this.enumClassDescriptor = enumClassDescriptor;
         this.outerClassInternalNameForExpression = outerClassInternalNameForExpression;
         this.mappingsClassInternalName = outerClassInternalNameForExpression + MAPPINGS_CLASS_NAME_POSTFIX;
         this.fieldNumber = fieldNumber;
+        this.isPublicAbi = isPublicAbi;
     }
 
     public int getIndexByEntry(@NotNull EnumValue value) {
@@ -80,6 +83,10 @@ public class WhenByEnumsMapping {
         return mappingsClassInternalName;
     }
 
+    public boolean isPublicAbi() {
+        return isPublicAbi;
+    }
+
     @NotNull
     public Iterable<Map.Entry<EnumValue, Integer>> enumValuesToIntMapping() {
         return map.entrySet();

compiler/backend/src/org/jetbrains/kotlin/codegen/writeAnnotationUtil.kt

@@ -27,6 +27,7 @@ fun writeKotlinMetadata(
     cb: ClassBuilder,
     state: GenerationState,
     kind: KotlinClassHeader.Kind,
+    publicAbi: Boolean,
     extraFlags: Int,
     action: (AnnotationVisitor) -> Unit
 ) {
@@ -43,6 +44,9 @@ fun writeKotlinMetadata(
     if (state.languageVersionSettings.getFlag(JvmAnalysisFlags.strictMetadataVersionSemantics)) {
         flags = flags or JvmAnnotationNames.METADATA_STRICT_VERSION_SEMANTICS_FLAG
     }
+    if (publicAbi) {
+        flags = flags or JvmAnnotationNames.METADATA_PUBLIC_ABI_FLAG
+    }
     if (flags != 0) {
         av.visit(JvmAnnotationNames.METADATA_EXTRA_INT_FIELD_NAME, flags)
     }
@@ -50,8 +54,8 @@ fun writeKotlinMetadata(
     av.visitEnd()
 }
 
-fun writeSyntheticClassMetadata(cb: ClassBuilder, state: GenerationState) {
-    writeKotlinMetadata(cb, state, KotlinClassHeader.Kind.SYNTHETIC_CLASS, 0) { _ ->
+fun writeSyntheticClassMetadata(cb: ClassBuilder, state: GenerationState, publicAbi: Boolean) {
+    writeKotlinMetadata(cb, state, KotlinClassHeader.Kind.SYNTHETIC_CLASS, publicAbi, 0) { _ ->
         // Do nothing
     }
 }