[FIR] Don't expand typealiases in annotations during type deserialization

During extracting type attributes from annotations we should expand
  typealiases of annotation type to handle cases when user makes a typealias
  on some special annotation, like `kotlin.internal.Exact`. And to exapnd
  typealias we should resolve annotation class id to symbol.
This leads to a cycle during class deserialization, if some nested annotation
  is used as type annotation in some declaration in the same class

```
interface SomeInterface {
    interface NestedInterface : @Ann Some

    interface Some

    @Target(AnnotationTarget.TYPE)
    annotation class Ann
}
```
Attempt to find symbol for SomeInterface.Ann during deserialization of
  SomeInterface.NestedInterface wil lead to second attempt to deserialize
  class SomeInterface, which eventually leads to StackOverFlow. And at the
  same time expanding typealiases for annotations from binaries has not
  much sense, because types in binaries are already expanded

So to fix this issue it's enough to just not expand typealiases on type
  annotations for types of deserialized declarations

^KT-57876 Fixed

compiler/fir/fir-deserialization/src/org/jetbrains/kotlin/fir/deserialization/FirTypeDeserializer.kt

@@ -111,13 +111,13 @@ class FirTypeDeserializer(
     fun typeRef(proto: ProtoBuf.Type): FirTypeRef {
         return buildResolvedTypeRef {
             annotations += annotationDeserializer.loadTypeAnnotations(proto, nameResolver)
-            type = type(proto, annotations.computeTypeAttributes(moduleData.session))
+            type = type(proto, annotations.computeTypeAttributes(moduleData.session, shouldExpandTypeAliases = false))
         }
     }
 
     private fun attributesFromAnnotations(proto: ProtoBuf.Type): ConeAttributes =
         annotationDeserializer.loadTypeAnnotations(proto, nameResolver)
-            .computeTypeAttributes(moduleData.session)
+            .computeTypeAttributes(moduleData.session, shouldExpandTypeAliases = false)
 
     fun type(proto: ProtoBuf.Type): ConeKotlinType {
         return type(proto, attributesFromAnnotations(proto))

compiler/fir/fir2ir/tests-gen/org/jetbrains/kotlin/test/runners/codegen/FirLightTreeBlackBoxCodegenTestGenerated.java

@@ -149,6 +149,18 @@ public class FirLightTreeBlackBoxCodegenTestGenerated extends AbstractFirLightTr
             runTest("compiler/testData/codegen/box/annotations/delegatedPropertySetter.kt");
         }
 
+        @Test
+        @TestMetadata("deserializationOfNestedAnnotationOnType_1.kt")
+        public void testDeserializationOfNestedAnnotationOnType_1() throws Exception {
+            runTest("compiler/testData/codegen/box/annotations/deserializationOfNestedAnnotationOnType_1.kt");
+        }
+
+        @Test
+        @TestMetadata("deserializationOfNestedAnnotationOnType_2.kt")
+        public void testDeserializationOfNestedAnnotationOnType_2() throws Exception {
+            runTest("compiler/testData/codegen/box/annotations/deserializationOfNestedAnnotationOnType_2.kt");
+        }
+
         @Test
         @TestMetadata("divisionByZeroInJava.kt")
         public void testDivisionByZeroInJava() throws Exception {

compiler/fir/fir2ir/tests-gen/org/jetbrains/kotlin/test/runners/codegen/FirPsiBlackBoxCodegenTestGenerated.java

@@ -149,6 +149,18 @@ public class FirPsiBlackBoxCodegenTestGenerated extends AbstractFirPsiBlackBoxCo
             runTest("compiler/testData/codegen/box/annotations/delegatedPropertySetter.kt");
         }
 
+        @Test
+        @TestMetadata("deserializationOfNestedAnnotationOnType_1.kt")
+        public void testDeserializationOfNestedAnnotationOnType_1() throws Exception {
+            runTest("compiler/testData/codegen/box/annotations/deserializationOfNestedAnnotationOnType_1.kt");
+        }
+
+        @Test
+        @TestMetadata("deserializationOfNestedAnnotationOnType_2.kt")
+        public void testDeserializationOfNestedAnnotationOnType_2() throws Exception {
+            runTest("compiler/testData/codegen/box/annotations/deserializationOfNestedAnnotationOnType_2.kt");
+        }
+
         @Test
         @TestMetadata("divisionByZeroInJava.kt")
         public void testDivisionByZeroInJava() throws Exception {

compiler/fir/providers/src/org/jetbrains/kotlin/fir/CopyUtils.kt

@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.fir
 
 import org.jetbrains.kotlin.builtins.StandardNames
 import org.jetbrains.kotlin.fir.declarations.FirDeclaration
+import org.jetbrains.kotlin.fir.declarations.toAnnotationClassId
 import org.jetbrains.kotlin.fir.declarations.utils.expandedConeType
 import org.jetbrains.kotlin.fir.diagnostics.ConeDiagnostic
 import org.jetbrains.kotlin.fir.expressions.*
@@ -72,10 +73,15 @@ fun FirTypeRef.errorTypeFromPrototype(
     }
 }
 
+/**
+ * [shouldExpandTypeAliases] should be set to `false` if this function is called during deserialization of some binary declaration
+ * For details see KT-57876
+ */
 fun List<FirAnnotation>.computeTypeAttributes(
     session: FirSession,
     predefined: List<ConeAttribute<*>> = emptyList(),
     containerDeclaration: FirDeclaration? = null,
+    shouldExpandTypeAliases: Boolean
 ): ConeAttributes {
     if (this.isEmpty()) {
         if (predefined.isEmpty()) return ConeAttributes.Empty
@@ -85,7 +91,11 @@ fun List<FirAnnotation>.computeTypeAttributes(
     attributes += predefined
     val customAnnotations = mutableListOf<FirAnnotation>()
     for (annotation in this) {
-        when (annotation.tryExpandClassId(session)) {
+        val classId = when (shouldExpandTypeAliases) {
+            true -> annotation.tryExpandClassId(session)
+            false -> annotation.typeRef.coneType.classId
+        }
+        when (classId) {
             CompilerConeAttributes.Exact.ANNOTATION_CLASS_ID -> attributes += CompilerConeAttributes.Exact
             CompilerConeAttributes.NoInfer.ANNOTATION_CLASS_ID -> attributes += CompilerConeAttributes.NoInfer
             CompilerConeAttributes.ExtensionFunctionType.ANNOTATION_CLASS_ID -> attributes += CompilerConeAttributes.ExtensionFunctionType

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/resolve/providers/impl/FirTypeResolverImpl.kt

@@ -319,7 +319,7 @@ class FirTypeResolverImpl(private val session: FirSession) : FirTypeResolver() {
             return ConeErrorType(
                 diagnostic,
                 typeArguments = resultingArguments,
-                attributes = typeRef.annotations.computeTypeAttributes(session)
+                attributes = typeRef.annotations.computeTypeAttributes(session, shouldExpandTypeAliases = true)
             )
         }
 
@@ -337,7 +337,7 @@ class FirTypeResolverImpl(private val session: FirSession) : FirTypeResolver() {
         return symbol.constructType(
             resultingArguments,
             typeRef.isMarkedNullable,
-            typeRef.annotations.computeTypeAttributes(session, containerDeclaration = containerDeclaration)
+            typeRef.annotations.computeTypeAttributes(session, containerDeclaration = containerDeclaration, shouldExpandTypeAliases = true)
         ).also {
             val lookupTag = it.lookupTag
             if (lookupTag is ConeClassLikeLookupTagImpl && symbol is FirClassLikeSymbol<*>) {
@@ -479,7 +479,8 @@ class FirTypeResolverImpl(private val session: FirSession) : FirTypeResolver() {
                     add(CompilerConeAttributes.ContextFunctionTypeParams(typeRef.contextReceiverTypeRefs.size))
                 }
             },
-            containerDeclaration
+            containerDeclaration,
+            shouldExpandTypeAliases = true
         )
         return ConeClassLikeTypeImpl(
             classId.toLookupTag(),

compiler/testData/codegen/box/annotations/deserializationOfNestedAnnotationOnType_1.kt

@@ -0,0 +1,16 @@
+// TARGET_BACKEND: JVM_IR
+// ISSUE: KT-57876
+
+// MODULE: lib
+interface EnvironmentKeyProvider {
+    interface NestedInterface : @EnvironmentKeyDescription Some
+
+    interface Some
+
+    @Target(AnnotationTarget.TYPE)
+    annotation class EnvironmentKeyDescription
+}
+
+// MODULE: main(lib)
+fun bar(arg: EnvironmentKeyProvider.NestedInterface) {}
+fun box() = "OK"

compiler/testData/codegen/box/annotations/deserializationOfNestedAnnotationOnType_2.kt

@@ -0,0 +1,14 @@
+// TARGET_BACKEND: JVM_IR
+// ISSUE: KT-57876
+
+// MODULE: lib
+interface EnvironmentKeyProvider {
+    @Target(AnnotationTarget.TYPE)
+    annotation class EnvironmentKeyDescription
+
+    fun getKnownKeys(arg: @EnvironmentKeyDescription String)
+}
+
+// MODULE: main(lib)
+fun foo(arg: EnvironmentKeyProvider) {}
+fun box() = "OK"

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/IrBlackBoxCodegenTestGenerated.java

@@ -149,6 +149,18 @@ public class IrBlackBoxCodegenTestGenerated extends AbstractIrBlackBoxCodegenTes
             runTest("compiler/testData/codegen/box/annotations/delegatedPropertySetter.kt");
         }
 
+        @Test
+        @TestMetadata("deserializationOfNestedAnnotationOnType_1.kt")
+        public void testDeserializationOfNestedAnnotationOnType_1() throws Exception {
+            runTest("compiler/testData/codegen/box/annotations/deserializationOfNestedAnnotationOnType_1.kt");
+        }
+
+        @Test
+        @TestMetadata("deserializationOfNestedAnnotationOnType_2.kt")
+        public void testDeserializationOfNestedAnnotationOnType_2() throws Exception {
+            runTest("compiler/testData/codegen/box/annotations/deserializationOfNestedAnnotationOnType_2.kt");
+        }
+
         @Test
         @TestMetadata("divisionByZeroInJava.kt")
         public void testDivisionByZeroInJava() throws Exception {

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/IrBlackBoxCodegenWithIrInlinerTestGenerated.java

@@ -149,6 +149,18 @@ public class IrBlackBoxCodegenWithIrInlinerTestGenerated extends AbstractIrBlack
             runTest("compiler/testData/codegen/box/annotations/delegatedPropertySetter.kt");
         }
 
+        @Test
+        @TestMetadata("deserializationOfNestedAnnotationOnType_1.kt")
+        public void testDeserializationOfNestedAnnotationOnType_1() throws Exception {
+            runTest("compiler/testData/codegen/box/annotations/deserializationOfNestedAnnotationOnType_1.kt");
+        }
+
+        @Test
+        @TestMetadata("deserializationOfNestedAnnotationOnType_2.kt")
+        public void testDeserializationOfNestedAnnotationOnType_2() throws Exception {
+            runTest("compiler/testData/codegen/box/annotations/deserializationOfNestedAnnotationOnType_2.kt");
+        }
+
         @Test
         @TestMetadata("divisionByZeroInJava.kt")
         public void testDivisionByZeroInJava() throws Exception {