[FIR] KT-56665: Filter-out valid cycles

^KT-56665 Fixed

Merge-request: KT-MR-8861
Merged-by: Nikolay Lunyak <Nikolay.Lunyak@jetbrains.com>

analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/DiagnosisCompilerTestFE10TestdataTestGenerated.java

@@ -489,6 +489,12 @@ public class DiagnosisCompilerTestFE10TestdataTestGenerated extends AbstractDiag
             runTest("compiler/testData/diagnostics/tests/kt55733.kt");
         }
 
+        @Test
+        @TestMetadata("kt56665.kt")
+        public void testKt56665() throws Exception {
+            runTest("compiler/testData/diagnostics/tests/kt56665.kt");
+        }
+
         @Test
         @TestMetadata("LValueAssignment.kt")
         public void testLValueAssignment() throws Exception {

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirOldFrontendDiagnosticsTestGenerated.java

@@ -489,6 +489,12 @@ public class FirOldFrontendDiagnosticsTestGenerated extends AbstractFirDiagnosti
             runTest("compiler/testData/diagnostics/tests/kt55733.kt");
         }
 
+        @Test
+        @TestMetadata("kt56665.kt")
+        public void testKt56665() throws Exception {
+            runTest("compiler/testData/diagnostics/tests/kt56665.kt");
+        }
+
         @Test
         @TestMetadata("LValueAssignment.kt")
         public void testLValueAssignment() throws Exception {

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirOldFrontendDiagnosticsWithLightTreeTestGenerated.java

@@ -489,6 +489,12 @@ public class FirOldFrontendDiagnosticsWithLightTreeTestGenerated extends Abstrac
             runTest("compiler/testData/diagnostics/tests/kt55733.kt");
         }
 
+        @Test
+        @TestMetadata("kt56665.kt")
+        public void testKt56665() throws Exception {
+            runTest("compiler/testData/diagnostics/tests/kt56665.kt");
+        }
+
         @Test
         @TestMetadata("LValueAssignment.kt")
         public void testLValueAssignment() throws Exception {

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/resolve/transformers/FirSupertypesResolution.kt

@@ -560,8 +560,15 @@ class SupertypeComputationSession {
         val path = mutableListOf<FirClassLikeDeclaration>()
         val pathSet = mutableSetOf<FirClassLikeDeclaration>()
 
-        fun checkIsInLoop(classLikeDecl: FirClassLikeDeclaration?) {
+        fun checkIsInLoop(
+            classLikeDecl: FirClassLikeDeclaration?,
+            wasSubtypingInvolved: Boolean,
+            wereTypeArgumentsInvolved: Boolean,
+        ) {
             if (classLikeDecl == null) return
+            require(!wasSubtypingInvolved || !wereTypeArgumentsInvolved) {
+                "This must hold by induction, because otherwise such a loop is allowed"
+            }
 
             val supertypeRefs: List<FirResolvedTypeRef>
             val supertypeComputationStatus = supertypeStatusMap[classLikeDecl]
@@ -596,24 +603,42 @@ class SupertypeComputationSession {
             if (!parentId.isRoot) {
                 val parentSymbol = session.symbolProvider.getClassLikeSymbolByClassId(ClassId.fromString(parentId.asString()))
                 if (parentSymbol is FirRegularClassSymbol) {
-                    checkIsInLoop(parentSymbol.fir)
+                    checkIsInLoop(parentSymbol.fir, wasSubtypingInvolved, wereTypeArgumentsInvolved)
                 }
             }
 
             val isTypeAlias = classLikeDecl is FirTypeAlias
+            val isSubtypingCurrentlyInvolved = !isTypeAlias
+
+            // This is an optimization that prevents collecting
+            // loops we don't want to report anyway.
+            if (wereTypeArgumentsInvolved && isSubtypingCurrentlyInvolved) {
+                path.removeAt(path.size - 1)
+                pathSet.remove(classLikeDecl)
+                return
+            }
+
+            val isSubtypingInvolved = wasSubtypingInvolved || isSubtypingCurrentlyInvolved
             var isErrorInSupertypesFound = false
             val resultSupertypeRefs = mutableListOf<FirResolvedTypeRef>()
             for (supertypeRef in supertypeRefs) {
                 val supertypeFir = supertypeRef.firClassLike(session)
-                checkIsInLoop(supertypeFir)
+                checkIsInLoop(supertypeFir, isSubtypingInvolved, wereTypeArgumentsInvolved)
+
+                // This is an optimization that prevents collecting
+                // loops we don't want to report anyway.
+                if (!isSubtypingInvolved) {
+                    val areTypeArgumentsCurrentlyInvolved = true
 
-                if (isTypeAlias) {
                     fun checkTypeArgumentsRecursively(type: ConeKotlinType, visitedTypes: MutableSet<ConeKotlinType>) {
                         if (type in visitedTypes) return
                         visitedTypes += type
                         for (typeArgument in type.typeArguments) {
                             if (typeArgument is ConeClassLikeType) {
-                                checkIsInLoop(typeArgument.lookupTag.toSymbol(session)?.fir)
+                                checkIsInLoop(
+                                    typeArgument.lookupTag.toSymbol(session)?.fir,
+                                    wasSubtypingInvolved, areTypeArgumentsCurrentlyInvolved,
+                                )
                                 checkTypeArgumentsRecursively(typeArgument, visitedTypes)
                             }
                         }
@@ -645,7 +670,7 @@ class SupertypeComputationSession {
         }
 
         for (classifier in newClassifiersForBreakingLoops) {
-            checkIsInLoop(classifier)
+            checkIsInLoop(classifier, wasSubtypingInvolved = false, wereTypeArgumentsInvolved = false)
             require(path.isEmpty()) {
                 "Path should be empty"
             }

compiler/testData/diagnostics/tests/kt56665.kt

@@ -0,0 +1,11 @@
+// FIR_IDENTICAL
+
+private typealias Bar = Foo<Gau>
+internal class Gau : Bar
+
+internal class Gau2 : Bar2
+private typealias Bar2 = Foo<Gau2>
+
+//internal class Gau : Foo<Gau>
+
+interface Foo<T>

compiler/testData/diagnostics/tests/kt56665.txt

@@ -0,0 +1,24 @@
+package
+
+public interface Foo</*0*/ T> {
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+internal final class Gau : Bar /* = Foo<Gau> */ {
+    public constructor Gau()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+internal final class Gau2 : Bar2 /* = Foo<Gau2> */ {
+    public constructor Gau2()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+private typealias Bar = Foo<Gau>
+private typealias Bar2 = Foo<Gau2>
+

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/DiagnosticTestGenerated.java

@@ -489,6 +489,12 @@ public class DiagnosticTestGenerated extends AbstractDiagnosticTest {
             runTest("compiler/testData/diagnostics/tests/kt55733.kt");
         }
 
+        @Test
+        @TestMetadata("kt56665.kt")
+        public void testKt56665() throws Exception {
+            runTest("compiler/testData/diagnostics/tests/kt56665.kt");
+        }
+
         @Test
         @TestMetadata("LValueAssignment.kt")
         public void testLValueAssignment() throws Exception {