[FIR] Implement RECURSIVE_TYPEALIAS_EXPANSION, CYCLIC_INHERITANCE_HIERARCHY diagnostics, fix stackoverlow exception in case if typealias points to type with type arguments

compiler/fir/analysis-tests/testData/resolve/fakeRecursiveSupertype.fir.txt

@@ -5,7 +5,7 @@ FILE: fakeRecursiveSupertype.kt
         }
 
     }
-    public open class Your : R|His| {
+    public open class Your : <ERROR TYPE REF: Loop in supertype: /Your -> /His> {
         public constructor(): R|Your| {
             super<R|His|>()
         }

compiler/fir/analysis-tests/testData/resolve/fakeRecursiveSupertype.kt

@@ -1,7 +1,7 @@
 import incorrect.directory.My
 
-open class My : <!OTHER_ERROR!>My<!>()
+open class My : <!CYCLIC_INHERITANCE_HIERARCHY!>My<!>()
 
-open class Your : His()
+open class Your : <!CYCLIC_INHERITANCE_HIERARCHY!>His<!>()
 
-open class His : <!OTHER_ERROR!>Your<!>()
+open class His : <!CYCLIC_INHERITANCE_HIERARCHY!>Your<!>()

compiler/fir/analysis-tests/testData/resolve/fakeRecursiveTypealias.kt

@@ -2,4 +2,4 @@ import incorrect.directory.Your
 
 typealias My = <!UNRESOLVED_REFERENCE!>incorrect.directory.My<!>
 
-typealias Your = <!OTHER_ERROR!>Your<!>
+typealias Your = <!RECURSIVE_TYPEALIAS_EXPANSION!>Your<!>

compiler/fir/analysis-tests/testData/resolveWithStdlib/problems/KJKComplexHierarchyNestedLoop.fir.txt

@@ -1,5 +1,5 @@
 FILE: K1.kt
-    public final class K2 : R|J1| {
+    public final class K2 : <ERROR TYPE REF: Loop in supertype: /K2 -> /J1> {
         public constructor(): R|K2| {
             super<R|J1|>()
         }

compiler/fir/analysis-tests/testData/resolveWithStdlib/problems/KJKComplexHierarchyNestedLoop.kt

@@ -1,6 +1,6 @@
 // FIR_IDE_IGNORE
 // FILE: K1.kt
-class K2: J1() {
+class K2: <!CYCLIC_INHERITANCE_HIERARCHY!>J1<!>() {
     class Q : <!UNRESOLVED_REFERENCE!>Nested<!>()
     fun bar() {
         <!UNRESOLVED_REFERENCE!>foo<!>()

compiler/fir/checkers/checkers-component-generator/src/org/jetbrains/kotlin/fir/checkers/generator/diagnostics/FirDiagnosticsList.kt

@@ -134,7 +134,7 @@ object DIAGNOSTICS_LIST : DiagnosticList("FirErrors") {
         val SUPERTYPE_NOT_A_CLASS_OR_INTERFACE by error<KtElement> {
             parameter<String>("reason")
         }
-
+        val CYCLIC_INHERITANCE_HIERARCHY by error<PsiElement>()
     }
 
     val CONSTRUCTOR_PROBLEMS by object : DiagnosticGroup("Constructor problems") {
@@ -867,6 +867,7 @@ object DIAGNOSTICS_LIST : DiagnosticList("FirErrors") {
 
     val TYPE_ALIAS by object : DiagnosticGroup("Type alias") {
         val TOPLEVEL_TYPEALIASES_ONLY by error<KtTypeAlias>()
+        val RECURSIVE_TYPEALIAS_EXPANSION by error<KtTypeAlias>()
     }
 
     val EXTENDED_CHECKERS by object : DiagnosticGroup("Extended checkers") {

compiler/fir/checkers/gen/org/jetbrains/kotlin/fir/analysis/diagnostics/FirErrors.kt

@@ -143,6 +143,7 @@ object FirErrors {
     val SEALED_SUPERTYPE by error0<KtTypeReference>()
     val SEALED_SUPERTYPE_IN_LOCAL_CLASS by error0<KtTypeReference>()
     val SUPERTYPE_NOT_A_CLASS_OR_INTERFACE by error1<KtElement, String>()
+    val CYCLIC_INHERITANCE_HIERARCHY by error0<PsiElement>()
 
     // Constructor problems
     val CONSTRUCTOR_IN_OBJECT by error0<KtDeclaration>(SourceElementPositioningStrategies.DECLARATION_SIGNATURE)
@@ -481,6 +482,7 @@ object FirErrors {
 
     // Type alias
     val TOPLEVEL_TYPEALIASES_ONLY by error0<KtTypeAlias>()
+    val RECURSIVE_TYPEALIAS_EXPANSION by error0<KtTypeAlias>()
 
     // Extended checkers
     val REDUNDANT_VISIBILITY_MODIFIER by warning0<KtModifierListOwner>(SourceElementPositioningStrategies.VISIBILITY_MODIFIER)

compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/CommonTypeCheckers.kt

@@ -5,14 +5,11 @@
 
 package org.jetbrains.kotlin.fir.analysis.checkers
 
-import org.jetbrains.kotlin.fir.analysis.checkers.type.FirSuspendModifierChecker
-import org.jetbrains.kotlin.fir.analysis.checkers.type.FirTypeAnnotationChecker
-import org.jetbrains.kotlin.fir.analysis.checkers.type.FirTypeRefChecker
-import org.jetbrains.kotlin.fir.analysis.checkers.type.TypeCheckers
+import org.jetbrains.kotlin.fir.analysis.checkers.type.*
 
 object CommonTypeCheckers : TypeCheckers() {
     override val typeRefCheckers: Set<FirTypeRefChecker> = setOf(
         FirTypeAnnotationChecker,
-        FirSuspendModifierChecker,
+        FirSuspendModifierChecker
     )
 }

compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/diagnostics/FirDefaultErrorMessages.kt

@@ -82,6 +82,7 @@ import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.CONST_VAL_WITH_NO
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.CREATING_AN_INSTANCE_OF_ABSTRACT_CLASS
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.CYCLIC_CONSTRUCTOR_DELEGATION_CALL
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.CYCLIC_GENERIC_UPPER_BOUND
+import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.CYCLIC_INHERITANCE_HIERARCHY
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.DATA_CLASS_NOT_PROPERTY_PARAMETER
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.DATA_CLASS_VARARG_PARAMETER
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.DATA_CLASS_WITHOUT_PARAMETERS
@@ -247,6 +248,7 @@ import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.QUALIFIED_SUPERTY
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.RECURSION_IN_IMPLICIT_TYPES
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.RECURSION_IN_INLINE
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.RECURSION_IN_SUPERTYPES
+import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.RECURSIVE_TYPEALIAS_EXPANSION
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.REDECLARATION
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.REDUNDANT_ANNOTATION_TARGET
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.REDUNDANT_CALL_OF_CONVERSION_METHOD
@@ -438,7 +440,7 @@ class FirDefaultErrorMessages : DefaultErrorMessages.Extension {
             map.put(SEALED_SUPERTYPE, "This type is sealed, so it can be inherited by only its own nested classes or objects")
             map.put(SEALED_SUPERTYPE_IN_LOCAL_CLASS, "Local class cannot extend a sealed class")
             map.put(SUPERTYPE_NOT_A_CLASS_OR_INTERFACE, "Supertype is not a class or interface", TO_STRING)
-
+            map.put(CYCLIC_INHERITANCE_HIERARCHY, "There's a cycle in the inheritance hierarchy for this type")
 
             // Constructor problems
             map.put(CONSTRUCTOR_IN_OBJECT, "Constructors are not allowed for objects")
@@ -1109,6 +1111,7 @@ class FirDefaultErrorMessages : DefaultErrorMessages.Extension {
 
             // Type alias
             map.put(TOPLEVEL_TYPEALIASES_ONLY, "Nested and local type aliases are not supported")
+            map.put(RECURSIVE_TYPEALIAS_EXPANSION, "Recursive type alias in expansion")
 
             // Returns
             map.put(RETURN_NOT_ALLOWED, "'return' is not allowed here")

compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/diagnostics/coneDiagnosticToFirDiagnostic.kt

@@ -62,7 +62,7 @@ private fun ConeDiagnostic.toFirDiagnostic(
         FirErrors.NO_TYPE_ARGUMENTS_ON_RHS.on(qualifiedAccessSource ?: source, this.desiredCount, this.type)
     is ConeSimpleDiagnostic -> when (source.kind) {
         is FirFakeSourceElementKind -> null
-        else -> this.getFactory(source).on(qualifiedAccessSource ?: source)
+        else -> this.getFactory(source)?.on(qualifiedAccessSource ?: source)
     }
     is ConeInstanceAccessBeforeSuperCall -> FirErrors.INSTANCE_ACCESS_BEFORE_SUPER_CALL.on(source, this.target)
     is ConeStubDiagnostic -> null
@@ -277,7 +277,7 @@ private fun ConstraintSystemError.toDiagnostic(
 private val NewConstraintError.lowerConeType: ConeKotlinType get() = lowerType as ConeKotlinType
 private val NewConstraintError.upperConeType: ConeKotlinType get() = upperType as ConeKotlinType
 
-private fun ConeSimpleDiagnostic.getFactory(source: FirSourceElement): FirDiagnosticFactory0<*> {
+private fun ConeSimpleDiagnostic.getFactory(source: FirSourceElement): FirDiagnosticFactory0<*>? {
     @Suppress("UNCHECKED_CAST")
     return when (kind) {
         DiagnosticKind.Syntax -> FirErrors.SYNTAX
@@ -309,11 +309,15 @@ private fun ConeSimpleDiagnostic.getFactory(source: FirSourceElement): FirDiagno
         DiagnosticKind.IntLiteralOutOfRange -> FirErrors.INT_LITERAL_OUT_OF_RANGE
         DiagnosticKind.FloatLiteralOutOfRange -> FirErrors.FLOAT_LITERAL_OUT_OF_RANGE
         DiagnosticKind.WrongLongSuffix -> FirErrors.WRONG_LONG_SUFFIX
-        DiagnosticKind.Other -> FirErrors.OTHER_ERROR
         DiagnosticKind.IncorrectCharacterLiteral -> FirErrors.INCORRECT_CHARACTER_LITERAL
         DiagnosticKind.EmptyCharacterLiteral -> FirErrors.EMPTY_CHARACTER_LITERAL
         DiagnosticKind.TooManyCharactersInCharacterLiteral -> FirErrors.TOO_MANY_CHARACTERS_IN_CHARACTER_LITERAL
         DiagnosticKind.IllegalEscape -> FirErrors.ILLEGAL_ESCAPE
+        DiagnosticKind.RecursiveTypealiasExpansion -> FirErrors.RECURSIVE_TYPEALIAS_EXPANSION
+        DiagnosticKind.LoopInSupertype -> FirErrors.CYCLIC_INHERITANCE_HIERARCHY
+        DiagnosticKind.UnresolvedSupertype,
+        DiagnosticKind.UnresolvedExpandedType,
+        DiagnosticKind.Other -> FirErrors.OTHER_ERROR
         else -> throw IllegalArgumentException("Unsupported diagnostic kind: $kind at $javaClass")
     }
 }

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/resolve/transformers/FirSupertypesResolution.kt

@@ -11,6 +11,7 @@ import kotlinx.collections.immutable.toPersistentList
 import org.jetbrains.kotlin.fir.*
 import org.jetbrains.kotlin.fir.declarations.*
 import org.jetbrains.kotlin.fir.diagnostics.ConeSimpleDiagnostic
+import org.jetbrains.kotlin.fir.diagnostics.DiagnosticKind
 import org.jetbrains.kotlin.fir.expressions.FirStatement
 import org.jetbrains.kotlin.fir.extensions.extensionService
 import org.jetbrains.kotlin.fir.extensions.predicateBasedProvider
@@ -27,13 +28,16 @@ import org.jetbrains.kotlin.fir.scopes.impl.FirMemberTypeParameterScope
 import org.jetbrains.kotlin.fir.scopes.impl.nestedClassifierScope
 import org.jetbrains.kotlin.fir.scopes.impl.wrapNestedClassifierScopeWithSubstitutionForSuperType
 import org.jetbrains.kotlin.fir.symbols.impl.FirClassLikeSymbol
+import org.jetbrains.kotlin.fir.symbols.impl.FirRegularClassSymbol
 import org.jetbrains.kotlin.fir.symbols.impl.FirTypeAliasSymbol
 import org.jetbrains.kotlin.fir.types.*
 import org.jetbrains.kotlin.fir.types.builder.buildErrorTypeRef
-import org.jetbrains.kotlin.fir.visitors.*
+import org.jetbrains.kotlin.fir.visitors.FirDefaultTransformer
+import org.jetbrains.kotlin.fir.visitors.FirDefaultVisitor
+import org.jetbrains.kotlin.fir.visitors.FirTransformer
 import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.types.model.TypeArgumentMarker
 import org.jetbrains.kotlin.utils.addIfNotNull
-import org.jetbrains.kotlin.utils.addToStdlib.safeAs
 
 class FirSupertypeResolverProcessor(session: FirSession, scopeSession: ScopeSession) :
     FirTransformerBasedResolveProcessor(session, scopeSession) {
@@ -273,7 +277,11 @@ class FirSupertypeResolverVisitor(
         when (val status = supertypeComputationSession.getSupertypesComputationStatus(classLikeDeclaration)) {
             is SupertypeComputationStatus.Computed -> return status.supertypeRefs
             is SupertypeComputationStatus.Computing -> return listOf(
-                createErrorTypeRef(classLikeDeclaration, "Loop in supertype definition for ${classLikeDeclaration.symbol.classId}")
+                createErrorTypeRef(
+                    classLikeDeclaration,
+                    "Loop in supertype definition for ${classLikeDeclaration.symbol.classId}",
+                    if (classLikeDeclaration is FirTypeAlias) DiagnosticKind.RecursiveTypealiasExpansion else DiagnosticKind.LoopInSupertype
+                )
             )
         }
 
@@ -326,7 +334,11 @@ class FirSupertypeResolverVisitor(
                             diagnostic = ConeTypeParameterSupertype(typeParameterType.lookupTag.typeParameterSymbol)
                         }
                     superTypeRef !is FirResolvedTypeRef ->
-                        createErrorTypeRef(superTypeRef, "Unresolved super-type: ${superTypeRef.render()}")
+                        createErrorTypeRef(
+                            superTypeRef,
+                            "Unresolved super-type: ${superTypeRef.render()}",
+                            DiagnosticKind.UnresolvedSupertype
+                        )
                     else ->
                         superTypeRef
                 }
@@ -360,18 +372,26 @@ class FirSupertypeResolverVisitor(
                     ?: return@resolveSpecificClassLikeSupertypes listOf(
                         createErrorTypeRef(
                             typeAlias.expandedTypeRef,
-                            "Unresolved expanded typeRef for ${typeAlias.symbol.classId}"
+                            "Unresolved expanded typeRef for ${typeAlias.symbol.classId}",
+                            DiagnosticKind.UnresolvedExpandedType
                         )
                     )
 
-            val type = resolvedTypeRef.type
-            if (type is ConeClassLikeType) {
-                val expansionTypeAlias = type.lookupTag.toSymbol(session)?.safeAs<FirTypeAliasSymbol>()?.fir
-                if (expansionTypeAlias != null) {
-                    visitTypeAlias(expansionTypeAlias, null)
+            fun visitNestedTypeAliases(type: TypeArgumentMarker) {
+                if (type is ConeClassLikeType) {
+                    val symbol = type.lookupTag.toSymbol(session)
+                    if (symbol is FirTypeAliasSymbol) {
+                        visitTypeAlias(symbol.fir, null)
+                    } else if (symbol is FirClassLikeSymbol) {
+                        for (typeArgument in type.typeArguments) {
+                            visitNestedTypeAliases(typeArgument)
+                        }
+                    }
                 }
             }
 
+            visitNestedTypeAliases(resolvedTypeRef.type)
+
             listOf(resolvedTypeRef)
         }
     }
@@ -381,9 +401,9 @@ class FirSupertypeResolverVisitor(
     }
 }
 
-private fun createErrorTypeRef(fir: FirElement, message: String) = buildErrorTypeRef {
+private fun createErrorTypeRef(fir: FirElement, message: String, kind: DiagnosticKind) = buildErrorTypeRef {
     source = fir.source
-    diagnostic = ConeSimpleDiagnostic(message)
+    diagnostic = ConeSimpleDiagnostic(message, kind)
 }
 
 class SupertypeComputationSession {
@@ -434,51 +454,95 @@ class SupertypeComputationSession {
     }
 
     private val newClassifiersForBreakingLoops = mutableListOf<FirClassLikeDeclaration<*>>()
-    private val breakLoopsDfsVisited = hashSetOf<FirClassLikeDeclaration<*>>()
 
     fun breakLoops(session: FirSession) {
-        val inProcess = hashSetOf<FirClassLikeDeclaration<*>>()
+        val visitedClassLikeDecls = mutableSetOf<FirClassLikeDeclaration<*>>()
+        val loopedClassLikeDecls = mutableSetOf<FirClassLikeDeclaration<*>>()
+        val path = mutableListOf<FirClassLikeDeclaration<*>>()
+        val pathSet = mutableSetOf<FirClassLikeDeclaration<*>>()
 
-        fun dfs(classLikeDeclaration: FirClassLikeDeclaration<*>) {
-            if (classLikeDeclaration in breakLoopsDfsVisited) return
-            val supertypeComputationStatus = supertypeStatusMap[classLikeDeclaration] ?: return
-            if (classLikeDeclaration in inProcess) return
+        fun checkIsInLoop(classLikeDecl: FirClassLikeDeclaration<*>?) {
+            if (classLikeDecl == null) return
 
-            inProcess.add(classLikeDeclaration)
-
-            require(supertypeComputationStatus is SupertypeComputationStatus.Computed) {
-                "Expected computed supertypes in breakLoops for ${classLikeDeclaration.symbol.classId}"
+            val supertypeRefs: List<FirResolvedTypeRef>
+            val supertypeComputationStatus = supertypeStatusMap[classLikeDecl]
+            supertypeRefs = if (supertypeComputationStatus != null) {
+                require(supertypeComputationStatus is SupertypeComputationStatus.Computed) {
+                    "Expected computed supertypes in breakLoops for ${classLikeDecl.symbol.classId}"
+                }
+                supertypeComputationStatus.supertypeRefs
+            } else {
+                when (classLikeDecl) {
+                    is FirRegularClass ->
+                        classLikeDecl.superTypeRefs.filterIsInstance<FirResolvedTypeRef>()
+                    is FirTypeAlias ->
+                        (classLikeDecl.expandedTypeRef as? FirResolvedTypeRef)?.let { listOf(it) } ?: listOf()
+                    else -> return
+                }
             }
 
-            val typeRefs = supertypeComputationStatus.supertypeRefs
-            val resultingTypeRefs = mutableListOf<FirResolvedTypeRef>()
-            var wereChanges = false
+            if (classLikeDecl in visitedClassLikeDecls) {
+                if (classLikeDecl in pathSet) {
+                    loopedClassLikeDecls.add(classLikeDecl)
+                    loopedClassLikeDecls.addAll(path.takeLastWhile { element -> element != classLikeDecl })
+                }
+                return
+            }
 
-            for (typeRef in typeRefs) {
-                val fir = typeRef.firClassLike(session)
-                fir?.let(::dfs)
-                resultingTypeRefs.add(
-                    if (fir in inProcess) {
-                        wereChanges = true
+            path.add(classLikeDecl)
+            pathSet.add(classLikeDecl)
+            visitedClassLikeDecls.add(classLikeDecl)
+
+            val parentId = classLikeDecl.symbol.classId.relativeClassName.parent()
+            if (!parentId.isRoot) {
+                val parentSymbol = session.symbolProvider.getClassLikeSymbolByFqName(ClassId.fromString(parentId.asString()))
+                if (parentSymbol is FirRegularClassSymbol) {
+                    checkIsInLoop(parentSymbol.fir)
+                }
+            }
+
+            val isTypeAlias = classLikeDecl is FirTypeAlias
+            var isErrorInSupertypesFound = false
+            val resultSupertypeRefs = mutableListOf<FirResolvedTypeRef>()
+            for (supertypeRef in supertypeRefs) {
+                val supertypeFir = supertypeRef.firClassLike(session)
+                checkIsInLoop(supertypeFir)
+
+                if (isTypeAlias) {
+                    for (typeArgument in supertypeRef.type.typeArguments) {
+                        if (typeArgument is ConeClassLikeType) {
+                            checkIsInLoop(typeArgument.lookupTag.toSymbol(session)?.fir)
+                        }
+                    }
+                }
+
+                resultSupertypeRefs.add(
+                    if (classLikeDecl in loopedClassLikeDecls) {
+                        isErrorInSupertypesFound = true
                         createErrorTypeRef(
-                            typeRef,
-                            "Loop in supertype: ${classLikeDeclaration.symbol.classId} -> ${fir?.symbol?.classId}"
+                            supertypeRef,
+                            "Loop in supertype: ${classLikeDecl.symbol.classId} -> ${supertypeFir?.symbol?.classId}",
+                            if (isTypeAlias) DiagnosticKind.RecursiveTypealiasExpansion else DiagnosticKind.LoopInSupertype
                         )
-                    } else
-                        typeRef
+                    } else {
+                        supertypeRef
+                    }
                 )
             }
 
-            if (wereChanges) {
-                supertypeStatusMap[classLikeDeclaration] = SupertypeComputationStatus.Computed(resultingTypeRefs)
+            if (isErrorInSupertypesFound) {
+                supertypeStatusMap[classLikeDecl] = SupertypeComputationStatus.Computed(resultSupertypeRefs)
             }
 
-            inProcess.remove(classLikeDeclaration)
-            breakLoopsDfsVisited.add(classLikeDeclaration)
+            path.removeAt(path.size - 1)
+            pathSet.remove(classLikeDecl)
         }
 
         for (classifier in newClassifiersForBreakingLoops) {
-            dfs(classifier)
+            checkIsInLoop(classifier)
+            require(path.isEmpty()) {
+                "Path should be empty"
+            }
         }
         newClassifiersForBreakingLoops.clear()
     }

compiler/fir/tree/src/org/jetbrains/kotlin/fir/diagnostics/ConeSimpleDiagnostic.kt

@@ -36,6 +36,11 @@ enum class DiagnosticKind {
     IllegalProjectionUsage,
     MissingStdlibClass,
 
+    LoopInSupertype,
+    RecursiveTypealiasExpansion,
+    UnresolvedSupertype,
+    UnresolvedExpandedType,
+
     IncorrectCharacterLiteral,
     EmptyCharacterLiteral,
     TooManyCharactersInCharacterLiteral,