FIR: fix infinite recursion with equality operator checker

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirOldFrontendDiagnosticsTestGenerated.java

@@ -620,6 +620,12 @@ public class FirOldFrontendDiagnosticsTestGenerated extends AbstractFirDiagnosti
             runTest("compiler/testData/diagnostics/tests/RecursiveTypeInference.kt");
         }
 
+        @Test
+        @TestMetadata("RecursiveTypeParameterEqualityCheck.kt")
+        public void testRecursiveTypeParameterEqualityCheck() throws Exception {
+            runTest("compiler/testData/diagnostics/tests/RecursiveTypeParameterEqualityCheck.kt");
+        }
+
         @Test
         @TestMetadata("ReserveYield.kt")
         public void testReserveYield() throws Exception {

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirOldFrontendDiagnosticsWithLightTreeTestGenerated.java

@@ -620,6 +620,12 @@ public class FirOldFrontendDiagnosticsWithLightTreeTestGenerated extends Abstrac
             runTest("compiler/testData/diagnostics/tests/RecursiveTypeInference.kt");
         }
 
+        @Test
+        @TestMetadata("RecursiveTypeParameterEqualityCheck.kt")
+        public void testRecursiveTypeParameterEqualityCheck() throws Exception {
+            runTest("compiler/testData/diagnostics/tests/RecursiveTypeParameterEqualityCheck.kt");
+        }
+
         @Test
         @TestMetadata("ReserveYield.kt")
         public void testReserveYield() throws Exception {

compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/ConeTypeCompatibilityChecker.kt

@@ -98,7 +98,8 @@ internal object ConeTypeCompatibilityChecker {
     private fun ConeInferenceContext.areCompatible(
         upperBounds: Set<ConeClassLikeType>,
         lowerBounds: Set<ConeClassLikeType>,
-        compatibilityUpperBound: Compatibility
+        compatibilityUpperBound: Compatibility,
+        checkedTypeParameters: MutableSet<FirTypeParameterRef> = mutableSetOf(),
     ): Compatibility {
         val upperBoundClasses: Set<FirClassWithSuperClasses> = upperBounds.mapNotNull { it.toFirClassWithSuperClasses(this) }.toSet()
 
@@ -134,8 +135,17 @@ internal object ConeTypeCompatibilityChecker {
             }
         }
         var result = Compatibility.COMPATIBLE
-        val typeArgsCompatibility = typeArgumentMapping.values.asSequence()
+        val typeArgsCompatibility = typeArgumentMapping.asSequence()
-            .map { (upper, lower, compatibilityUpperBound) -> areCompatible(upper, lower, compatibilityUpperBound) }
+            .map { (paramRef, boundTypeArguments) ->
+                val (upper, lower, compatibility) = boundTypeArguments
+                if (paramRef in checkedTypeParameters) {
+                    // if we are already checking this type parameter, simply bail out to prevent infinite recursion.
+                    Compatibility.COMPATIBLE
+                } else {
+                    checkedTypeParameters.add(paramRef)
+                    areCompatible(upper, lower, compatibility, checkedTypeParameters)
+                }
+            }
         for (compatibility in typeArgsCompatibility) {
             if (compatibility == compatibilityUpperBound) return compatibility
             if (compatibility > result) {

compiler/testData/diagnostics/tests/RecursiveTypeParameterEqualityCheck.fir.kt

@@ -0,0 +1,9 @@
+interface Foo<out T>
+interface Bar1 : Foo<Bar1>
+interface Bar2 : Foo<Bar2>
+class Bar3 : Foo<Bar3>
+
+fun test(b1: Bar1, b2: Bar2, b3: Bar3) {
+    b1 == b2
+    <!EQUALITY_NOT_APPLICABLE_WARNING!>b1 == b3<!>
+}

compiler/testData/diagnostics/tests/RecursiveTypeParameterEqualityCheck.kt

@@ -0,0 +1,9 @@
+interface Foo<out T>
+interface Bar1 : Foo<Bar1>
+interface Bar2 : Foo<Bar2>
+class Bar3 : Foo<Bar3>
+
+fun test(b1: Bar1, b2: Bar2, b3: Bar3) {
+    b1 == b2
+    <!EQUALITY_NOT_APPLICABLE!>b1 == b3<!>
+}

compiler/testData/diagnostics/tests/RecursiveTypeParameterEqualityCheck.txt

@@ -0,0 +1,28 @@
+package
+
+public fun test(/*0*/ b1: Bar1, /*1*/ b2: Bar2, /*2*/ b3: Bar3): kotlin.Unit
+
+public interface Bar1 : Foo<Bar1> {
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public interface Bar2 : Foo<Bar2> {
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public final class Bar3 : Foo<Bar3> {
+    public constructor Bar3()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public interface Foo</*0*/ out T> {
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/DiagnosticTestGenerated.java

@@ -620,6 +620,12 @@ public class DiagnosticTestGenerated extends AbstractDiagnosticTest {
             runTest("compiler/testData/diagnostics/tests/RecursiveTypeInference.kt");
         }
 
+        @Test
+        @TestMetadata("RecursiveTypeParameterEqualityCheck.kt")
+        public void testRecursiveTypeParameterEqualityCheck() throws Exception {
+            runTest("compiler/testData/diagnostics/tests/RecursiveTypeParameterEqualityCheck.kt");
+        }
+
         @Test
         @TestMetadata("ReserveYield.kt")
         public void testReserveYield() throws Exception {