[NI] Introduce type depth in CS calculator to avoid SOE

compiler/resolution/src/org/jetbrains/kotlin/resolve/calls/NewCommonSuperTypeCalculator.kt

@@ -25,6 +25,11 @@ import org.jetbrains.kotlin.types.typeUtil.asTypeProjection
 object NewCommonSuperTypeCalculator {
 
     fun commonSuperType(types: List<UnwrappedType>): UnwrappedType {
+        val maxDepth = types.maxBy { it.typeDepth() }?.typeDepth() ?: 0
+        return commonSuperType(types, -maxDepth)
+    }
+
+    private fun commonSuperType(types: List<UnwrappedType>, depth: Int): UnwrappedType {
         if (types.isEmpty()) throw IllegalStateException("Empty collection for input")
 
         types.singleOrNull()?.let { return it }
@@ -44,19 +49,19 @@ object NewCommonSuperTypeCalculator {
             }
         }
 
-        val lowerSuperType = commonSuperTypeForSimpleTypes(lowers)
+        val lowerSuperType = commonSuperTypeForSimpleTypes(lowers, depth)
         if (!thereIsFlexibleTypes) return lowerSuperType
 
-        val upperSuperType = commonSuperTypeForSimpleTypes(types.map { it.upperIfFlexible() })
+        val upperSuperType = commonSuperTypeForSimpleTypes(types.map { it.upperIfFlexible() }, depth)
         return FlexibleTypeImpl(lowerSuperType, upperSuperType)
     }
 
-    private fun commonSuperTypeForSimpleTypes(types: List<SimpleType>): SimpleType {
+    private fun commonSuperTypeForSimpleTypes(types: List<SimpleType>, depth: Int): SimpleType {
         // i.e. result type also should be marked nullable
         val notAllNotNull = types.any { !NullabilityChecker.isSubtypeOfAny(it) }
         val notNullTypes = if (notAllNotNull) types.map { it.makeNullableAsSpecified(false) } else types
 
-        val commonSuperTypes = commonSuperTypeForNotNullTypes(notNullTypes)
+        val commonSuperTypes = commonSuperTypeForNotNullTypes(notNullTypes, depth)
 
         return if (notAllNotNull) commonSuperTypes.makeNullableAsSpecified(true) else commonSuperTypes
     }
@@ -71,7 +76,7 @@ object NewCommonSuperTypeCalculator {
         return result
     }
 
-    private fun commonSuperTypeForNotNullTypes(types: List<SimpleType>): SimpleType {
+    private fun commonSuperTypeForNotNullTypes(types: List<SimpleType>, depth: Int): SimpleType {
         val uniqueTypes = types.uniquify()
         val filteredType = uniqueTypes.filterNot { type ->
             uniqueTypes.any { other -> type != other && NewKotlinTypeChecker.isSubtypeOf(type, other)}
@@ -81,11 +86,11 @@ object NewCommonSuperTypeCalculator {
 
         filteredType.singleOrNull()?.let { return it }
 
-        return findSuperTypeConstructorsAndIntersectResult(filteredType)
+        return findSuperTypeConstructorsAndIntersectResult(filteredType, depth)
     }
 
-    private fun findSuperTypeConstructorsAndIntersectResult(types: List<SimpleType>): SimpleType {
+    private fun findSuperTypeConstructorsAndIntersectResult(types: List<SimpleType>, depth: Int): SimpleType {
-        return intersectTypes(allCommonSuperTypeConstructors(types).map { superTypeWithGivenConstructor(types, it) })
+        return intersectTypes(allCommonSuperTypeConstructors(types).map { superTypeWithGivenConstructor(types, it, depth) })
     }
 
     /**
@@ -109,7 +114,11 @@ object NewCommonSuperTypeCalculator {
         type.anySuperTypeConstructor { add(it); false }
     }
 
-    private fun superTypeWithGivenConstructor(types: List<SimpleType>, constructor: TypeConstructor): SimpleType {
+    private fun superTypeWithGivenConstructor(
+            types: List<SimpleType>,
+            constructor: TypeConstructor,
+            depth: Int
+    ): SimpleType {
         if (constructor.parameters.isEmpty()) return KotlinTypeFactory.simpleType(Annotations.EMPTY, constructor, emptyList(), nullable = false)
 
         val typeCheckerContext = TypeCheckerContext(false)
@@ -143,7 +152,7 @@ object NewCommonSuperTypeCalculator {
                         StarProjectionImpl(parameter)
                     }
                     else {
-                        calculateArgument(parameter, typeProjections)
+                        calculateArgument(parameter, typeProjections, depth)
                     }
 
             arguments.add(argument)
@@ -152,7 +161,11 @@ object NewCommonSuperTypeCalculator {
     }
 
     // no star projections in arguments
-    private fun calculateArgument(parameter: TypeParameterDescriptor, arguments: List<TypeProjection>): TypeProjection {
+    private fun calculateArgument(parameter: TypeParameterDescriptor, arguments: List<TypeProjection>, depth: Int): TypeProjection {
+        if (depth > 3) {
+            return StarProjectionImpl(parameter)
+        }
+
         // Inv<A>, Inv<A> = Inv<A>
         if (parameter.variance == Variance.INVARIANT && arguments.all { it.projectionKind == Variance.INVARIANT }) {
             val first = arguments.first()
@@ -183,7 +196,7 @@ object NewCommonSuperTypeCalculator {
         // CS(Out<X>, Out<Y>) = Out<CS(X, Y)>
         // CS(In<X>, In<Y>) = In<X & Y>
         if (asOut) {
-            val type = commonSuperType(arguments.map { it.type.unwrap() })
+            val type = commonSuperType(arguments.map { it.type.unwrap() }, depth + 1)
             return if (parameter.variance != Variance.INVARIANT) return type.asTypeProjection() else TypeProjectionImpl(Variance.OUT_VARIANCE, type)
         }
         else {

compiler/testData/codegen/box/regressions/supertypeDepth.kt

@@ -0,0 +1,18 @@
+class A : FirstOwner<Holder<A>>
+class B : SecondOwner<Holder<B>>
+
+interface FirstOwner<T : StubElement<*>> : SecondOwner<T>
+interface SecondOwner<T : StubElement<*>>
+
+interface StubElement<T>
+
+interface Holder<T> : StubElement<T>
+
+fun test(a: A?, b: B) {
+    val c = a ?: b
+}
+
+fun box(): String {
+    test(A(), B())
+    return "OK"
+}

compiler/tests-ir-jvm/tests/org/jetbrains/kotlin/codegen/ir/IrBlackBoxCodegenTestGenerated.java

@@ -16952,6 +16952,12 @@ public class IrBlackBoxCodegenTestGenerated extends AbstractIrBlackBoxCodegenTes
             doTest(fileName);
         }
 
+        @TestMetadata("supertypeDepth.kt")
+        public void testSupertypeDepth() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/regressions/supertypeDepth.kt");
+            doTest(fileName);
+        }
+
         @TestMetadata("typeCastException.kt")
         public void testTypeCastException() throws Exception {
             String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/regressions/typeCastException.kt");

compiler/tests/org/jetbrains/kotlin/codegen/BlackBoxCodegenTestGenerated.java

@@ -16952,6 +16952,12 @@ public class BlackBoxCodegenTestGenerated extends AbstractBlackBoxCodegenTest {
             doTest(fileName);
         }
 
+        @TestMetadata("supertypeDepth.kt")
+        public void testSupertypeDepth() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/regressions/supertypeDepth.kt");
+            doTest(fileName);
+        }
+
         @TestMetadata("typeCastException.kt")
         public void testTypeCastException() throws Exception {
             String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/regressions/typeCastException.kt");

compiler/tests/org/jetbrains/kotlin/codegen/LightAnalysisModeTestGenerated.java

@@ -16952,6 +16952,12 @@ public class LightAnalysisModeTestGenerated extends AbstractLightAnalysisModeTes
             doTest(fileName);
         }
 
+        @TestMetadata("supertypeDepth.kt")
+        public void testSupertypeDepth() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/regressions/supertypeDepth.kt");
+            doTest(fileName);
+        }
+
         @TestMetadata("typeCastException.kt")
         public void testTypeCastException() throws Exception {
             String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/regressions/typeCastException.kt");

js/js.tests/test/org/jetbrains/kotlin/js/test/semantics/JsCodegenBoxTestGenerated.java

@@ -20732,6 +20732,12 @@ public class JsCodegenBoxTestGenerated extends AbstractJsCodegenBoxTest {
             throw new AssertionError("Looks like this test can be unmuted. Remove IGNORE_BACKEND directive for that.");
         }
 
+        @TestMetadata("supertypeDepth.kt")
+        public void testSupertypeDepth() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/regressions/supertypeDepth.kt");
+            doTest(fileName);
+        }
+
         @TestMetadata("typeCastException.kt")
         public void testTypeCastException() throws Exception {
             String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/regressions/typeCastException.kt");