Fix StackOverflowError while mapping recursive intersection-type

#KT-10972 Fixed

compiler/backend/src/org/jetbrains/kotlin/codegen/state/KotlinTypeMapper.java

@@ -76,6 +76,7 @@ import org.jetbrains.kotlin.serialization.deserialization.DeserializedType;
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedCallableMemberDescriptor;
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedClassDescriptor;
 import org.jetbrains.kotlin.types.*;
+import org.jetbrains.kotlin.types.typeUtil.TypeUtilsKt;
 import org.jetbrains.kotlin.util.OperatorNameConventions;
 import org.jetbrains.org.objectweb.asm.Type;
 import org.jetbrains.org.objectweb.asm.commons.Method;
@@ -429,6 +430,15 @@ public class KotlinTypeMapper {
         TypeConstructor constructor = jetType.getConstructor();
         if (constructor instanceof IntersectionTypeConstructor) {
             jetType = CommonSupertypes.commonSupertype(new ArrayList<KotlinType>(constructor.getSupertypes()));
+
+            // interface In<in E>
+            // open class A : In<A>
+            // open class B : In<B>
+            // commonSupertype(A, B) = In<A & B>
+            // So replace arguments with star-projections to prevent infinite recursive mapping
+            // It's not very important because such types anyway are prohibited in declarations
+            jetType = TypeUtilsKt.replaceArgumentsWithStarProjections(jetType);
+
             constructor = jetType.getConstructor();
         }
         DeclarationDescriptor descriptor = constructor.getDeclarationDescriptor();

compiler/testData/codegen/box/regressions/nestedIntersection.kt

@@ -0,0 +1,16 @@
+// WITH_RUNTIME
+
+interface In<in E>
+open class A : In<A>
+open class B : In<B>
+
+inline fun <reified T : Any> select(x: T, y: T) = T::class.java.simpleName
+
+// This test checks mostly that no StackOverflow happens while mapping type argument of select-call (In<A & B>)
+// See KT-10972
+fun foo(): String = select(A(), B())
+
+fun box(): String {
+    if (foo() != "In") return "fail"
+    return "OK"
+}

compiler/tests/org/jetbrains/kotlin/codegen/BlackBoxCodegenTestGenerated.java

@@ -12193,6 +12193,12 @@ public class BlackBoxCodegenTestGenerated extends AbstractBlackBoxCodegenTest {
             doTest(fileName);
         }
 
+        @TestMetadata("nestedIntersection.kt")
+        public void testNestedIntersection() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/regressions/nestedIntersection.kt");
+            doTest(fileName);
+        }
+
         @TestMetadata("objectCaptureOuterConstructorProperty.kt")
         public void testObjectCaptureOuterConstructorProperty() throws Exception {
             String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/regressions/objectCaptureOuterConstructorProperty.kt");

core/descriptor.loader.java/src/org/jetbrains/kotlin/load/java/lazy/types/LazyJavaTypeResolver.kt

@@ -37,6 +37,7 @@ import org.jetbrains.kotlin.types.*
 import org.jetbrains.kotlin.types.Variance.*
 import org.jetbrains.kotlin.types.typeUtil.createProjection
 import org.jetbrains.kotlin.types.typeUtil.replaceAnnotations
+import org.jetbrains.kotlin.types.typeUtil.replaceArgumentsWithStarProjections
 import org.jetbrains.kotlin.utils.sure
 import org.jetbrains.kotlin.utils.toReadOnlyList
 
@@ -415,27 +416,3 @@ internal fun TypeParameterDescriptor.getErasedUpperBound(
     return defaultValue()
 }
 
-private fun KotlinType.replaceArgumentsWithStarProjections(): KotlinType {
-    if (constructor.parameters.isEmpty() || constructor.declarationDescriptor == null) return this
-
-    // We could just create JetTypeImpl with current type constructor and star projections,
-    // but we want to preserve flexibility of type, and that it what TypeSubstitutor does
-    return TypeSubstitutor.create(ConstantStarSubstitution).substitute(this, Variance.INVARIANT)!!
-}
-
-private object ConstantStarSubstitution : TypeSubstitution() {
-    override fun get(key: KotlinType): TypeProjection? {
-        // Let substitutor deal with flexibility
-        if (key.isFlexible()) return null
-
-        val newProjections = key.constructor.parameters.map(::StarProjectionImpl)
-
-        val substitution = TypeConstructorSubstitution.create(key.constructor, newProjections)
-
-        return TypeProjectionImpl(
-                TypeSubstitutor.create(substitution).substitute(key.constructor.declarationDescriptor!!.defaultType, Variance.INVARIANT)!!
-        )
-    }
-
-    override fun isEmpty() = false
-}

core/descriptors/src/org/jetbrains/kotlin/types/TypeUtils.kt

@@ -154,3 +154,28 @@ fun KotlinType.getImmediateSuperclassNotAny(): KotlinType? {
 
 fun KotlinType.asTypeProjection(): TypeProjection = TypeProjectionImpl(this)
 fun KotlinType.contains(predicate: (KotlinType) -> Boolean) = TypeUtils.contains(this, predicate)
+
+fun KotlinType.replaceArgumentsWithStarProjections(): KotlinType {
+    if (constructor.parameters.isEmpty() || constructor.declarationDescriptor == null) return this
+
+    // We could just create JetTypeImpl with current type constructor and star projections,
+    // but we want to preserve flexibility of type, and that it what TypeSubstitutor does
+    return TypeSubstitutor.create(ConstantStarSubstitution).substitute(this, Variance.INVARIANT)!!
+}
+
+private object ConstantStarSubstitution : TypeSubstitution() {
+    override fun get(key: KotlinType): TypeProjection? {
+        // Let substitutor deal with flexibility
+        if (key.isFlexible()) return null
+
+        val newProjections = key.constructor.parameters.map(::StarProjectionImpl)
+
+        val substitution = TypeConstructorSubstitution.create(key.constructor, newProjections)
+
+        return TypeProjectionImpl(
+                TypeSubstitutor.create(substitution).substitute(key.constructor.declarationDescriptor!!.defaultType, Variance.INVARIANT)!!
+        )
+    }
+
+    override fun isEmpty() = false
+}