[FIR] Fix processing of nested type arguments, extract extractTypeRefAndSourceFromTypeArgument method to FirHelpers and use it from FirClassVarianceChecker and FirConflictingProjectionChecker

compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/FirHelpers.kt

@@ -578,3 +578,41 @@ internal fun checkCondition(condition: FirExpression, context: CheckerContext, r
         reporter.reportOn(condition.source, FirErrors.CONDITION_TYPE_MISMATCH, coneType, context)
     }
 }
+
+fun extractTypeRefAndSourceFromTypeArgument(typeRef: FirTypeRef?, index: Int): Pair<FirTypeRef, FirSourceElement?>? {
+    if (typeRef is FirResolvedTypeRef) {
+        val delegatedTypeRef = typeRef.delegatedTypeRef
+        if (delegatedTypeRef is FirUserTypeRef) {
+            var currentTypeArguments: List<FirTypeProjection>? = null
+            var currentIndex = index
+            val qualifier = delegatedTypeRef.qualifier
+
+            for (i in qualifier.size - 1 downTo 0) {
+                val typeArguments = qualifier[i].typeArgumentList.typeArguments
+                if (currentIndex < typeArguments.size) {
+                    currentTypeArguments = typeArguments
+                    break
+                } else {
+                    currentIndex -= typeArguments.size
+                }
+            }
+
+            val typeArgument = currentTypeArguments?.elementAtOrNull(currentIndex)
+            if (typeArgument is FirTypeProjectionWithVariance) {
+                return Pair(typeArgument.typeRef, typeArgument.source)
+            }
+        } else if (delegatedTypeRef is FirFunctionTypeRef) {
+            val valueParameters = delegatedTypeRef.valueParameters
+            if (index < valueParameters.size) {
+                val valueParamTypeRef = valueParameters.elementAt(index).returnTypeRef
+                return Pair(valueParamTypeRef, valueParamTypeRef.source)
+            }
+            if (index == valueParameters.size) {
+                val returnTypeRef = delegatedTypeRef.returnTypeRef
+                return Pair(returnTypeRef, returnTypeRef.source)
+            }
+        }
+    }
+
+    return null
+}

compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/declaration/FirClassVarianceChecker.kt

@@ -8,6 +8,7 @@ package org.jetbrains.kotlin.fir.analysis.checkers.declaration
 import org.jetbrains.kotlin.descriptors.Visibilities
 import org.jetbrains.kotlin.fir.*
 import org.jetbrains.kotlin.fir.analysis.checkers.context.CheckerContext
+import org.jetbrains.kotlin.fir.analysis.checkers.extractTypeRefAndSourceFromTypeArgument
 import org.jetbrains.kotlin.fir.analysis.diagnostics.DiagnosticReporter
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors
 import org.jetbrains.kotlin.fir.declarations.*
@@ -154,11 +155,11 @@ object FirClassVarianceChecker : FirClassChecker() {
                     }
 
                     if (newVariance != null) {
-                        val subTypeRef = extractTypeArgumentTypeRef(typeRef, index)
+                        val subTypeRefAndSource = extractTypeRefAndSourceFromTypeArgument(typeRef, index)
 
                         checkVarianceConflict(
-                            typeArgumentType, newVariance, subTypeRef, containingType,
+                            typeArgumentType, newVariance, subTypeRefAndSource?.first, containingType,
-                            context, reporter,subTypeRef?.source ?: source,
+                            context, reporter, subTypeRefAndSource?.first?.source ?: source,
                             fullyExpandedType != type
                         )
                     }
@@ -166,25 +167,4 @@ object FirClassVarianceChecker : FirClassChecker() {
             }
         }
     }
-
-    private fun extractTypeArgumentTypeRef(typeRef: FirTypeRef?, index: Int): FirTypeRef? {
-        if (typeRef is FirResolvedTypeRef) {
-            val delegatedTypeRef = typeRef.delegatedTypeRef
-            if (delegatedTypeRef is FirUserTypeRef) {
-                val typeArgument = delegatedTypeRef.qualifier[0].typeArgumentList.typeArguments.elementAtOrNull(index)
-                if (typeArgument is FirTypeProjectionWithVariance) {
-                    return typeArgument.typeRef
-                }
-            } else if (delegatedTypeRef is FirFunctionTypeRef) {
-                if (index < delegatedTypeRef.valueParameters.size) {
-                    return delegatedTypeRef.valueParameters.elementAt(index).returnTypeRef
-                }
-                if (index == delegatedTypeRef.valueParameters.size) {
-                    return delegatedTypeRef.returnTypeRef
-                }
-            }
-        }
-
-        return null
-    }
 }

compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/declaration/FirProjectionRelationChecker.kt

@@ -45,17 +45,19 @@ object FirProjectionRelationChecker : FirBasicDeclarationChecker() {
     private fun checkTypeRef(
         typeRef: FirTypeRef,
         context: CheckerContext,
-        reporter: DiagnosticReporter,
+        reporter: DiagnosticReporter
     ) {
         val type = typeRef.coneTypeSafe<ConeClassLikeType>()
         val fullyExpandedType = type?.fullyExpandedType(context.session) ?: return
         val declaration = fullyExpandedType.toSymbol(context.session)?.fir.safeAs<FirRegularClass>() ?: return
+        val typeParameters = declaration.typeParameters
+        val typeArguments = type.typeArguments
 
-        val size = minOf(declaration.typeParameters.size, typeRef.coneType.typeArguments.size)
+        val size = minOf(typeParameters.size, typeArguments.size)
 
         for (it in 0 until size) {
-            val proto = declaration.typeParameters[it]
+            val proto = typeParameters[it]
-            val actual = typeRef.coneType.typeArguments[it]
+            val actual = typeArguments[it]
             val fullyExpandedProjection = fullyExpandedType.typeArguments[it]
 
             val protoVariance = proto.safeAs<FirTypeParameterRef>()
@@ -88,6 +90,8 @@ object FirProjectionRelationChecker : FirBasicDeclarationChecker() {
                     context
                 )
             }
+
+            checkTypeRef(typeArgTypeRef, context, reporter)
         }
     }
 

compiler/testData/diagnostics/tests/typealias/substitutionVariance.fir.kt

@@ -65,3 +65,6 @@ fun invOut_Inv(x: Inv3<Int>) = x
 fun invOut_In(x: Inv3<<!CONFLICTING_PROJECTION_IN_TYPEALIAS_EXPANSION!>in<!> Int>) = x
 fun invOut_Out(x: Inv3<out Int>) = x
 fun invOut_Star(x: Inv3<*>) = x
+
+fun nested_conflicting_type_argument(x: In<Out<<!CONFLICTING_PROJECTION!>in<!> Int>>) = x
+fun nested_redundant_type_argument(x: In<Out<<!REDUNDANT_PROJECTION!>out<!> Int>>) = x

compiler/testData/diagnostics/tests/typealias/substitutionVariance.kt

@@ -65,3 +65,6 @@ fun invOut_Inv(x: Inv3<Int>) = x
 fun invOut_In(x: <!CONFLICTING_PROJECTION_IN_TYPEALIAS_EXPANSION!>Inv3<in Int><!>) = x
 fun invOut_Out(x: Inv3<out Int>) = x
 fun invOut_Star(x: Inv3<*>) = x
+
+fun nested_conflicting_type_argument(x: In<Out<<!CONFLICTING_PROJECTION!>in<!> Int>>) = x
+fun nested_redundant_type_argument(x: In<Out<<!REDUNDANT_PROJECTION!>out<!> Int>>) = x

compiler/testData/diagnostics/tests/typealias/substitutionVariance.ni.txt

@@ -25,6 +25,8 @@ public fun invOut_In(/*0*/ x: Inv3<in kotlin.Int> /* = Inv<in kotlin.Int> */): I
 public fun invOut_Inv(/*0*/ x: Inv3<kotlin.Int> /* = Inv<out kotlin.Int> */): Inv3<kotlin.Int> /* = Inv<out kotlin.Int> */
 public fun invOut_Out(/*0*/ x: Inv3<out kotlin.Int> /* = Inv<out kotlin.Int> */): Inv3<out kotlin.Int> /* = Inv<out kotlin.Int> */
 public fun invOut_Star(/*0*/ x: Inv3<*> /* = Inv<*> */): Inv3<*> /* = Inv<*> */
+public fun nested_conflicting_type_argument(/*0*/ x: In<Out<in kotlin.Int>>): In<[ERROR : Inconsistent type: Out<in Int> (0 parameter has declared variance: out, but argument variance is in)]>
+public fun nested_redundant_type_argument(/*0*/ x: In<Out<out kotlin.Int>>): In<Out<out kotlin.Int>>
 public fun outIn_In(/*0*/ x: Out2<in kotlin.Int> /* = Out<in kotlin.Int> */): [ERROR : Inconsistent type: Out<in Int> (0 parameter has declared variance: out, but argument variance is in)]
 public fun outIn_Inv(/*0*/ x: Out2<kotlin.Int> /* = Out<in kotlin.Int> */): [ERROR : Inconsistent type: Out<in Int> (0 parameter has declared variance: out, but argument variance is in)]
 public fun outIn_Out(/*0*/ x: Out2<out kotlin.Int> /* = Out<out kotlin.Int> */): Out2<out kotlin.Int> /* = Out<out kotlin.Int> */
@@ -70,3 +72,4 @@ public typealias Out1</*0*/ T> = Out<T>
 public typealias Out2</*0*/ T> = Out<in T>
 public typealias Out3</*0*/ T> = Out<out T>
 public typealias Out4</*0*/ T> = Out<*>
+

compiler/tests-spec/testData/diagnostics/notLinked/dfa/pos/9.fir.kt

@@ -23,7 +23,7 @@ fun case_1(x: Out<<!REDUNDANT_PROJECTION!>out<!> Int?>?) {
  * UNEXPECTED BEHAVIOUR
  * ISSUES: KT-28598
  */
-fun case_2(a: Out<<!REDUNDANT_PROJECTION!>out<!> Out<out Out<out Out<out Out<out Out<out Int?>?>?>?>?>?>?) {
+fun case_2(a: Out<<!REDUNDANT_PROJECTION!>out<!> Out<<!REDUNDANT_PROJECTION!>out<!> Out<<!REDUNDANT_PROJECTION!>out<!> Out<<!REDUNDANT_PROJECTION!>out<!> Out<<!REDUNDANT_PROJECTION!>out<!> Out<<!REDUNDANT_PROJECTION!>out<!> Int?>?>?>?>?>?>?) {
     if (a != null) {
         val b = <!DEBUG_INFO_EXPRESSION_TYPE("Out<out Out<out Out<out Out<out Out<out Out<out kotlin.Int?>?>?>?>?>?>? & Out<out Out<out Out<out Out<out Out<out Out<out kotlin.Int?>?>?>?>?>?>")!>a<!>.get()
         if (b != null) {