Avoid subtype checking between type variable of self type and captured star projection only for invariant positions

Subtyping with non-invariant positions may produce useful constraints

^KT-46001 Fixed

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirOldFrontendDiagnosticsTestGenerated.java

@@ -11913,6 +11913,12 @@ public class FirOldFrontendDiagnosticsTestGenerated extends AbstractFirDiagnosti
                 runTest("compiler/testData/diagnostics/tests/inference/dependantOnVarianceNullable.kt");
             }
 
+            @Test
+            @TestMetadata("discardInapplicableCandidateWithNotSatisfyingSelfType.kt")
+            public void testDiscardInapplicableCandidateWithNotSatisfyingSelfType() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/inference/discardInapplicableCandidateWithNotSatisfyingSelfType.kt");
+            }
+
             @Test
             @TestMetadata("equalitySubstitutionInsideNonInvariantType.kt")
             public void testEqualitySubstitutionInsideNonInvariantType() throws Exception {
@@ -12387,6 +12393,12 @@ public class FirOldFrontendDiagnosticsTestGenerated extends AbstractFirDiagnosti
                 runTest("compiler/testData/diagnostics/tests/inference/useFunctionLiteralsToInferType.kt");
             }
 
+            @Test
+            @TestMetadata("violatingUpperBoundForSelfType.kt")
+            public void testViolatingUpperBoundForSelfType() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/inference/violatingUpperBoundForSelfType.kt");
+            }
+
             @Nested
             @TestMetadata("compiler/testData/diagnostics/tests/inference/builderInference")
             @TestDataPath("$PROJECT_ROOT")

compiler/testData/diagnostics/tests/inference/discardInapplicableCandidateWithNotSatisfyingSelfType.kt

@@ -0,0 +1,13 @@
+// FIR_IDENTICAL
+// WITH_RUNTIME
+
+interface WithChildren<out T>
+
+fun <T : WithChildren<*>> WithChildren<WithChildren<*>>.test() {
+    withDescendants()
+}
+
+fun <T : WithChildren<T>> T.withDescendants() {}
+
+@JvmName("foo")
+fun WithChildren<*>.withDescendants() {}

compiler/testData/diagnostics/tests/inference/discardInapplicableCandidateWithNotSatisfyingSelfType.txt

@@ -0,0 +1,11 @@
+package
+
+public fun </*0*/ T : WithChildren<*>> WithChildren<WithChildren<*>>.test(): kotlin.Unit
+public fun </*0*/ T : WithChildren<T>> T.withDescendants(): kotlin.Unit
+@kotlin.jvm.JvmName(name = "foo") public fun WithChildren<*>.withDescendants(): kotlin.Unit
+
+public interface WithChildren</*0*/ out T> {
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}

compiler/testData/diagnostics/tests/inference/violatingUpperBoundForSelfType.fir.kt

@@ -0,0 +1,9 @@
+// WITH_RUNTIME
+// !DIAGNOSTICS: -UNUSED_PARAMETER -CAST_NEVER_SUCCEEDS
+
+fun <E : Enum<E>> createMap(enumClass: Class<E>) {}
+
+fun reproduce() {
+    val enumClass: Class<Enum<*>> = "any" as Class<Enum<*>>
+    <!INAPPLICABLE_CANDIDATE!>createMap<!>(enumClass)
+}

compiler/testData/diagnostics/tests/inference/violatingUpperBoundForSelfType.kt

@@ -0,0 +1,9 @@
+// WITH_RUNTIME
+// !DIAGNOSTICS: -UNUSED_PARAMETER -CAST_NEVER_SUCCEEDS
+
+fun <E : Enum<E>> createMap(enumClass: Class<E>) {}
+
+fun reproduce() {
+    val enumClass: Class<Enum<*>> = "any" as Class<Enum<*>>
+    createMap(<!TYPE_MISMATCH, TYPE_MISMATCH!>enumClass<!>)
+}

compiler/testData/diagnostics/tests/inference/violatingUpperBoundForSelfType.txt

@@ -0,0 +1,4 @@
+package
+
+public fun </*0*/ E : kotlin.Enum<E>> createMap(/*0*/ enumClass: java.lang.Class<E>): kotlin.Unit
+public fun reproduce(): kotlin.Unit

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/DiagnosticTestGenerated.java

@@ -11919,6 +11919,12 @@ public class DiagnosticTestGenerated extends AbstractDiagnosticTest {
                 runTest("compiler/testData/diagnostics/tests/inference/dependantOnVarianceNullable.kt");
             }
 
+            @Test
+            @TestMetadata("discardInapplicableCandidateWithNotSatisfyingSelfType.kt")
+            public void testDiscardInapplicableCandidateWithNotSatisfyingSelfType() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/inference/discardInapplicableCandidateWithNotSatisfyingSelfType.kt");
+            }
+
             @Test
             @TestMetadata("equalitySubstitutionInsideNonInvariantType.kt")
             public void testEqualitySubstitutionInsideNonInvariantType() throws Exception {
@@ -12393,6 +12399,12 @@ public class DiagnosticTestGenerated extends AbstractDiagnosticTest {
                 runTest("compiler/testData/diagnostics/tests/inference/useFunctionLiteralsToInferType.kt");
             }
 
+            @Test
+            @TestMetadata("violatingUpperBoundForSelfType.kt")
+            public void testViolatingUpperBoundForSelfType() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/inference/violatingUpperBoundForSelfType.kt");
+            }
+
             @Nested
             @TestMetadata("compiler/testData/diagnostics/tests/inference/builderInference")
             @TestDataPath("$PROJECT_ROOT")

core/compiler.common/src/org/jetbrains/kotlin/types/AbstractTypeChecker.kt

@@ -406,9 +406,13 @@ object AbstractTypeChecker {
                 it.getType()
             }
 
-            val isTypeVariableAgainstStarProjectionForSelfType =
+            val variance = effectiveVariance(superTypeConstructor.getParameter(index).getVariance(), superProjection.getVariance())
+                ?: return isErrorTypeEqualsToAnything // todo exception?
+
+            val isTypeVariableAgainstStarProjectionForSelfType = if (variance == TypeVariance.INV) {
                 isTypeVariableAgainstStarProjectionForSelfType(subArgumentType, superArgumentType, superTypeConstructor) ||
                         isTypeVariableAgainstStarProjectionForSelfType(superArgumentType, subArgumentType, superTypeConstructor)
+            } else false
 
             /*
              * We don't check subtyping between types like CapturedType(*) and TypeVariable(E) if the corresponding type parameter forms self type, for instance, Enum<E: Enum<E>>.
@@ -417,10 +421,8 @@ object AbstractTypeChecker {
              * Instead this type check we move on self-type level anyway: checking CapturedType(out Enum<*>) against TypeVariable(E).
              * This subtyping can already be successful and not add unwanted constraints in the type inference context.
              */
-            if (isTypeVariableAgainstStarProjectionForSelfType) continue
-
-            val variance = effectiveVariance(superTypeConstructor.getParameter(index).getVariance(), superProjection.getVariance())
-                ?: return isErrorTypeEqualsToAnything // todo exception?
+            if (isTypeVariableAgainstStarProjectionForSelfType)
+                continue
 
             val correctArgument = runWithArgumentsSettings(subArgumentType) {
                 when (variance) {