[NI] Prioritize variables with trivial constraints over complex ones

Consider the following constraint system (from the test example):

Nothing? <: V1
F!! <: V2
Inv<V1> <: S
Inv<V2> <: S

Where V1, V2, S are type variables, and F has nullable upper bound.
Type variable fixation order should be: V2 -> V1 -> S, and the problem
was that previously after fixation of type variable V2 we were trying
to fix S (before V1), so we had the following constraints on S:
Inv<F!!> <: S
Inv<V1> <: S
=> S were fixed to Inv<F!!>

And after this V1 was fixed to F!! which is contradictory as Nothing?
is not a subtype of F!!.

 #KT-33033 Fixed

compiler/fir/resolve/tests/org/jetbrains/kotlin/fir/FirDiagnosticsSmokeTestGenerated.java

@@ -10334,6 +10334,11 @@ public class FirDiagnosticsSmokeTestGenerated extends AbstractFirDiagnosticsSmok
                 runTest("compiler/testData/diagnostics/tests/inference/constraints/errorUpperBoundConstraint.kt");
             }
 
+            @TestMetadata("fixTypeVariableWithNothingConstraintEarlierThanComplexVariable.kt")
+            public void testFixTypeVariableWithNothingConstraintEarlierThanComplexVariable() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/inference/constraints/fixTypeVariableWithNothingConstraintEarlierThanComplexVariable.kt");
+            }
+
             @TestMetadata("ignoreConstraintFromImplicitInNothing.kt")
             public void testIgnoreConstraintFromImplicitInNothing() throws Exception {
                 runTest("compiler/testData/diagnostics/tests/inference/constraints/ignoreConstraintFromImplicitInNothing.kt");

compiler/resolution/src/org/jetbrains/kotlin/resolve/calls/inference/components/VariableFixationFinder.kt

@@ -52,8 +52,8 @@ class VariableFixationFinder(
     private enum class TypeVariableFixationReadiness {
         FORBIDDEN,
         WITHOUT_PROPER_ARGUMENT_CONSTRAINT, // proper constraint from arguments -- not from upper bound for type parameters
-        WITH_TRIVIAL_OR_NON_PROPER_CONSTRAINTS, // proper trivial constraint from arguments, Nothing <: T
         WITH_COMPLEX_DEPENDENCY, // if type variable T has constraint with non fixed type variable inside (non-top-level): T <: Foo<S>
+        WITH_TRIVIAL_OR_NON_PROPER_CONSTRAINTS, // proper trivial constraint from arguments, Nothing <: T
         RELATED_TO_ANY_OUTPUT_TYPE,
         READY_FOR_FIXATION,
     }
@@ -65,8 +65,8 @@ class VariableFixationFinder(
         !notFixedTypeVariables.contains(variable) ||
                 dependencyProvider.isVariableRelatedToTopLevelType(variable) -> TypeVariableFixationReadiness.FORBIDDEN
         !variableHasProperArgumentConstraints(variable) -> TypeVariableFixationReadiness.WITHOUT_PROPER_ARGUMENT_CONSTRAINT
-        variableHasTrivialOrNonProperConstraints(variable) -> TypeVariableFixationReadiness.WITH_TRIVIAL_OR_NON_PROPER_CONSTRAINTS
         hasDependencyToOtherTypeVariables(variable) -> TypeVariableFixationReadiness.WITH_COMPLEX_DEPENDENCY
+        variableHasTrivialOrNonProperConstraints(variable) -> TypeVariableFixationReadiness.WITH_TRIVIAL_OR_NON_PROPER_CONSTRAINTS
         dependencyProvider.isVariableRelatedToAnyOutputType(variable) -> TypeVariableFixationReadiness.RELATED_TO_ANY_OUTPUT_TYPE
         else -> TypeVariableFixationReadiness.READY_FOR_FIXATION
     }

compiler/testData/diagnostics/tests/inference/constraints/fixTypeVariableWithNothingConstraintEarlierThanComplexVariable.kt

@@ -0,0 +1,16 @@
+// !DIAGNOSTICS: -UNUSED_PARAMETER
+
+class Inv<T>
+
+fun <K> foo(t: K?) = Inv<K>()
+
+fun <T> bar(t: T) = foo(t)
+fun <V1> bar1(t: V1): Inv<V1> = foo(t)
+fun <V2> bar2(t: V2): Inv<V2> = foo(t)
+
+fun <S> select(x: S, y: S): S = x
+
+fun <T> fail(t: T?) = if (t == null) bar(<!DEBUG_INFO_CONSTANT!>t<!>) else bar(<!DEBUG_INFO_SMARTCAST!>t<!>)
+fun <F> fail1(t: F?, n: Nothing?) = select(bar1(<!DEBUG_INFO_CONSTANT!>n<!>), bar2(t!!))
+fun <F> fail2(t: F?, n: Nothing?) = if (t == null) bar1(<!DEBUG_INFO_CONSTANT!>t<!>) else bar2(<!DEBUG_INFO_SMARTCAST!>t<!>)
+fun <F> fail3(t: F?) = select(bar1(null), bar2(t?.let { it }))

compiler/testData/diagnostics/tests/inference/constraints/fixTypeVariableWithNothingConstraintEarlierThanComplexVariable.txt

@@ -0,0 +1,18 @@
+package
+
+public fun </*0*/ T> bar(/*0*/ t: T): Inv<T>
+public fun </*0*/ V1> bar1(/*0*/ t: V1): Inv<V1>
+public fun </*0*/ V2> bar2(/*0*/ t: V2): Inv<V2>
+public fun </*0*/ T> fail(/*0*/ t: T?): Inv<out T?>
+public fun </*0*/ F> fail1(/*0*/ t: F?, /*1*/ n: kotlin.Nothing?): Inv<out F?>
+public fun </*0*/ F> fail2(/*0*/ t: F?, /*1*/ n: kotlin.Nothing?): Inv<out F?>
+public fun </*0*/ F> fail3(/*0*/ t: F?): Inv<out F?>
+public fun </*0*/ K> foo(/*0*/ t: K?): Inv<K>
+public fun </*0*/ S> select(/*0*/ x: S, /*1*/ y: S): S
+
+public final class Inv</*0*/ T> {
+    public constructor Inv</*0*/ T>()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}

compiler/tests/org/jetbrains/kotlin/checkers/DiagnosticsTestGenerated.java

@@ -10341,6 +10341,11 @@ public class DiagnosticsTestGenerated extends AbstractDiagnosticsTest {
                     runTest("compiler/testData/diagnostics/tests/inference/constraints/errorUpperBoundConstraint.kt");
                 }
 
+                @TestMetadata("fixTypeVariableWithNothingConstraintEarlierThanComplexVariable.kt")
+                public void testFixTypeVariableWithNothingConstraintEarlierThanComplexVariable() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/constraints/fixTypeVariableWithNothingConstraintEarlierThanComplexVariable.kt");
+                }
+
                 @TestMetadata("ignoreConstraintFromImplicitInNothing.kt")
                 public void testIgnoreConstraintFromImplicitInNothing() throws Exception {
                     runTest("compiler/testData/diagnostics/tests/inference/constraints/ignoreConstraintFromImplicitInNothing.kt");

compiler/tests/org/jetbrains/kotlin/checkers/javac/DiagnosticsUsingJavacTestGenerated.java

@@ -10336,6 +10336,11 @@ public class DiagnosticsUsingJavacTestGenerated extends AbstractDiagnosticsUsing
                     runTest("compiler/testData/diagnostics/tests/inference/constraints/errorUpperBoundConstraint.kt");
                 }
 
+                @TestMetadata("fixTypeVariableWithNothingConstraintEarlierThanComplexVariable.kt")
+                public void testFixTypeVariableWithNothingConstraintEarlierThanComplexVariable() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/constraints/fixTypeVariableWithNothingConstraintEarlierThanComplexVariable.kt");
+                }
+
                 @TestMetadata("ignoreConstraintFromImplicitInNothing.kt")
                 public void testIgnoreConstraintFromImplicitInNothing() throws Exception {
                     runTest("compiler/testData/diagnostics/tests/inference/constraints/ignoreConstraintFromImplicitInNothing.kt");