[FIR] Forbid erroneous ===-checks

It was decided to forbid such comparisons,
as we know how `===` works. Also, added some more
test cases, just for comparison.

Reusing the proper `canHaveSubtypes()`
from `TypeUtils` prevents a breaking change
in:

- `comparingTripleWithPair.kt`
- `comparisonOfGenericInterfaceWithGenericClass.kt`

But it does lead to warnings
(instead of errors) in
`incompatibleEnumEntryClasses.kt`, which is an
unrelated mistake that will be fixed in the next
commit.

The refactoring in `canHaveSubtypes()` is purely
cosmetic - otherwise reading these conditions is hard
(and they don't fit my screen vertically).

^KT-62646
^KT-65541
^KT-57779

compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/expression/FirEqualityCompatibilityChecker.kt

@@ -25,7 +25,6 @@ import org.jetbrains.kotlin.fir.expressions.*
 import org.jetbrains.kotlin.fir.isPrimitiveType
 import org.jetbrains.kotlin.fir.resolve.fullyExpandedType
 import org.jetbrains.kotlin.fir.symbols.impl.FirClassSymbol
-import org.jetbrains.kotlin.fir.symbols.impl.FirRegularClassSymbol
 import org.jetbrains.kotlin.fir.types.*
 import org.jetbrains.kotlin.fir.types.impl.ConeClassLikeTypeImpl
 import org.jetbrains.kotlin.name.StandardClassIds
@@ -101,15 +100,11 @@ object FirEqualityCompatibilityChecker : FirEqualityOperatorCallChecker(MppCheck
     }
 
     private fun checkIdentityApplicability(l: TypeInfo, r: TypeInfo, context: CheckerContext): Applicability {
-        // The compiler should only check comparisons
-        // when identity-less types or builtins are involved.
-
-        val oneIsBuiltin = l.isBuiltin || r.isBuiltin
         val oneIsNotNull = !l.type.isNullable || !r.type.isNullable
 
         return when {
             l.isIdentityLess || r.isIdentityLess -> Applicability.INAPPLICABLE_AS_IDENTITY_LESS
-            oneIsBuiltin && oneIsNotNull && shouldReportAsPerRules1(l, r, context) -> getInapplicabilityFor(l, r)
+            oneIsNotNull && shouldReportAsPerRules1(l, r, context) -> getInapplicabilityFor(l, r)
             else -> Applicability.APPLICABLE
         }
     }
@@ -126,13 +121,12 @@ object FirEqualityCompatibilityChecker : FirEqualityOperatorCallChecker(MppCheck
     }
 
     private fun shouldReportAsPerRules1(l: TypeInfo, r: TypeInfo, context: CheckerContext): Boolean {
-        // Builtins are always final classes, so
-        // we only need to check if one is related
-        // to the other
+        val oneIsFinal = l.isFinal || r.isFinal
 
         return when {
             l.type.isNothingOrNullableNothing || r.type.isNothingOrNullableNothing -> false
-            else -> !l.isSubtypeOf(r, context) && !r.isSubtypeOf(l, context)
+            oneIsFinal -> !l.isSubtypeOf(r, context) && !r.isSubtypeOf(l, context)
+            else -> false
         }
     }
 
@@ -302,6 +296,7 @@ private class TypeInfo(
     val isPrimitive: Boolean,
     val isBuiltin: Boolean,
     val isValueClass: Boolean,
+    val isFinal: Boolean,
     val canHaveSubtypesAccordingToK1: Boolean,
 ) {
     override fun toString() = "$type"
@@ -309,20 +304,6 @@ private class TypeInfo(
 
 private val FirClassSymbol<*>.isBuiltin get() = isPrimitiveType() || classId == StandardClassIds.String || isEnumClass
 
-// This property is used to replicate K1 behavior, and it
-// tries to match the `TypeUtils.canHaveSubtypes(typeChecker, type)`
-// check in the K1 intersector.
-// In K2 enum classes are final, though enum entries are their subclasses.
-private fun ConeKotlinType.canHaveSubtypesAccordingToK1(session: FirSession): Boolean {
-    val symbol = toSymbol(session)
-
-    return when {
-        symbol is FirRegularClassSymbol && symbol.isEnumClass -> true
-        symbol is FirClassSymbol<*> && symbol.isFinalClass -> false
-        else -> true
-    }
-}
-
 private val TypeInfo.isNullableEnum get() = isEnumClass && type.isNullable
 
 private val TypeInfo.isIdentityLess get() = isPrimitive || isValueClass
@@ -350,7 +331,9 @@ private fun ConeKotlinType.toTypeInfo(session: FirSession): TypeInfo {
         isPrimitive = bounds.any { it.isPrimitiveOrNullablePrimitive },
         isBuiltin = bounds.any { it.toClassSymbol(session)?.isBuiltin == true },
         isValueClass = bounds.any { it.toClassSymbol(session)?.isInline == true },
-        canHaveSubtypesAccordingToK1(session),
+        isFinal = bounds.any { it.toClassSymbol(session)?.isFinalClass == true },
+        // In K1's intersector, `canHaveSubtypes()` is called for `nullabilityStripped`.
+        withNullability(ConeNullability.NOT_NULL, session.typeContext).canHaveSubtypes(session),
     )
 }
 

compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/expression/FirPrivateToThisAccessChecker.kt

@@ -38,7 +38,8 @@ object FirPrivateToThisAccessChecker : FirQualifiedAccessExpressionChecker(MppCh
             // If there was a visibility diagnostic, no need to report another one about visibility
             when (reference.diagnostic) {
                 is ConeVisibilityError,
-                is ConeSetterVisibilityError -> return
+                is ConeSetterVisibilityError
+                -> return
             }
         }
         val dispatchReceiver = expression.dispatchReceiver ?: return
@@ -147,12 +148,4 @@ object FirPrivateToThisAccessChecker : FirQualifiedAccessExpressionChecker(MppCh
         }
         return false
     }
-
-    private val ConeTypeProjection.variance: Variance
-        get() = when (this.kind) {
-            ProjectionKind.STAR -> Variance.OUT_VARIANCE
-            ProjectionKind.IN -> Variance.IN_VARIANCE
-            ProjectionKind.OUT -> Variance.OUT_VARIANCE
-            ProjectionKind.INVARIANT -> Variance.INVARIANT
-        }
 }

compiler/fir/cones/src/org/jetbrains/kotlin/fir/types/ConeTypeProjection.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.kotlin.fir.types
 
+import org.jetbrains.kotlin.types.Variance
 import org.jetbrains.kotlin.types.model.TypeArgumentMarker
 
 enum class ProjectionKind {
@@ -84,3 +85,11 @@ fun ConeKotlinTypeProjection.replaceType(newType: ConeKotlinType): ConeKotlinTyp
         is ConeKotlinTypeConflictingProjection -> ConeKotlinTypeConflictingProjection(newType)
     }
 }
+
+val ConeTypeProjection.variance: Variance
+    get() = when (this.kind) {
+        ProjectionKind.STAR -> Variance.OUT_VARIANCE
+        ProjectionKind.IN -> Variance.IN_VARIANCE
+        ProjectionKind.OUT -> Variance.OUT_VARIANCE
+        ProjectionKind.INVARIANT -> Variance.INVARIANT
+    }

compiler/fir/providers/src/org/jetbrains/kotlin/fir/types/TypeUtils.kt

@@ -584,12 +584,16 @@ fun FirCallableDeclaration.isSubtypeOf(
     )
 }
 
+/**
+ * The original K1 function: [org.jetbrains.kotlin.types.TypeUtils.canHaveSubtypes].
+ */
 fun ConeKotlinType.canHaveSubtypes(session: FirSession): Boolean {
     if (this.isMarkedNullable) {
         return true
     }
     val expandedType = fullyExpandedType(session)
     val classSymbol = expandedType.toSymbol(session) as? FirRegularClassSymbol ?: return true
+    // In K2 enum classes are final, though enum entries are their subclasses (which is a compiler implementation detail).
     if (classSymbol.isEnumClass || classSymbol.isExpect || classSymbol.modality != Modality.FINAL) {
         return true
     }
@@ -603,49 +607,18 @@ fun ConeKotlinType.canHaveSubtypes(session: FirSession): Boolean {
 
         val argument = typeProjection.type!! //safe because it is not a star
 
-        when (typeParameterSymbol.variance) {
-            Variance.INVARIANT ->
-                when (typeProjection.kind) {
-                    ProjectionKind.INVARIANT ->
-                        if (lowerThanBound(session.typeContext, argument, typeParameterSymbol) || argument.canHaveSubtypes(session)) {
-                            return true
-                        }
+        val canHaveSubtypes = when (typeProjection.variance) {
+            Variance.OUT_VARIANCE -> argument.canHaveSubtypes(session)
+            Variance.IN_VARIANCE -> argument.lowerThanBound(typeParameterSymbol, session)
+            Variance.INVARIANT -> when (typeParameterSymbol.variance) {
+                Variance.OUT_VARIANCE -> argument.canHaveSubtypes(session)
+                Variance.IN_VARIANCE -> argument.lowerThanBound(typeParameterSymbol, session)
+                Variance.INVARIANT -> argument.canHaveSubtypes(session) || argument.lowerThanBound(typeParameterSymbol, session)
+            }
+        }
 
-                    ProjectionKind.IN ->
-                        if (lowerThanBound(session.typeContext, argument, typeParameterSymbol)) {
-                            return true
-                        }
-
-                    ProjectionKind.OUT ->
-                        if (argument.canHaveSubtypes(session)) {
-                            return true
-                        }
-
-                    ProjectionKind.STAR ->
-                        return true
-                }
-
-            Variance.IN_VARIANCE ->
-                if (typeProjection.kind != ProjectionKind.OUT) {
-                    if (lowerThanBound(session.typeContext, argument, typeParameterSymbol)) {
-                        return true
-                    }
-                } else {
-                    if (argument.canHaveSubtypes(session)) {
-                        return true
-                    }
-                }
-
-            Variance.OUT_VARIANCE ->
-                if (typeProjection.kind != ProjectionKind.IN) {
-                    if (argument.canHaveSubtypes(session)) {
-                        return true
-                    }
-                } else {
-                    if (lowerThanBound(session.typeContext, argument, typeParameterSymbol)) {
-                        return true
-                    }
-                }
+        if (canHaveSubtypes) {
+            return true
         }
     }
 
@@ -668,9 +641,14 @@ fun ConeClassLikeType.toClassSymbol(session: FirSession): FirClassSymbol<*>? {
     return fullyExpandedType(session).toSymbol(session) as? FirClassSymbol<*>
 }
 
-private fun lowerThanBound(context: ConeInferenceContext, argument: ConeKotlinType, typeParameterSymbol: FirTypeParameterSymbol): Boolean {
+/**
+ * The original K1 function: [org.jetbrains.kotlin.types.TypeUtils.lowerThanBound].
+ * This function returns `true` if `argument` suits any bound rather than the
+ * intersection of them all, and it expects there to be at least a single bound.
+ */
+private fun ConeKotlinType.lowerThanBound(typeParameterSymbol: FirTypeParameterSymbol, session: FirSession): Boolean {
     typeParameterSymbol.resolvedBounds.forEach { boundTypeRef ->
-        if (argument != boundTypeRef.coneType && argument.isSubtypeOf(context, boundTypeRef.coneType)) {
+        if (this != boundTypeRef.coneType && isSubtypeOf(session.typeContext, boundTypeRef.coneType)) {
             return true
         }
     }