[FIR] Use isSubtypeOfClass with platform types mapping instead of isSubtypeOf during checking CAST_NEVER_SUCCEEDS

Refactor and generalize code

^KT-62783 Fixed

analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/DiagnosticCompilerTestFE10TestdataTestGenerated.java

@@ -5790,6 +5790,12 @@ public class DiagnosticCompilerTestFE10TestdataTestGenerated extends AbstractDia
                 public void testNoGenericsUnrelated() throws Exception {
                     runTest("compiler/testData/diagnostics/tests/cast/neverSucceeds/NoGenericsUnrelated.kt");
                 }
+
+                @Test
+                @TestMetadata("NullableExprToItsNonNullableGenericBaseClass.kt")
+                public void testNullableExprToItsNonNullableGenericBaseClass() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/cast/neverSucceeds/NullableExprToItsNonNullableGenericBaseClass.kt");
+                }
             }
         }
 

analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/LLFirPreresolvedReversedDiagnosticCompilerFE10TestDataTestGenerated.java

@@ -5790,6 +5790,12 @@ public class LLFirPreresolvedReversedDiagnosticCompilerFE10TestDataTestGenerated
                 public void testNoGenericsUnrelated() throws Exception {
                     runTest("compiler/testData/diagnostics/tests/cast/neverSucceeds/NoGenericsUnrelated.kt");
                 }
+
+                @Test
+                @TestMetadata("NullableExprToItsNonNullableGenericBaseClass.kt")
+                public void testNullableExprToItsNonNullableGenericBaseClass() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/cast/neverSucceeds/NullableExprToItsNonNullableGenericBaseClass.kt");
+                }
             }
         }
 

compiler/fir/analysis-tests/testData/resolve/checkers/unsuccessfullCastToStartProjectedType.kt

@@ -5,5 +5,5 @@ class Foo1<A> : Foo<Int>()
 class Foo2 : Foo<Int>()
 
 fun process(foo: Foo<Long>) {
-    foo <!CAST_NEVER_SUCCEEDS!>as<!> Foo1<*>
+    foo as Foo1<*>
 }

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirLightTreeOldFrontendDiagnosticsTestGenerated.java

@@ -5784,6 +5784,12 @@ public class FirLightTreeOldFrontendDiagnosticsTestGenerated extends AbstractFir
                 public void testNoGenericsUnrelated() throws Exception {
                     runTest("compiler/testData/diagnostics/tests/cast/neverSucceeds/NoGenericsUnrelated.kt");
                 }
+
+                @Test
+                @TestMetadata("NullableExprToItsNonNullableGenericBaseClass.kt")
+                public void testNullableExprToItsNonNullableGenericBaseClass() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/cast/neverSucceeds/NullableExprToItsNonNullableGenericBaseClass.kt");
+                }
             }
         }
 

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirPsiOldFrontendDiagnosticsTestGenerated.java

@@ -5790,6 +5790,12 @@ public class FirPsiOldFrontendDiagnosticsTestGenerated extends AbstractFirPsiDia
                 public void testNoGenericsUnrelated() throws Exception {
                     runTest("compiler/testData/diagnostics/tests/cast/neverSucceeds/NoGenericsUnrelated.kt");
                 }
+
+                @Test
+                @TestMetadata("NullableExprToItsNonNullableGenericBaseClass.kt")
+                public void testNullableExprToItsNonNullableGenericBaseClass() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/cast/neverSucceeds/NullableExprToItsNonNullableGenericBaseClass.kt");
+                }
             }
         }
 

compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/FirCastDiagnosticsHelpers.kt

@@ -11,11 +11,13 @@ import org.jetbrains.kotlin.fir.declarations.utils.isInterface
 import org.jetbrains.kotlin.fir.resolve.defaultType
 import org.jetbrains.kotlin.fir.resolve.substitution.ConeSubstitutorByMap
 import org.jetbrains.kotlin.fir.scopes.platformClassMapper
+import org.jetbrains.kotlin.fir.symbols.impl.FirClassSymbol
 import org.jetbrains.kotlin.fir.symbols.impl.FirRegularClassSymbol
 import org.jetbrains.kotlin.fir.symbols.impl.FirTypeParameterSymbol
 import org.jetbrains.kotlin.fir.types.*
 import org.jetbrains.kotlin.types.AbstractTypeChecker
 import org.jetbrains.kotlin.types.AbstractTypeChecker.findCorrespondingSupertypes
+import org.jetbrains.kotlin.types.TypeCheckerState
 import org.jetbrains.kotlin.types.model.typeConstructor
 
 enum class CastingType {
@@ -32,13 +34,16 @@ fun checkCasting(
 ): CastingType {
     val lhsLowerType = lhsType.lowerBoundIfFlexible()
     val rhsLowerType = rhsType.lowerBoundIfFlexible()
+
+    if (lhsLowerType is ConeErrorType || rhsLowerType is ConeErrorType) return CastingType.Possible
+
     val session = context.session
 
     if (lhsLowerType is ConeIntersectionType) {
         var result = false
         for (intersectedType in lhsLowerType.intersectedTypes) {
             val isIntersectedCastPossible = checkCasting(intersectedType, rhsLowerType, isSafeCase, context)
-            val intersectedTypeSymbol = intersectedType.toRegularClassSymbol(context.session)
+            val intersectedTypeSymbol = intersectedType.toRegularClassSymbol(session)
             if (intersectedTypeSymbol?.isInterface == false && isIntersectedCastPossible == CastingType.Impossible) {
                 return CastingType.Impossible // Any class type in intersection type should be subtype of RHS
             }
@@ -59,20 +64,42 @@ fun checkCasting(
         return if (lhsNullable) CastingType.Possible else CastingType.Impossible
     }
     if (lhsNullable && rhsNullable) return CastingType.Possible
-    val lhsClassSymbol = lhsLowerType.toRegularClassSymbol(context.session)
-    val rhsClassSymbol = rhsLowerType.toRegularClassSymbol(context.session)
-    if (isRelated(lhsLowerType, rhsLowerType, lhsClassSymbol, rhsClassSymbol, context)) return CastingType.Possible
+
     // This is an oversimplification (which does not render the method incomplete):
     // we consider any type parameter capable of taking any value, which may be made more precise if we considered bounds
     if (lhsLowerType is ConeTypeParameterType || rhsLowerType is ConeTypeParameterType) return CastingType.Possible
 
-    if (isFinal(lhsLowerType, session) || isFinal(rhsLowerType, session)) return CastingType.Impossible
-    if (lhsClassSymbol?.isInterface == true || rhsClassSymbol?.isInterface == true) return CastingType.Possible
+    val lhsClassSymbol = lhsLowerType.toRegularClassSymbol(session)
+    val rhsClassSymbol = rhsLowerType.toRegularClassSymbol(session)
+    val lhsNormalizedType = getCorrespondingKotlinClass(lhsClassSymbol?.defaultType() ?: lhsLowerType, session)
+    val rhsNormalizedType = getCorrespondingKotlinClass(rhsClassSymbol?.defaultType() ?: rhsLowerType, session)
+
+    val state = session.typeContext.newTypeCheckerState(errorTypesEqualToAnything = false, stubTypesEqualToAnything = false)
+
+    // It's an optimization, the code below with `isRoughSubtypeOf` also checks subtyping, but it's slower
+    if (AbstractTypeChecker.isSubtypeOf(state, lhsNormalizedType, rhsNormalizedType) ||
+        AbstractTypeChecker.isSubtypeOf(state, rhsNormalizedType, lhsNormalizedType)
+    ) {
+        return CastingType.Possible
+    }
+
+    if (isRoughSubtypeOf(lhsNormalizedType, rhsNormalizedType, state, session) ||
+        isRoughSubtypeOf(rhsNormalizedType, lhsNormalizedType, state, session)
+    ) {
+        return CastingType.Possible
+    }
+
+    if (isFinal(lhsNormalizedType, session) || isFinal(rhsNormalizedType, session)) return CastingType.Impossible
+
+    val lhsNormalizedTypeSymbol = lhsNormalizedType.toSymbol(session) as? FirClassSymbol<*>
+    val rhsNormalizedTypeSymbol = rhsNormalizedType.toSymbol(session) as? FirClassSymbol<*>
+    if (lhsNormalizedTypeSymbol?.isInterface == true || rhsNormalizedTypeSymbol?.isInterface == true) return CastingType.Possible
+
     return CastingType.Impossible
 }
 
 /**
- * Two types are related, roughly, when one of them is a subtype of the other constructing class
+ * One type is roughly subtype of another superType when one of type's supertype constructor equals another superType constructor.
  *
  * Note that some types have platform-specific counterparts, i.e. kotlin.String is mapped to java.lang.String,
  * such types (and all their sub- and supertypes) are related too.
@@ -80,30 +107,29 @@ fun checkCasting(
  * Due to limitations in PlatformToKotlinClassMap, we only consider mapping of platform classes to Kotlin classed
  * (i.e. java.lang.String -> kotlin.String) and ignore mappings that go the other way.
  */
-private fun isRelated(
-    aType: ConeSimpleKotlinType,
-    bType: ConeSimpleKotlinType,
-    aClassSymbol: FirRegularClassSymbol?,
-    bClassSymbol: FirRegularClassSymbol?,
-    context: CheckerContext
+private fun isRoughSubtypeOf(
+    type: ConeSimpleKotlinType,
+    superType: ConeSimpleKotlinType,
+    state: TypeCheckerState,
+    session: FirSession
 ): Boolean {
-    val typeContext = context.session.typeContext
+    var result = false
+    val superTypeConstructor = superType.typeConstructor(state.typeSystemContext)
+    state.anySupertype(type, { typeMarker ->
+        val correspondingKotlinClass = getCorrespondingKotlinClass(typeMarker as ConeSimpleKotlinType, session)
+        if (correspondingKotlinClass.typeConstructor(state.typeSystemContext) == superTypeConstructor) {
+            result = true
+            true
+        } else {
+            false
+        }
+    }, { TypeCheckerState.SupertypesPolicy.LowerIfFlexible })
 
-    if (AbstractTypeChecker.isSubtypeOf(typeContext, aType, bType) ||
-        AbstractTypeChecker.isSubtypeOf(typeContext, bType, aType)
-    ) {
-        return true
-    }
+    return result
+}
 
-    fun getCorrespondingKotlinClass(type: ConeSimpleKotlinType): ConeKotlinType {
-        return context.session.platformClassMapper.getCorrespondingKotlinClass(type.classId)?.defaultType(listOf()) ?: type
-    }
-
-    val aNormalizedType = getCorrespondingKotlinClass(aClassSymbol?.defaultType() ?: aType)
-    val bNormalizedType = getCorrespondingKotlinClass(bClassSymbol?.defaultType() ?: bType)
-
-    return AbstractTypeChecker.isSubtypeOf(typeContext, aNormalizedType, bNormalizedType) ||
-            AbstractTypeChecker.isSubtypeOf(typeContext, bNormalizedType, aNormalizedType)
+private fun getCorrespondingKotlinClass(type: ConeSimpleKotlinType, session: FirSession): ConeSimpleKotlinType {
+    return session.platformClassMapper.getCorrespondingKotlinClass(type.classId)?.defaultType(emptyList()) ?: type
 }
 
 private fun isFinal(type: ConeSimpleKotlinType, session: FirSession): Boolean {

compiler/testData/diagnostics/tests/cast/neverSucceeds/NullableExprToItsNonNullableGenericBaseClass.fir.kt

@@ -0,0 +1,19 @@
+// ISSUE: KT-62783
+
+abstract class Foo<T>
+
+class FooBar : Foo<Any>()
+
+fun test1(value: FooBar) {
+    value as Foo<*>
+    value as? Foo<*>
+    value as Foo<*>?
+    value as? Foo<*>?
+}
+
+fun test2(value: FooBar?) {
+    value as Foo<*>
+    value as? Foo<*>
+    value as Foo<*>?
+    value as? Foo<*>?
+}

compiler/testData/diagnostics/tests/cast/neverSucceeds/NullableExprToItsNonNullableGenericBaseClass.kt

@@ -0,0 +1,19 @@
+// ISSUE: KT-62783
+
+abstract class Foo<T>
+
+class FooBar : Foo<Any>()
+
+fun test1(value: FooBar) {
+    value <!USELESS_CAST!>as Foo<*><!>
+    value <!USELESS_CAST!>as? Foo<*><!>
+    value <!USELESS_CAST!>as Foo<*>?<!>
+    value <!USELESS_CAST!>as? Foo<*>?<!>
+}
+
+fun test2(value: FooBar?) {
+    value as Foo<*>
+    value <!USELESS_CAST!>as? Foo<*><!>
+    value <!USELESS_CAST!>as Foo<*>?<!>
+    value <!USELESS_CAST!>as? Foo<*>?<!>
+}

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/DiagnosticTestGenerated.java

@@ -5790,6 +5790,12 @@ public class DiagnosticTestGenerated extends AbstractDiagnosticTest {
                 public void testNoGenericsUnrelated() throws Exception {
                     runTest("compiler/testData/diagnostics/tests/cast/neverSucceeds/NoGenericsUnrelated.kt");
                 }
+
+                @Test
+                @TestMetadata("NullableExprToItsNonNullableGenericBaseClass.kt")
+                public void testNullableExprToItsNonNullableGenericBaseClass() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/cast/neverSucceeds/NullableExprToItsNonNullableGenericBaseClass.kt");
+                }
             }
         }