FIR: Fix case when smartcast receiver is used for call to private method

^KT-54432 Fixed

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirOldFrontendDiagnosticsTestGenerated.java

@@ -33508,6 +33508,18 @@ public class FirOldFrontendDiagnosticsTestGenerated extends AbstractFirDiagnosti
                 runTest("compiler/testData/diagnostics/tests/visibility/invisiblePrivateThroughSubClass.kt");
             }
 
+            @Test
+            @TestMetadata("invisiblePrivateThroughSubClassDoubleSmartCast.kt")
+            public void testInvisiblePrivateThroughSubClassDoubleSmartCast() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/visibility/invisiblePrivateThroughSubClassDoubleSmartCast.kt");
+            }
+
+            @Test
+            @TestMetadata("invisiblePrivateThroughSubClassSmartCast.kt")
+            public void testInvisiblePrivateThroughSubClassSmartCast() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/visibility/invisiblePrivateThroughSubClassSmartCast.kt");
+            }
+
             @Test
             @TestMetadata("invisibleSetterOfJavaClass.kt")
             public void testInvisibleSetterOfJavaClass() throws Exception {

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirOldFrontendDiagnosticsWithLightTreeTestGenerated.java

@@ -33508,6 +33508,18 @@ public class FirOldFrontendDiagnosticsWithLightTreeTestGenerated extends Abstrac
                 runTest("compiler/testData/diagnostics/tests/visibility/invisiblePrivateThroughSubClass.kt");
             }
 
+            @Test
+            @TestMetadata("invisiblePrivateThroughSubClassDoubleSmartCast.kt")
+            public void testInvisiblePrivateThroughSubClassDoubleSmartCast() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/visibility/invisiblePrivateThroughSubClassDoubleSmartCast.kt");
+            }
+
+            @Test
+            @TestMetadata("invisiblePrivateThroughSubClassSmartCast.kt")
+            public void testInvisiblePrivateThroughSubClassSmartCast() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/visibility/invisiblePrivateThroughSubClassSmartCast.kt");
+            }
+
             @Test
             @TestMetadata("invisibleSetterOfJavaClass.kt")
             public void testInvisibleSetterOfJavaClass() throws Exception {

compiler/fir/fir2ir/tests-gen/org/jetbrains/kotlin/test/runners/codegen/FirBlackBoxCodegenTestGenerated.java

@@ -45671,6 +45671,12 @@ public class FirBlackBoxCodegenTestGenerated extends AbstractFirBlackBoxCodegenT
             KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/codegen/box/smartCasts"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.JVM_IR, true);
         }
 
+        @Test
+        @TestMetadata("avoidSmartCastToDerivedForPrivate.kt")
+        public void testAvoidSmartCastToDerivedForPrivate() throws Exception {
+            runTest("compiler/testData/codegen/box/smartCasts/avoidSmartCastToDerivedForPrivate.kt");
+        }
+
         @Test
         @TestMetadata("complexExplicitReceiver.kt")
         public void testComplexExplicitReceiver() throws Exception {
@@ -45791,6 +45797,12 @@ public class FirBlackBoxCodegenTestGenerated extends AbstractFirBlackBoxCodegenT
             runTest("compiler/testData/codegen/box/smartCasts/kt48163_smartCastToThrowable.kt");
         }
 
+        @Test
+        @TestMetadata("kt52432.kt")
+        public void testKt52432() throws Exception {
+            runTest("compiler/testData/codegen/box/smartCasts/kt52432.kt");
+        }
+
         @Test
         @TestMetadata("lambdaArgumentWithoutType.kt")
         public void testLambdaArgumentWithoutType() throws Exception {

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/resolve/calls/Candidate.kt

@@ -27,7 +27,7 @@ import org.jetbrains.kotlin.resolve.calls.tower.isSuccess
 
 class Candidate(
     override val symbol: FirBasedSymbol<*>,
-    override val dispatchReceiverValue: ReceiverValue?,
+    override var dispatchReceiverValue: ReceiverValue?,
     // In most cases, it contains zero or single element
     // More than one, only in case of context receiver group
     val givenExtensionReceiverOptions: List<ReceiverValue>,

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/resolve/calls/VisibilityUtils.kt

@@ -6,17 +6,20 @@
 package org.jetbrains.kotlin.fir.resolve.calls
 
 import org.jetbrains.kotlin.descriptors.Visibilities
-import org.jetbrains.kotlin.fir.FirVisibilityChecker
+import org.jetbrains.kotlin.fir.*
 import org.jetbrains.kotlin.fir.declarations.FirBackingField
 import org.jetbrains.kotlin.fir.declarations.FirCallableDeclaration
 import org.jetbrains.kotlin.fir.declarations.FirMemberDeclaration
 import org.jetbrains.kotlin.fir.declarations.FirProperty
 import org.jetbrains.kotlin.fir.declarations.utils.getExplicitBackingField
 import org.jetbrains.kotlin.fir.declarations.utils.visibility
+import org.jetbrains.kotlin.fir.expressions.FirExpressionWithSmartcast
 import org.jetbrains.kotlin.fir.expressions.FirVariableAssignment
-import org.jetbrains.kotlin.fir.isIntersectionOverride
-import org.jetbrains.kotlin.fir.isSubstitutionOverride
-import org.jetbrains.kotlin.fir.originalIfFakeOverride
+import org.jetbrains.kotlin.fir.expressions.builder.buildExpressionWithSmartcast
+import org.jetbrains.kotlin.fir.types.coneType
+import org.jetbrains.kotlin.fir.types.isNullableNothing
+import org.jetbrains.kotlin.fir.types.makeConeTypeDefinitelyNotNullOrNotNull
+import org.jetbrains.kotlin.fir.types.typeContext
 
 fun FirVisibilityChecker.isVisible(
     declaration: FirMemberDeclaration,
@@ -55,27 +58,59 @@ fun FirVisibilityChecker.isVisible(
 ): Boolean {
     val callInfo = candidate.callInfo
 
-    val visible = isVisible(
-        declaration,
-        callInfo,
-        candidate.dispatchReceiverValue
-    )
+    if (!isVisible(declaration, callInfo, candidate.dispatchReceiverValue)) {
+        val dispatchReceiverWithoutSmartCastType =
+            removeSmartCastTypeForAttemptToFitVisibility(candidate.dispatchReceiverValue, candidate.callInfo.session) ?: return false
 
-    if (visible) {
-        val backingField = declaration.getBackingFieldIfApplicable()
-        if (backingField != null) {
-            candidate.hasVisibleBackingField = isVisible(
-                backingField,
-                callInfo.session,
-                callInfo.containingFile,
-                callInfo.containingDeclarations,
-                candidate.dispatchReceiverValue,
-                candidate.callInfo.callSite is FirVariableAssignment,
-            )
+        if (!isVisible(declaration, callInfo, dispatchReceiverWithoutSmartCastType)) return false
+
+        candidate.dispatchReceiverValue = dispatchReceiverWithoutSmartCastType
+    }
+
+    val backingField = declaration.getBackingFieldIfApplicable()
+    if (backingField != null) {
+        candidate.hasVisibleBackingField = isVisible(
+            backingField,
+            callInfo.session,
+            callInfo.containingFile,
+            callInfo.containingDeclarations,
+            candidate.dispatchReceiverValue,
+            candidate.callInfo.callSite is FirVariableAssignment,
+        )
+    }
+
+    return true
+}
+
+private fun removeSmartCastTypeForAttemptToFitVisibility(dispatchReceiverValue: ReceiverValue?, session: FirSession): ReceiverValue? {
+    val expressionWithSmartcastIfStable =
+        (dispatchReceiverValue?.receiverExpression as? FirExpressionWithSmartcast)?.takeIf { it.isStable } ?: return null
+
+    if (dispatchReceiverValue.type.isNullableNothing) return null
+
+    val originalTypeNotNullable =
+        expressionWithSmartcastIfStable.originalType.coneType.makeConeTypeDefinitelyNotNullOrNotNull(session.typeContext)
+
+    // Basically, this `if` is just for sake of optimizaton
+    // We have only nullability enhancement, here, so return initial smart cast receiver value
+    if (originalTypeNotNullable == dispatchReceiverValue.type) return null
+
+    val expressionForReceiver = with(session.typeContext) {
+        when {
+            expressionWithSmartcastIfStable.originalType.coneType.isNullableType() && !dispatchReceiverValue.type.isNullableType() ->
+                buildExpressionWithSmartcast {
+                    originalExpression = expressionWithSmartcastIfStable.originalExpression
+                    smartcastType =
+                        expressionWithSmartcastIfStable.originalExpression.typeRef.resolvedTypeFromPrototype(originalTypeNotNullable)
+                    typesFromSmartCast = listOf(originalTypeNotNullable)
+                    smartcastStability = expressionWithSmartcastIfStable.smartcastStability
+                }
+            else -> expressionWithSmartcastIfStable.originalExpression
         }
     }
 
-    return visible
+    return ExpressionReceiverValue(expressionForReceiver)
+
 }
 
 private fun FirMemberDeclaration.getBackingFieldIfApplicable(): FirBackingField? {

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/resolve/calls/tower/TowerLevels.kt

@@ -138,11 +138,7 @@ class MemberScopeTowerLevel(
             }
 
             val visibleCandidatesFromSmartcast = visibleCandidates.filter { candidatesMapping.getValue(it) }
-            if (visibleCandidatesFromSmartcast.isNotEmpty()) {
-                candidates += visibleCandidatesFromSmartcast
-            } else {
-                group.filterNotTo(candidates) { candidatesMapping.getValue(it) }
-            }
+            candidates += visibleCandidatesFromSmartcast.ifEmpty { group }
         }
         consumeCandidates(output, candidates)
     }