FIR: Rewrite visibility checking

Unbound it from implicit receiver stack as it only needs scope structure/declaration nestedness
Semantics for protected has been changed in a way it works in old FE

NB: We should report additional diagnostic in case of CallCompanionProtectedNonStatic.fir.kt
(see KT-38814)

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/FirCallResolver.kt

@@ -121,7 +121,7 @@ class FirCallResolver(
             typeArguments,
             session,
             file,
-            transformer.components.implicitReceiverStack,
+            transformer.components.containingDeclarations,
         )
         towerResolver.reset()
         val result = towerResolver.runResolver(
@@ -279,7 +279,7 @@ class FirCallResolver(
             typeArguments = typeArguments,
             session,
             file,
-            implicitReceiverStack,
+            containingDeclarations,
         )
         towerResolver.reset()
         val result = towerResolver.runResolverForDelegatingConstructor(
@@ -334,7 +334,7 @@ class FirCallResolver(
             emptyList(),
             session,
             file,
-            transformer.components.implicitReceiverStack,
+            transformer.components.containingDeclarations,
             candidateForCommonInvokeReceiver = null,
             // Additional things for callable reference resolve
             expectedType,

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/resolve/BodyResolveComponents.kt

@@ -31,6 +31,7 @@ interface SessionHolder {
 interface BodyResolveComponents : SessionHolder {
     val returnTypeCalculator: ReturnTypeCalculator
     val implicitReceiverStack: ImplicitReceiverStack
+    val containingDeclarations: List<FirDeclaration>
     val fileImportsScope: List<FirScope>
     val typeParametersScopes: List<FirScope>
     val localScopes: FirLocalScopes

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/resolve/calls/Candidate.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.fir.resolve.calls
 
 import org.jetbrains.kotlin.fir.FirSession
+import org.jetbrains.kotlin.fir.declarations.FirDeclaration
 import org.jetbrains.kotlin.fir.declarations.FirFile
 import org.jetbrains.kotlin.fir.declarations.FirValueParameter
 import org.jetbrains.kotlin.fir.expressions.FirArgumentList
@@ -16,7 +17,6 @@ import org.jetbrains.kotlin.fir.expressions.impl.FirExpressionStub
 import org.jetbrains.kotlin.fir.expressions.impl.FirNoReceiverExpression
 import org.jetbrains.kotlin.fir.resolve.BodyResolveComponents
 import org.jetbrains.kotlin.fir.resolve.DoubleColonLHS
-import org.jetbrains.kotlin.fir.resolve.ImplicitReceiverStack
 import org.jetbrains.kotlin.fir.resolve.inference.PostponedResolvedAtom
 import org.jetbrains.kotlin.fir.resolve.substitution.ConeSubstitutor
 import org.jetbrains.kotlin.fir.symbols.AbstractFirBasedSymbol
@@ -42,7 +42,7 @@ data class CallInfo(
     val typeArguments: List<FirTypeProjection>,
     val session: FirSession,
     val containingFile: FirFile,
-    val implicitReceiverStack: ImplicitReceiverStack,
+    val containingDeclarations: List<FirDeclaration>,
 
     val candidateForCommonInvokeReceiver: Candidate? = null,
 

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/resolve/calls/ResolverParts.kt

@@ -335,54 +335,72 @@ internal object CheckVisibility : CheckerStage() {
     private fun ClassId.isSame(other: ClassId): Boolean =
         packageFqName == other.packageFqName && relativeClassName == other.relativeClassName
 
-    private fun ImplicitReceiverStack.canSeePrivateMemberOf(ownerId: ClassId): Boolean {
-        for (implicitReceiverValue in receiversAsReversed()) {
-            if (implicitReceiverValue !is ImplicitDispatchReceiverValue) continue
-            if (implicitReceiverValue.companionFromSupertype) continue
-            val boundSymbol = implicitReceiverValue.boundSymbol
+    private fun canSeePrivateMemberOf(
+        containingDeclarationOfUseSite: List<FirDeclaration>,
+        ownerId: ClassId,
+        session: FirSession
+    ): Boolean {
+        ownerId.ownerIfCompanion(session)?.let { companionOwnerClassId ->
+            return canSeePrivateMemberOf(containingDeclarationOfUseSite, companionOwnerClassId, session)
+        }
+
+        for (declaration in containingDeclarationOfUseSite) {
+            if (declaration !is FirClass<*>) continue
+            val boundSymbol = declaration.symbol
             if (boundSymbol.classId.isSame(ownerId)) {
                 return true
             }
         }
+
         return false
     }
 
-    private fun FirRegularClassSymbol.canSeeProtectedMemberOf(
-        ownerId: ClassId,
-        session: FirSession,
-        visited: MutableSet<ClassId>
-    ): Boolean {
-        if (classId in visited) return false
-        visited += classId
+    private fun ClassId.ownerIfCompanion(session: FirSession): ClassId? {
+        if (outerClassId == null || isLocal) return null
+        val ownerSymbol = session.firSymbolProvider.getClassLikeSymbolByFqName(this) as? FirRegularClassSymbol
+
+        return outerClassId.takeIf { ownerSymbol?.fir?.isCompanion == true }
+    }
+
+    private fun FirClass<*>.isSubClass(ownerId: ClassId, session: FirSession): Boolean {
         if (classId.isSame(ownerId)) return true
 
-        fir.companionObject?.let { companion ->
-            if (companion.classId.isSame(ownerId)) return true
+        return lookupSuperTypes(this, lookupInterfaces = true, deep = true, session).any { superType ->
+            (superType as? ConeClassLikeType)?.fullyExpandedType(session)?.lookupTag?.classId?.isSame(ownerId) == true
+        }
+    }
+
+    private fun canSeeProtectedMemberOf(
+        containingUseSiteClass: FirClass<*>,
+        dispatchReceiver: ReceiverValue?,
+        ownerId: ClassId, session: FirSession
+    ): Boolean {
+        dispatchReceiver?.ownerIfCompanion(session)?.let { companionOwnerClassId ->
+            if (containingUseSiteClass.isSubClass(companionOwnerClassId, session)) return true
         }
 
-        val superTypes = fir.superConeTypes
-        for (superType in superTypes) {
-            val superTypeSymbol = superType.lookupTag.toSymbol(session) as? FirRegularClassSymbol ?: continue
-            if (superTypeSymbol.canSeeProtectedMemberOf(ownerId, session, visited)) return true
+        // TODO: Add check for receiver, see org.jetbrains.kotlin.descriptors.Visibility#doesReceiverFitForProtectedVisibility
+        return containingUseSiteClass.isSubClass(ownerId, session)
+    }
+
+    private fun canSeeProtectedMemberOf(
+        containingDeclarationOfUseSite: List<FirDeclaration>,
+        dispatchReceiver: ReceiverValue?,
+        ownerId: ClassId, session: FirSession
+    ): Boolean {
+        if (canSeePrivateMemberOf(containingDeclarationOfUseSite, ownerId, session)) return true
+
+        for (containingDeclaration in containingDeclarationOfUseSite) {
+            if (containingDeclaration !is FirClass<*>) continue
+            val boundSymbol = containingDeclaration.symbol
+            if (canSeeProtectedMemberOf(boundSymbol.fir, dispatchReceiver, ownerId, session)) return true
         }
+
         return false
     }
 
-    private fun ImplicitReceiverStack.canSeeProtectedMemberOf(ownerId: ClassId, session: FirSession): Boolean {
-        if (canSeePrivateMemberOf(ownerId)) return true
-        val visited = mutableSetOf<ClassId>()
-        for (implicitReceiverValue in receiversAsReversed()) {
-            if (implicitReceiverValue !is ImplicitDispatchReceiverValue) continue
-            if (implicitReceiverValue.companionFromSupertype) continue
-            val boundSymbol = implicitReceiverValue.boundSymbol
-            val superTypes = boundSymbol.fir.superConeTypes
-            for (superType in superTypes) {
-                val superTypeSymbol = superType.lookupTag.toSymbol(session) as? FirRegularClassSymbol ?: continue
-                if (superTypeSymbol.canSeeProtectedMemberOf(ownerId, session, visited)) return true
-            }
-        }
-        return false
-    }
+    private fun ReceiverValue?.ownerIfCompanion(session: FirSession): ClassId? =
+        (this?.type as? ConeClassLikeType)?.lookupTag?.classId?.ownerIfCompanion(session)
 
     private fun AbstractFirBasedSymbol<*>.getOwnerId(): ClassId? {
         return when (this) {
@@ -409,10 +427,11 @@ internal object CheckVisibility : CheckerStage() {
         declaration: FirMemberDeclaration,
         symbol: AbstractFirBasedSymbol<*>,
         sink: CheckerSink,
-        callInfo: CallInfo
+        candidate: Candidate
     ): Boolean {
+        val callInfo = candidate.callInfo
         val useSiteFile = callInfo.containingFile
-        val implicitReceiverStack = callInfo.implicitReceiverStack
+        val containingDeclarations = callInfo.containingDeclarations
         val session = callInfo.session
         val provider = session.firProvider
         val candidateFile = when (symbol) {
@@ -435,20 +454,20 @@ internal object CheckVisibility : CheckerStage() {
                         candidateFile == useSiteFile
                     } else {
                         // Member: visible inside parent class, including all its member classes
-                        implicitReceiverStack.canSeePrivateMemberOf(ownerId)
+                        canSeePrivateMemberOf(containingDeclarations, ownerId, session)
                     }
                 } else {
                     false
                 }
             }
             Visibilities.PROTECTED -> {
-                ownerId != null && implicitReceiverStack.canSeeProtectedMemberOf(ownerId, session)
+                ownerId != null && canSeeProtectedMemberOf(containingDeclarations, candidate.dispatchReceiverValue, ownerId, session)
             }
             JavaVisibilities.PROTECTED_AND_PACKAGE, JavaVisibilities.PROTECTED_STATIC_VISIBILITY -> {
                 if (symbol.packageFqName() == useSiteFile.packageFqName) {
                     true
                 } else {
-                    ownerId != null && implicitReceiverStack.canSeeProtectedMemberOf(ownerId, session)
+                    ownerId != null && canSeeProtectedMemberOf(containingDeclarations, candidate.dispatchReceiverValue, ownerId, session)
                 }
             }
             else -> true
@@ -465,7 +484,7 @@ internal object CheckVisibility : CheckerStage() {
         val symbol = candidate.symbol
         val declaration = symbol.fir
         if (declaration is FirMemberDeclaration) {
-            if (!checkVisibility(declaration, symbol, sink, callInfo)) {
+            if (!checkVisibility(declaration, symbol, sink, candidate)) {
                 return
             }
         }
@@ -479,7 +498,7 @@ internal object CheckVisibility : CheckerStage() {
                 if (classSymbol.fir.classKind.isSingleton) {
                     sink.yieldApplicability(CandidateApplicability.HIDDEN)
                 }
-                checkVisibility(classSymbol.fir, classSymbol, sink, callInfo)
+                checkVisibility(classSymbol.fir, classSymbol, sink, candidate)
             }
         }
     }

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/resolve/transformers/FirSyntheticCallGenerator.kt

@@ -158,7 +158,7 @@ class FirSyntheticCallGenerator(
         typeArguments = emptyList(),
         session = session,
         containingFile = file,
-        implicitReceiverStack = implicitReceiverStack
+        containingDeclarations = containingDeclarations
     )
 
     private fun generateSyntheticSelectTypeParameter(): Pair<FirTypeParameter, FirResolvedTypeRef> {
@@ -276,4 +276,4 @@ private object UpdateReference : FirTransformer<FirNamedReferenceWithCandidate>(
     override fun transformReference(reference: FirReference, data: FirNamedReferenceWithCandidate): CompositeTransformResult<FirReference> {
         return data.compose()
     }
-}
+}

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/resolve/transformers/body/resolve/FirAbstractBodyResolveTransformer.kt

@@ -215,6 +215,7 @@ abstract class FirAbstractBodyResolveTransformer(phase: FirResolvePhase) : FirAb
         override val localScopes: FirLocalScopes get() = context.localScopes
         override val file: FirFile get() = context.file
         override val implicitReceiverStack: ImplicitReceiverStack get() = context.implicitReceiverStack
+        override val containingDeclarations: List<FirDeclaration> get() = context.containers
         override val localContextForAnonymousFunctions: LocalContextForAnonymousFunctions get() = context.localContextForAnonymousFunctions
         override val returnTypeCalculator: ReturnTypeCalculator get() = context.returnTypeCalculator
         override val container: FirDeclaration get() = context.containerIfAny!!

compiler/testData/codegen/box/statics/protectedStatic.kt

@@ -1,4 +1,3 @@
-// IGNORE_BACKEND_FIR: JVM_IR
 // TARGET_BACKEND: JVM
 
 // FILE: First.java

compiler/testData/diagnostics/tests/typealias/typeAliasConstructorInSuperCall.fir.kt

@@ -3,12 +3,12 @@ open class MyBase protected constructor() {
 }
 typealias MyAlias = MyBase
 
-class MyDerived1 : <!INAPPLICABLE_CANDIDATE!>MyAlias<!>()
+class MyDerived1 : MyAlias()
 class MyDerived1a : MyBase()
 
-class MyDerived2 : <!INAPPLICABLE_CANDIDATE!>MyAlias<!>(null)
+class MyDerived2 : MyAlias(null)
 class MyDerived2a : MyBase(null)
 
 class MyDerived3 : MyAlias {
-    constructor(x: Nothing?) : <!INAPPLICABLE_CANDIDATE!>super<!>(x)
-}
+    constructor(x: Nothing?) : super(x)
+}

compiler/testData/diagnostics/testsWithStdLib/CallCompanionProtectedNonStatic.fir.kt

@@ -48,7 +48,7 @@ class Derived : Base() {
         foo() // Ok
         gav() // Ok
         bar()
-        <!INAPPLICABLE_CANDIDATE!>baz<!>()
+        baz()
         prop = 0
     }
 
@@ -57,7 +57,7 @@ class Derived : Base() {
             foo() // Ok
             gav() // Ok
             bar()
-            <!INAPPLICABLE_CANDIDATE!>baz<!>()
+            baz()
             prop = 0
         }
     }
@@ -66,7 +66,7 @@ class Derived : Base() {
         fun test2() {
             gav() // Ok
             bar()
-            <!INAPPLICABLE_CANDIDATE!>baz<!>()
+            baz()
             prop = 0
         }
     }