FIR: Refine visibility check for class members

analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/DiagnosisCompilerTestFE10TestdataTestGenerated.java

@@ -32019,6 +32019,18 @@ public class DiagnosisCompilerTestFE10TestdataTestGenerated extends AbstractDiag
                 runTest("compiler/testData/diagnostics/tests/visibility/lackOfInvisibleSetterOfJavaClassInSamePackage.kt");
             }
 
+            @Test
+            @TestMetadata("overrideOfMemberInPackagePrivateClass.kt")
+            public void testOverrideOfMemberInPackagePrivateClass() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/visibility/overrideOfMemberInPackagePrivateClass.kt");
+            }
+
+            @Test
+            @TestMetadata("privateCompanionInSuperClass.kt")
+            public void testPrivateCompanionInSuperClass() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/visibility/privateCompanionInSuperClass.kt");
+            }
+
             @Test
             @TestMetadata("protectedInternal.kt")
             public void testProtectedInternal() throws Exception {

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirOldFrontendDiagnosticsTestGenerated.java

@@ -32019,6 +32019,18 @@ public class FirOldFrontendDiagnosticsTestGenerated extends AbstractFirDiagnosti
                 runTest("compiler/testData/diagnostics/tests/visibility/lackOfInvisibleSetterOfJavaClassInSamePackage.kt");
             }
 
+            @Test
+            @TestMetadata("overrideOfMemberInPackagePrivateClass.kt")
+            public void testOverrideOfMemberInPackagePrivateClass() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/visibility/overrideOfMemberInPackagePrivateClass.kt");
+            }
+
+            @Test
+            @TestMetadata("privateCompanionInSuperClass.kt")
+            public void testPrivateCompanionInSuperClass() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/visibility/privateCompanionInSuperClass.kt");
+            }
+
             @Test
             @TestMetadata("protectedInternal.kt")
             public void testProtectedInternal() throws Exception {

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirOldFrontendDiagnosticsWithLightTreeTestGenerated.java

@@ -32019,6 +32019,18 @@ public class FirOldFrontendDiagnosticsWithLightTreeTestGenerated extends Abstrac
                 runTest("compiler/testData/diagnostics/tests/visibility/lackOfInvisibleSetterOfJavaClassInSamePackage.kt");
             }
 
+            @Test
+            @TestMetadata("overrideOfMemberInPackagePrivateClass.kt")
+            public void testOverrideOfMemberInPackagePrivateClass() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/visibility/overrideOfMemberInPackagePrivateClass.kt");
+            }
+
+            @Test
+            @TestMetadata("privateCompanionInSuperClass.kt")
+            public void testPrivateCompanionInSuperClass() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/visibility/privateCompanionInSuperClass.kt");
+            }
+
             @Test
             @TestMetadata("protectedInternal.kt")
             public void testProtectedInternal() throws Exception {

compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/declaration/FirOverrideChecker.kt

@@ -167,7 +167,14 @@ object FirOverrideChecker : FirClassChecker() {
             it.ensureResolved(FirResolvePhase.STATUS)
             @OptIn(SymbolInternals::class)
             val fir = it.fir
-            visibilityChecker.isVisible(fir, context.session, file, containingDeclarations, null)
+            visibilityChecker.isVisible(
+                fir,
+                context.session,
+                file,
+                containingDeclarations,
+                null,
+                skipCheckForContainingClassVisibility = true
+            )
         }
         if (!hasVisibleBase) {
             //NB: Old FE reports this in an attempt to override private member,

compiler/fir/providers/src/org/jetbrains/kotlin/fir/FirVisibilityChecker.kt

@@ -23,11 +23,10 @@ import org.jetbrains.kotlin.fir.resolve.providers.symbolProvider
 import org.jetbrains.kotlin.fir.resolve.toSymbol
 import org.jetbrains.kotlin.fir.resolve.typeWithStarProjections
 import org.jetbrains.kotlin.fir.symbols.ConeClassLikeLookupTag
+import org.jetbrains.kotlin.fir.symbols.ConeTypeParameterLookupTag
 import org.jetbrains.kotlin.fir.symbols.FirBasedSymbol
 import org.jetbrains.kotlin.fir.symbols.impl.*
-import org.jetbrains.kotlin.fir.types.ConeClassLikeType
+import org.jetbrains.kotlin.fir.types.*
-import org.jetbrains.kotlin.fir.types.coneType
-import org.jetbrains.kotlin.fir.types.typeContext
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.types.AbstractTypeChecker
@@ -67,10 +66,108 @@ abstract class FirVisibilityChecker : FirSessionComponent {
         containingDeclarations: List<FirDeclaration>,
         dispatchReceiver: ReceiverValue?,
         isCallToPropertySetter: Boolean = false,
+        // There's no need to check if containing class is visible in case we check if a member might be overridden in a subclass
+        // because visibility for its supertype that contain overridden member is being checked when resolving the type reference.
+        // Such flag is not necessary in FE1.0, since there are full structure of fake overrides and containing declaration for overridden
+        // is always visible since it's a supertype of a derived class.
+        skipCheckForContainingClassVisibility: Boolean = false,
+    ): Boolean {
+        if (!isSpecificDeclarationVisible(
+                declaration,
+                session,
+                useSiteFile,
+                containingDeclarations,
+                dispatchReceiver,
+                isCallToPropertySetter
+            )
+        ) {
+            return false
+        }
+
+        if (skipCheckForContainingClassVisibility) return true
+
+        val parentClass = declaration.containingNonLocalClass(session, dispatchReceiver) ?: return true
+        return generateSequence(parentClass) { it.containingNonLocalClass(session) }.all { parent ->
+            isSpecificDeclarationVisible(
+                parent,
+                session,
+                useSiteFile,
+                containingDeclarations,
+                dispatchReceiver,
+                isCallToPropertySetter
+            )
+        }
+    }
+
+    private fun FirMemberDeclaration.containingNonLocalClass(
+        session: FirSession,
+        dispatchReceiverValue: ReceiverValue?
+    ): FirClassLikeDeclaration? {
+        return when (this) {
+            is FirCallableDeclaration -> {
+                if (dispatchReceiverValue != null && dispatchReceiverType != null) {
+                    dispatchReceiverValue.type.findClassRepresentation(dispatchReceiverType!!, session)?.let { return it }
+                }
+
+                this.containingClass()?.toSymbol(session)?.fir
+            }
+            is FirClassLikeDeclaration -> containingNonLocalClass(session)
+        }
+    }
+
+    private fun FirClassLikeDeclaration.containingNonLocalClass(session: FirSession): FirClassLikeDeclaration? {
+        return when (this) {
+            is FirClass -> {
+                if (isLocal) return null
+
+                this.classId.outerClassId?.let { session.symbolProvider.getClassLikeSymbolByClassId(it)?.fir }
+            }
+            // Currently, type aliases are only top-level
+            is FirTypeAlias -> null
+        }
+    }
+
+    private fun ConeKotlinType.findClassRepresentation(
+        dispatchReceiverParameterType: ConeKotlinType,
+        session: FirSession
+    ): FirClassLikeDeclaration? =
+        when (this) {
+            is ConeClassLikeType -> this.fullyExpandedType(session).lookupTag.toSymbol(session)?.fir
+            is ConeFlexibleType -> lowerBound.findClassRepresentation(dispatchReceiverParameterType, session)
+            is ConeCapturedType -> constructor.supertypes.orEmpty()
+                .findClassRepresentationThatIsSubtypeOf(dispatchReceiverParameterType, session)
+            is ConeDefinitelyNotNullType -> original.findClassRepresentation(dispatchReceiverParameterType, session)
+            is ConeIntegerLiteralType -> possibleTypes.findClassRepresentationThatIsSubtypeOf(dispatchReceiverParameterType, session)
+            is ConeIntersectionType -> intersectedTypes.findClassRepresentationThatIsSubtypeOf(dispatchReceiverParameterType, session)
+            is ConeTypeParameterType -> lookupTag.findClassRepresentationThatIsSubtypeOf(dispatchReceiverParameterType, session)
+            is ConeTypeVariableType -> (this.lookupTag.originalTypeParameter as? ConeTypeParameterLookupTag)
+                ?.findClassRepresentationThatIsSubtypeOf(dispatchReceiverParameterType, session)
+            is ConeStubType -> (this.variable.typeConstructor.originalTypeParameter as? ConeTypeParameterLookupTag)
+                ?.findClassRepresentationThatIsSubtypeOf(dispatchReceiverParameterType, session)
+            is ConeLookupTagBasedType -> null
+        }
+
+    private fun ConeTypeParameterLookupTag.findClassRepresentationThatIsSubtypeOf(
+        supertype: ConeKotlinType,
+        session: FirSession
+    ): FirClassLikeDeclaration? =
+        typeParameterSymbol.fir.bounds.map { it.coneType }.findClassRepresentationThatIsSubtypeOf(supertype, session)
+
+    private fun Collection<ConeKotlinType>.findClassRepresentationThatIsSubtypeOf(
+        supertype: ConeKotlinType,
+        session: FirSession
+    ): FirClassLikeDeclaration? = firstOrNull { it.isSubtypeOf(supertype, session) }?.findClassRepresentation(supertype, session)
+
+    private fun isSpecificDeclarationVisible(
+        declaration: FirMemberDeclaration,
+        session: FirSession,
+        useSiteFile: FirFile,
+        containingDeclarations: List<FirDeclaration>,
+        dispatchReceiver: ReceiverValue?,
+        isCallToPropertySetter: Boolean = false,
     ): Boolean {
-        require(declaration is FirDeclaration)
-        val provider = session.firProvider
         val symbol = declaration.symbol
+        val provider = session.firProvider
         return when (declaration.visibility) {
             Visibilities.Internal -> {
                 declaration.moduleData == session.moduleData || session.moduleVisibilityChecker?.isInFriendModule(declaration) == true

compiler/testData/diagnostics/tests/annotations/requireKotlin.fir.kt

@@ -8,5 +8,5 @@ class IllegalVersion()
 @<!INVISIBLE_REFERENCE!>RequireKotlin<!>("1.2")
 class LegalMinimum()
 
-@<!INVISIBLE_REFERENCE!>RequireKotlin<!>("1.2", versionKind = <!INVISIBLE_REFERENCE!>RequireKotlinVersionKind<!>.COMPILER_VERSION, message = "Requires newer compiler version to be inlined correctly.")
+@<!INVISIBLE_REFERENCE!>RequireKotlin<!>("1.2", versionKind = <!INVISIBLE_REFERENCE!>RequireKotlinVersionKind<!>.<!INVISIBLE_REFERENCE!>COMPILER_VERSION<!>, message = "Requires newer compiler version to be inlined correctly.")
 class LegalStdLib()

compiler/testData/diagnostics/tests/classObjects/ClassObjectVisibility.fir.kt

@@ -5,7 +5,7 @@ fun test() {
   A.Companion.<!INVISIBLE_REFERENCE!>f<!>
   B.D
   CCC
-  CCC.classObjectVar
+  CCC.<!INVISIBLE_REFERENCE!>classObjectVar<!>
 }
 
 class A() {

compiler/testData/diagnostics/tests/classObjects/companionObjectOfPrivateClassVisibility.fir.kt

@@ -3,7 +3,7 @@ package test
 fun use() {
     Default.create()
 
-    Explicit.create()
+    Explicit.<!INVISIBLE_REFERENCE!>create<!>()
 }
 
 private class Default {

compiler/testData/diagnostics/tests/classObjects/invisibleClassObjects.fir.kt

@@ -40,12 +40,12 @@ fun test() {
     f(<!INVISIBLE_REFERENCE!>D<!>)
 
     A.foo()
-    <!INVISIBLE_REFERENCE!>B<!>.bar()
+    <!INVISIBLE_REFERENCE!>B<!>.<!INVISIBLE_REFERENCE!>bar<!>()
-    C.baz()
+    C.<!INVISIBLE_REFERENCE!>baz<!>()
-    <!INVISIBLE_REFERENCE!>D<!>.quux()
+    <!INVISIBLE_REFERENCE!>D<!>.<!INVISIBLE_REFERENCE!>quux<!>()
 
     a.A.foo()
-    a.C.baz()
+    a.C.<!INVISIBLE_REFERENCE!>baz<!>()
 }
 
 fun f(unused: Any) {}

compiler/testData/diagnostics/tests/classObjects/nestedClassInPrivateClassObject.fir.kt

@@ -12,4 +12,4 @@ class A {
 
 fun <!EXPOSED_FUNCTION_RETURN_TYPE!>f1<!>() = A.Companion.B.<!INVISIBLE_REFERENCE!>C<!>
 
-fun f2() = A.Companion.B.<!INVISIBLE_REFERENCE!>C<!>.foo()
+fun f2() = A.Companion.B.<!INVISIBLE_REFERENCE!>C<!>.<!INVISIBLE_REFERENCE!>foo<!>()

compiler/testData/diagnostics/tests/imports/ImportPrivateMember.fir.kt

@@ -31,7 +31,7 @@ private object TopLevelObject {
 fun testAccess() {
     <!INVISIBLE_REFERENCE!>NestedClass<!>()
     E1
-    InNested()
+    <!INVISIBLE_REFERENCE!>InNested<!>()
-    NestedEntry
+    <!INVISIBLE_REFERENCE!>NestedEntry<!>
     inObject()
 }

compiler/testData/diagnostics/tests/imports/ImportPrivateMemberFromOtherFile.fir.kt

@@ -22,5 +22,5 @@ import B.JC.JC1
 
 fun test() {
     <!INVISIBLE_REFERENCE!>O1<!>
-    JC1()
+    <!INVISIBLE_REFERENCE!>JC1<!>()
 }

compiler/testData/diagnostics/tests/imports/ImportPrivateMembersWithStar.fir.kt

@@ -26,7 +26,7 @@ private enum class TopLevelEnum(private val e: NestedEnum) {
 
 fun testAccess() {
     E1
-    NestedEntry
+    <!INVISIBLE_REFERENCE!>NestedEntry<!>
-    A1()
+    <!INVISIBLE_REFERENCE!>A1<!>()
     <!INVISIBLE_REFERENCE!>A2<!>
 }

compiler/testData/diagnostics/tests/j+k/privateNestedClassStaticMember.fir.kt

@@ -14,5 +14,5 @@ public class Foo {
 // FILE: 1.kt
 
 fun main() {
-    javaPackage.Foo.<!INVISIBLE_REFERENCE!>Bar<!>.doSmth()
+    javaPackage.Foo.<!INVISIBLE_REFERENCE!>Bar<!>.<!INVISIBLE_REFERENCE!>doSmth<!>()
 }

compiler/testData/diagnostics/tests/privateInFile/visibility.fir.kt

@@ -29,7 +29,7 @@ package a
 
 fun test() {
     val y = makeA()
-    y.bar()
+    y.<!INVISIBLE_REFERENCE!>bar<!>()
     <!INVISIBLE_REFERENCE!>foo<!>()
 
     val u : A = <!INVISIBLE_REFERENCE!>A<!>()

compiler/testData/diagnostics/tests/scopes/visibility2.fir.kt

@@ -24,7 +24,7 @@ import a.PO
 
 fun test() {
     val y = makeA()
-    y.bar()
+    y.<!INVISIBLE_REFERENCE!>bar<!>()
     <!INVISIBLE_REFERENCE!>foo<!>()
 
     val u : A = <!INVISIBLE_REFERENCE!>A<!>()

compiler/testData/diagnostics/tests/scopes/visibility3.fir.kt

@@ -25,7 +25,7 @@ package a
 
 fun test() {
     val y = makeA()
-    y.bar()
+    y.<!INVISIBLE_REFERENCE!>bar<!>()
     <!INVISIBLE_REFERENCE!>foo<!>()
 
     val u : A = <!INVISIBLE_REFERENCE!>A<!>()

compiler/testData/diagnostics/tests/smartCasts/smartcastToInvisibleType_java.fir.kt

@@ -33,7 +33,7 @@ fun testSmartcast(x: Any) {
 
 fun testInference(a: A, b: B) {
     val x = <!DEBUG_INFO_EXPRESSION_TYPE("foo.PackagePrivateInterface")!>select(a, b)<!>
-    x.foo()
+    x.<!INVISIBLE_REFERENCE!>foo<!>()
 }
 
 // FILE: samePackage.kt

compiler/testData/diagnostics/tests/smartCasts/smartcastToInvisibleType_kotlin.fir.kt

@@ -39,5 +39,5 @@ fun testSmartcast(x: Any) {
 
 fun testInference(a: A, b: B) {
     val x = <!DEBUG_INFO_EXPRESSION_TYPE("foo.PrivateInterface")!>select(a, b)<!>
-    x.foo()
+    x.<!INVISIBLE_REFERENCE!>foo<!>()
 }

compiler/testData/diagnostics/tests/visibility/overrideOfMemberInPackagePrivateClass.kt

@@ -0,0 +1,20 @@
+// FIR_IDENTICAL
+// SKIP_TXT
+// FILE: test/JavaBase.java
+package test;
+
+abstract /* package-private */ class JavaBase {
+    public void foo() {}
+}
+
+// FILE: test/JavaBase2.java
+package test;
+public class JavaBase2 extends JavaBase {}
+
+// FILE: main.kt
+
+import test.*
+
+class KotlinClass : JavaBase2() {
+    override fun foo() {}
+}

compiler/testData/diagnostics/tests/visibility/privateCompanionInSuperClass.fir.kt

@@ -0,0 +1,24 @@
+// SKIP_TXT
+open class BaseWithPrivate {
+    private companion object {
+        val X: Int = 1
+        val Y: Int = 1
+    }
+}
+
+open class Base {
+    companion object {
+        val X: String = ""
+    }
+}
+
+class Derived : Base() {
+    fun foo() {
+        object : BaseWithPrivate() {
+            fun bar() {
+                X.length
+                <!INVISIBLE_REFERENCE!>Y<!>.hashCode()
+            }
+        }
+    }
+}

compiler/testData/diagnostics/tests/visibility/privateCompanionInSuperClass.kt

@@ -0,0 +1,24 @@
+// SKIP_TXT
+open class BaseWithPrivate {
+    private companion object {
+        val X: Int = 1
+        val Y: Int = 1
+    }
+}
+
+open class Base {
+    companion object {
+        val X: String = ""
+    }
+}
+
+class Derived : Base() {
+    fun foo() {
+        object : BaseWithPrivate() {
+            fun bar() {
+                X.length
+                <!INVISIBLE_MEMBER!>Y<!>.hashCode()
+            }
+        }
+    }
+}

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/DiagnosticTestGenerated.java

@@ -32115,6 +32115,18 @@ public class DiagnosticTestGenerated extends AbstractDiagnosticTest {
                 runTest("compiler/testData/diagnostics/tests/visibility/lackOfInvisibleSetterOfJavaClassInSamePackage.kt");
             }
 
+            @Test
+            @TestMetadata("overrideOfMemberInPackagePrivateClass.kt")
+            public void testOverrideOfMemberInPackagePrivateClass() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/visibility/overrideOfMemberInPackagePrivateClass.kt");
+            }
+
+            @Test
+            @TestMetadata("privateCompanionInSuperClass.kt")
+            public void testPrivateCompanionInSuperClass() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/visibility/privateCompanionInSuperClass.kt");
+            }
+
             @Test
             @TestMetadata("protectedInternal.kt")
             public void testProtectedInternal() throws Exception {