[FIR] Handle protected effective visibility relation properly

#KT-38401 Fixed

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/FirEffectiveVisibilityImpl.kt

@@ -7,7 +7,10 @@ package org.jetbrains.kotlin.fir
 
 import org.jetbrains.kotlin.descriptors.Visibilities
 import org.jetbrains.kotlin.fir.FirEffectiveVisibility.Permissiveness
+import org.jetbrains.kotlin.fir.declarations.FirClass
 import org.jetbrains.kotlin.fir.declarations.FirRegularClass
+import org.jetbrains.kotlin.fir.resolve.lookupSuperTypes
+import org.jetbrains.kotlin.fir.types.classId
 
 sealed class FirEffectiveVisibilityImpl(
     override val name: String,
@@ -185,18 +188,27 @@ sealed class FirEffectiveVisibilityImpl(
         Permissiveness.MORE -> other
         Permissiveness.UNKNOWN -> Private
     }
+
+    companion object {
+        private fun containerRelation(first: FirRegularClass?, second: FirRegularClass?): Permissiveness =
+            if (first == null || second == null) {
+                Permissiveness.UNKNOWN
+            } else if (first == second) {
+                Permissiveness.SAME
+            } else {
+                when {
+                    first.collectSupertypes().any { it.classId == second.symbol.classId } -> Permissiveness.LESS
+                    second.collectSupertypes().any { it.classId == first.symbol.classId } -> Permissiveness.MORE
+                    else -> Permissiveness.UNKNOWN
+                }
+            }
+
+        private fun FirRegularClass.collectSupertypes() = lookupSuperTypes(
+            this as FirClass<*>,
+            lookupInterfaces = true,
+            deep = true,
+            useSiteSession = this.session
+        )
+    }
 }
 
-internal fun containerRelation(first: FirRegularClass?, second: FirRegularClass?): Permissiveness =
-    if (first == null || second == null) {
-        Permissiveness.UNKNOWN
-    } else if (first == second) {
-        Permissiveness.SAME
-        // TODO
-//    } else if (DescriptorUtils.isSubclass(first, second)) {
-//        Permissiveness.LESS
-//    } else if (DescriptorUtils.isSubclass(second, first)) {
-//        Permissiveness.MORE
-    } else {
-        Permissiveness.UNKNOWN
-    }

compiler/testData/diagnostics/tests/exposed/internalAndProtected.fir.kt

@@ -1,10 +0,0 @@
-public open class A {
-    protected open class B
-}
-
-public open class C : A() {
-    protected open class D {
-        // internal & protected(in C) <= protected(in A): Ok
-        internal open class E : <!EXPOSED_SUPER_CLASS!>A.B<!>()
-    }
-}

compiler/testData/diagnostics/tests/exposed/internalAndProtected.kt

@@ -1,3 +1,4 @@
+// FIR_IDENTICAL
 public open class A {
     protected open class B
 }

compiler/testData/diagnostics/tests/exposed/protectedJava.fir.kt

@@ -10,7 +10,7 @@ public abstract class Outer {
 
 class OuterDerived: Outer() {
     // valid, My has better visibility
-    protected class His: <!EXPOSED_SUPER_CLASS!>Outer.My<!>()
+    protected class His: Outer.My()
     // valid, My and Your have better visibility
     override fun <!EXPOSED_FUNCTION_RETURN_TYPE!>foo<!>(<!EXPOSED_PARAMETER_TYPE!>my: Outer.My<!>) = Outer.Your()
 }

compiler/testData/diagnostics/tests/exposed/protectedSameWay.fir.kt

@@ -7,7 +7,7 @@ abstract class Outer {
 
 class OuterDerived: Outer() {
     // valid, My has better visibility
-    protected class His: <!EXPOSED_SUPER_CLASS!>Outer.My<!>()
+    protected class His: Outer.My()
     // valid, My and Your have better visibility
     override fun <!EXPOSED_FUNCTION_RETURN_TYPE!>foo<!>(<!EXPOSED_PARAMETER_TYPE!>my: Outer.My<!>) = Outer.Your()
 }

compiler/testData/diagnostics/tests/imports/ImportProtectedClass.fir.kt

@@ -7,7 +7,7 @@ open class Foo {
 }
 
 class Bar: Foo() {
-    protected fun <!EXPOSED_FUNCTION_RETURN_TYPE!>foo<!>(): Nested? = null
+    protected fun foo(): Nested? = null
 }
 
 private fun foo(): Nested? = null