Effective visibility: lower bounds introduced for all protected and for protected and internal #KT-9540 Fixed

compiler/testData/diagnostics/tests/exposed/protectedInProtected.kt

@@ -0,0 +1,17 @@
+// See KT-9540 
+
+// all protected should have lower bound that is more permissive than private
+open class A {
+    private interface B
+    protected open class C {
+        protected interface D : <!EXPOSED_SUPER_INTERFACE!>B<!>
+    }
+}
+
+// protected and internal should have lower bound that is more permissive than private
+open class AA {
+    private interface BB
+    protected open class CC {
+        internal interface DD : <!EXPOSED_SUPER_INTERFACE!>BB<!>
+    }
+}

compiler/testData/diagnostics/tests/exposed/protectedInProtected.txt

@@ -0,0 +1,53 @@
+package
+
+public open class A {
+    public constructor A()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+
+    private interface B {
+        public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+        public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+        public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+    }
+
+    protected open class C {
+        public constructor C()
+        public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+        public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+        public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+
+        protected interface D : A.B {
+            public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+            public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+            public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+        }
+    }
+}
+
+public open class AA {
+    public constructor AA()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+
+    private interface BB {
+        public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+        public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+        public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+    }
+
+    protected open class CC {
+        public constructor CC()
+        public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+        public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+        public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+
+        internal interface DD : AA.BB {
+            public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+            public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+            public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+        }
+    }
+}

compiler/tests/org/jetbrains/kotlin/checkers/JetDiagnosticsTestGenerated.java

@@ -5826,6 +5826,12 @@ public class JetDiagnosticsTestGenerated extends AbstractJetDiagnosticsTest {
                 doTest(fileName);
             }
 
+            @TestMetadata("protectedInProtected.kt")
+            public void testProtectedInProtected() throws Exception {
+                String fileName = JetTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/exposed/protectedInProtected.kt");
+                doTest(fileName);
+            }
+
             @TestMetadata("protectedJava.kt")
             public void testProtectedJava() throws Exception {
                 String fileName = JetTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/exposed/protectedJava.kt");

core/descriptors/src/org/jetbrains/kotlin/descriptors/EffectiveVisibility.kt

@@ -24,6 +24,17 @@ sealed class EffectiveVisibility(val name: String) {
 
     override fun toString() = name
 
+    //                    Public
+    //                /--/   |  \-------------\
+    // Protected(Base)       |                 \
+    //       |         Protected(Other)        Internal
+    // Protected(Derived) |                   /
+    //             |      |                  /
+    //       ProtectedBound                 /
+    //                     \InternalProtected
+    //                              |
+    //                           Private
+
     object Private : EffectiveVisibility("private") {
         override fun relation(other: EffectiveVisibility) =
                 if (this == other) Permissiveness.SAME else Permissiveness.LESS
@@ -37,9 +48,15 @@ sealed class EffectiveVisibility(val name: String) {
     object Internal : EffectiveVisibility("internal") {
         override fun relation(other: EffectiveVisibility) = when (other) {
             Public -> Permissiveness.LESS
-            Private -> Permissiveness.MORE
+            Private, InternalProtected -> Permissiveness.MORE
-            is Protected -> Permissiveness.UNKNOWN
             Internal -> Permissiveness.SAME
+            ProtectedBound, is Protected -> Permissiveness.UNKNOWN
+        }
+
+        override fun lowerBound(other: EffectiveVisibility) = when (other) {
+            Public -> this
+            Private, InternalProtected, Internal -> other
+            ProtectedBound, is Protected -> InternalProtected
         }
     }
 
@@ -53,7 +70,7 @@ sealed class EffectiveVisibility(val name: String) {
 
         override fun relation(other: EffectiveVisibility) = when (other) {
             Public -> Permissiveness.LESS
-            Private -> Permissiveness.MORE
+            Private, ProtectedBound, InternalProtected -> Permissiveness.MORE
             is Protected -> {
                 if (container == null || other.container == null) {
                     Permissiveness.UNKNOWN
@@ -73,23 +90,59 @@ sealed class EffectiveVisibility(val name: String) {
             }
             Internal -> Permissiveness.UNKNOWN
         }
+
+        override fun lowerBound(other: EffectiveVisibility) = when (other) {
+            Public -> this
+            Private, ProtectedBound, InternalProtected -> other
+            is Protected -> when (relation(other)) {
+                Permissiveness.SAME, Permissiveness.MORE -> this
+                Permissiveness.LESS -> other
+                Permissiveness.UNKNOWN -> ProtectedBound
+            }
+            Internal -> InternalProtected
+        }
     }
 
-    private enum class Permissiveness {
+    // Lower bound for all protected visibilities
+    object ProtectedBound : EffectiveVisibility("protected(_)") {
+        override fun relation(other: EffectiveVisibility) = when (other) {
+            Public, is Protected -> Permissiveness.LESS
+            Private, InternalProtected -> Permissiveness.MORE
+            ProtectedBound -> Permissiveness.SAME
+            Internal -> Permissiveness.UNKNOWN
+        }
+
+        override fun lowerBound(other: EffectiveVisibility) = when (other) {
+            Public, is Protected -> this
+            Private, ProtectedBound, InternalProtected -> other
+            Internal -> InternalProtected
+        }
+    }
+
+    // Lower bound for Internal and Protected
+    object InternalProtected : EffectiveVisibility("internal/protected") {
+        override fun relation(other: EffectiveVisibility) = when (other) {
+            Public, is Protected, ProtectedBound, Internal -> Permissiveness.LESS
+            Private -> Permissiveness.MORE
+            InternalProtected -> Permissiveness.SAME
+        }
+    }
+
+    protected enum class Permissiveness {
         LESS,
         SAME,
         MORE,
         UNKNOWN
     }
 
-    abstract fun relation(other: EffectiveVisibility): Permissiveness
+    abstract protected fun relation(other: EffectiveVisibility): Permissiveness
 
     fun sameOrMorePermissive(other: EffectiveVisibility) = when (relation(other)) {
         Permissiveness.SAME, Permissiveness.MORE -> true
         Permissiveness.LESS, Permissiveness.UNKNOWN -> false
     }
 
-    fun lowerBound(other: EffectiveVisibility) = when (relation(other)) {
+    open protected fun lowerBound(other: EffectiveVisibility) = when (relation(other)) {
         Permissiveness.SAME, Permissiveness.LESS -> this
         Permissiveness.MORE -> other
         Permissiveness.UNKNOWN -> Private