Fix visibility of protected classes in bytecode

Protected should be translated to public as in Java so that everything would
work at runtime. The real visibility is still saved to an InnerClasses
attribute

 #KT-8269 Fixed
 #KT-9246 Fixed
 #KT-10143 Fixed

compiler/backend/src/org/jetbrains/kotlin/codegen/AsmUtil.java

@@ -265,6 +265,7 @@ public class AsmUtil {
             return getVisibilityAccessFlagForAnonymous(descriptor);
         }
         if (descriptor.getVisibility() == Visibilities.PUBLIC ||
+            descriptor.getVisibility() == Visibilities.PROTECTED ||
             // TODO: should be package private, but for now Kotlin's reflection can't access members of such classes
             descriptor.getVisibility() == Visibilities.LOCAL ||
             descriptor.getVisibility() == Visibilities.INTERNAL) {

compiler/testData/codegen/box/innerNested/protectedNestedClass.kt

@@ -0,0 +1,26 @@
+// See KT-9246 IllegalAccessError when trying to access protected nested class from parent class
+// TARGET_BACKEND: JVM
+// FILE: a.kt
+
+package a
+
+abstract class A {
+    protected class C {
+        fun result() = "OK"
+    }
+}
+
+// FILE: b.kt
+
+package b
+
+import a.A
+
+class B : A() {
+    protected val c = A.C()
+    val result: String get() = c.result()
+}
+
+fun box(): String {
+    return B().result
+}

compiler/testData/codegen/box/innerNested/protectedNestedClassFromJava.kt

@@ -0,0 +1,34 @@
+// See KT-8269 java.lang.IllegalAccessError on accessing protected inner class declared in Kotlin super class
+// TARGET_BACKEND: JVM
+// FILE: Test.kt
+
+package com.company
+
+import other.JavaClass
+
+open class Test {
+    protected class ProtectedClass
+}
+
+fun box(): String {
+    JavaClass.test()
+    return "OK"
+}
+
+// FILE: other/JavaClass.java
+
+package other;
+
+import com.company.Test;
+
+public class JavaClass {
+    static class JavaTest extends Test {
+        public static boolean foo(Object obj) {
+            return obj instanceof ProtectedClass;
+        }
+    }
+
+    public static void test() {
+        JavaTest.foo(new Object());
+    }
+}

compiler/testData/codegen/box/regressions/kt10143.kt

@@ -0,0 +1,33 @@
+// FILE: Outer.kt
+
+package another
+open class Outer {
+    protected class Stage(val run: () -> Unit)
+    protected class My(var stage: Stage? = null) {
+        fun initStage(f: () -> Unit): Stage {
+            stage = Stage(f)
+            return stage!!
+        }
+    }
+    protected fun my(init: My.() -> Unit): My {
+        val result = My()
+        result.init()
+        return result
+    }
+}
+
+// FILE: Main.kt
+
+package other
+class Derived : another.Outer() {
+    init {
+        my {
+            initStage { }
+        }
+    }
+}
+
+fun box(): String {
+    Derived()
+    return "OK"
+}

compiler/tests/org/jetbrains/kotlin/codegen/BlackBoxCodegenTestGenerated.java

@@ -6876,6 +6876,18 @@ public class BlackBoxCodegenTestGenerated extends AbstractBlackBoxCodegenTest {
             String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/innerNested/nestedSimple.kt");
             doTest(fileName);
         }
+
+        @TestMetadata("protectedNestedClass.kt")
+        public void testProtectedNestedClass() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/innerNested/protectedNestedClass.kt");
+            doTest(fileName);
+        }
+
+        @TestMetadata("protectedNestedClassFromJava.kt")
+        public void testProtectedNestedClassFromJava() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/innerNested/protectedNestedClassFromJava.kt");
+            doTest(fileName);
+        }
     }
 
     @TestMetadata("compiler/testData/codegen/box/instructions")
@@ -11875,6 +11887,12 @@ public class BlackBoxCodegenTestGenerated extends AbstractBlackBoxCodegenTest {
             doTest(fileName);
         }
 
+        @TestMetadata("kt10143.kt")
+        public void testKt10143() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/regressions/kt10143.kt");
+            doTest(fileName);
+        }
+
         @TestMetadata("kt10934.kt")
         public void testKt10934() throws Exception {
             String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/regressions/kt10934.kt");

js/js.tests/test/org/jetbrains/kotlin/js/test/semantics/InnerNestedTestGenerated.java

@@ -43,6 +43,18 @@ public class InnerNestedTestGenerated extends AbstractInnerNestedTest {
         doTest(fileName);
     }
 
+    @TestMetadata("protectedNestedClass.kt")
+    public void ignoredProtectedNestedClass() throws Exception {
+        String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/innerNested/protectedNestedClass.kt");
+        doTest(fileName);
+    }
+
+    @TestMetadata("protectedNestedClassFromJava.kt")
+    public void ignoredProtectedNestedClassFromJava() throws Exception {
+        String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/innerNested/protectedNestedClassFromJava.kt");
+        doTest(fileName);
+    }
+
     public void testAllFilesPresentInInnerNested() throws Exception {
         KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("compiler/testData/codegen/box/innerNested"), Pattern.compile("^(.+)\\.kt$"), true);
     }