Error diagnostic for private classes in inline functions; Fix for KT-7545: IllegalAccessError when accessing private nested class through inlined function from different package

#KT-7545 Fixed

compiler/frontend/src/org/jetbrains/kotlin/diagnostics/Errors.java

@@ -32,7 +32,6 @@ import org.jetbrains.kotlin.resolve.calls.inference.InferenceErrorData;
 import org.jetbrains.kotlin.resolve.calls.model.ResolvedCall;
 import org.jetbrains.kotlin.resolve.varianceChecker.VarianceChecker.VarianceConflictDiagnosticData;
 import org.jetbrains.kotlin.types.KotlinType;
-import org.jetbrains.kotlin.types.TypeProjection;
 
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
@@ -774,6 +773,7 @@ public interface Errors {
 
     //Inline and inlinable parameters
     DiagnosticFactory2<KtElement, DeclarationDescriptor, DeclarationDescriptor> INVISIBLE_MEMBER_FROM_INLINE = DiagnosticFactory2.create(ERROR, CALL_ELEMENT);
+    DiagnosticFactory2<KtElement, DeclarationDescriptor, DeclarationDescriptor> PRIVATE_CLASS_MEMBER_FROM_INLINE = DiagnosticFactory2.create(ERROR, CALL_ELEMENT);
     DiagnosticFactory1<KtElement, KtElement> NON_LOCAL_RETURN_NOT_ALLOWED = DiagnosticFactory1.create(ERROR, CALL_ELEMENT);
     DiagnosticFactory2<KtElement, KtNamedDeclaration, DeclarationDescriptor> NOT_YET_SUPPORTED_IN_INLINE = DiagnosticFactory2.create(ERROR);
     DiagnosticFactory1<PsiElement, DeclarationDescriptor> NOTHING_TO_INLINE = DiagnosticFactory1.create(WARNING);

compiler/frontend/src/org/jetbrains/kotlin/diagnostics/rendering/DefaultErrorMessages.java

@@ -733,6 +733,8 @@ public class DefaultErrorMessages {
 
         //Inline
         MAP.put(INVISIBLE_MEMBER_FROM_INLINE, "Public-API inline function cannot access non-public-API ''{0}''", SHORT_NAMES_IN_TYPES, SHORT_NAMES_IN_TYPES);
+        MAP.put(PRIVATE_CLASS_MEMBER_FROM_INLINE, "Non-private inline function cannot access members of private classes: ''{0}''", SHORT_NAMES_IN_TYPES, SHORT_NAMES_IN_TYPES);
+
         MAP.put(NOT_YET_SUPPORTED_IN_INLINE, "''{0}'' construction is not yet supported in inline functions", ELEMENT_TEXT, SHORT_NAMES_IN_TYPES);
         MAP.put(DECLARATION_CANT_BE_INLINED, "''inline'' modifier is not allowed on virtual members. Only private or final members can be inlined");
         MAP.put(NOTHING_TO_INLINE, "Expected performance impact of inlining ''{0}'' can be insignificant. Inlining works best for functions with lambda parameters", SHORT_NAMES_IN_TYPES);

compiler/frontend/src/org/jetbrains/kotlin/resolve/calls/checkers/InlineChecker.java

@@ -93,7 +93,7 @@ class InlineChecker implements CallChecker {
             }
         }
 
-        checkVisibility(targetDescriptor, expression, context);
+        checkVisibilityAndAccess(targetDescriptor, expression, context);
         checkRecursion(context, targetDescriptor, expression);
     }
 
@@ -238,11 +238,26 @@ class InlineChecker implements CallChecker {
         return isInvoke || InlineUtil.isInline(descriptor);
     }
 
-    private void checkVisibility(@NotNull CallableDescriptor declarationDescriptor, @NotNull KtElement expression, @NotNull BasicCallResolutionContext context){
+    private void checkVisibilityAndAccess(@NotNull CallableDescriptor declarationDescriptor, @NotNull KtElement expression, @NotNull BasicCallResolutionContext context){
         boolean declarationDescriptorIsPublicApi = DescriptorUtilsKt.isEffectivelyPublicApi(declarationDescriptor) || isDefinedInInlineFunction(declarationDescriptor);
         if (isEffectivelyPublicApiFunction && !declarationDescriptorIsPublicApi && declarationDescriptor.getVisibility() != Visibilities.LOCAL) {
             context.trace.report(Errors.INVISIBLE_MEMBER_FROM_INLINE.on(expression, declarationDescriptor, descriptor));
         }
+        else {
+            checkPrivateClassMemberAccess(declarationDescriptor, expression, context);
+        }
+    }
+
+    private void checkPrivateClassMemberAccess(
+            @NotNull DeclarationDescriptor declarationDescriptor,
+            @NotNull KtElement expression,
+            @NotNull BasicCallResolutionContext context
+    ) {
+        if (!Visibilities.isPrivate(descriptor.getVisibility())) {
+            if (DescriptorUtilsKt.isInsidePrivateClass(declarationDescriptor)) {
+                context.trace.report(Errors.PRIVATE_CLASS_MEMBER_FROM_INLINE.on(expression, declarationDescriptor, descriptor));
+            }
+        }
     }
 
     private boolean isDefinedInInlineFunction(@NotNull DeclarationDescriptorWithVisibility startDescriptor) {

compiler/testData/codegen/boxInline/private/nestedInPrivateClass.1.kt

@@ -0,0 +1,7 @@
+import test.*
+
+fun box(): String {
+    return call {
+        "O"
+    }
+}

compiler/testData/codegen/boxInline/private/nestedInPrivateClass.2.kt

@@ -0,0 +1,18 @@
+package test
+
+private class S public constructor() {
+    class Z {
+        fun a(): String {
+            return "K"
+        }
+    }
+}
+// This function exposes S.Z which is a class nested into a private class S (package-private in the byte code)
+// It can be accessed outside the `test` package now that S.Z. is public in the byte code, but it may be changed later
+internal inline fun call(s: () -> String): String {
+    return s() + test().a()
+}
+
+private fun test(): S.Z {
+    return S.Z()
+}

compiler/testData/codegen/boxInline/private/privateClass.1.kt

@@ -0,0 +1,7 @@
+import test.*
+
+fun box(): String {
+    return call {
+        "O"
+    }
+}

compiler/testData/codegen/boxInline/private/privateClass.2.kt

@@ -0,0 +1,23 @@
+package test
+
+private class S {
+    fun a(): String {
+        return "K"
+    }
+}
+
+// This function exposes S which is a private class (package-private in the byte code)
+// It can be accessed outside the `test` package, which may lead to IllegalAccessError.
+// This behavior may be changed later
+internal inline fun call(s: () -> String): String {
+    val s = test()
+    return s() + test2(s)
+}
+
+private fun test(): S {
+    return S()
+}
+
+private fun test2(s: S): String {
+    return s.a()
+}

compiler/testData/codegen/boxInline/private/privateClassExtensionLambda.1.kt

@@ -0,0 +1,5 @@
+import test.*
+
+fun box(): String {
+    return call()
+}

compiler/testData/codegen/boxInline/private/privateClassExtensionLambda.2.kt

@@ -0,0 +1,25 @@
+package test
+
+private class S {
+    fun a(): String {
+        return "K"
+    }
+
+    // This function exposes S which is a private class (package-private in the byte code)
+    // It can be accessed outside the `test` package, which may lead to IllegalAccessError.
+    // This behavior may be changed later
+    internal inline fun call(s: S.() -> String): String {
+        return call2(s)
+    }
+}
+
+@Suppress("PRIVATE_CLASS_MEMBER_FROM_INLINE", "EXPOSED_PARAMETER_TYPE", "EXPOSED_RECEIVER_TYPE")
+internal inline fun S.call2(s: S.() -> String): String {
+    return s() + a()
+}
+
+internal fun call(): String {
+    return S().call {
+        "O"
+    }
+}

compiler/testData/diagnostics/tests/inline/nonPublicMember/inNonPublicClass.kt

@@ -1,4 +1,4 @@
-// !DIAGNOSTICS: -UNUSED_EXPRESSION -UNUSED_PARAMETER -UNUSED_VARIABLE -NOTHING_TO_INLINE
+// !DIAGNOSTICS: -UNUSED_EXPRESSION -UNUSED_PARAMETER -UNUSED_VARIABLE -NOTHING_TO_INLINE -INVISIBLE_PRIVATE_CLASS_MEMBER_FROM_INLINE
 private class Z public constructor(){
     public val publicProperty:Int = 12
     public fun publicFun() {}
@@ -14,7 +14,7 @@ internal inline fun testInternal() {
     Z().publicFun()
 }
 
-private class Z2 {
+internal class Z2 {
     private val privateProperty = 11;
 
     public val publicProperty:Int = 12
@@ -34,7 +34,7 @@ private class Z2 {
         Z2().privateFun()
     }
 
-    inline fun testInternal() {
+    internal inline fun testInternal() {
         privateProperty
         privateFun()
         publicProperty

compiler/testData/diagnostics/tests/inline/nonPublicMember/inNonPublicClass.txt

@@ -12,7 +12,7 @@ private final class Z {
     public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
 }
 
-private final class Z2 {
+internal final class Z2 {
     public constructor Z2()
     private final val privateProperty: kotlin.Int = 11
     public final val publicProperty: kotlin.Int = 12

compiler/testData/diagnostics/tests/inline/nonPublicMember/inNonPublicInnerClass.kt

@@ -1,5 +1,5 @@
 // !DIAGNOSTICS: -UNUSED_EXPRESSION -UNUSED_PARAMETER -UNUSED_VARIABLE -NOTHING_TO_INLINE
-private class Z2 {
+internal class Z2 {
     private val privateProperty = 11;
 
     public val publicProperty:Int = 12
@@ -8,7 +8,7 @@ private class Z2 {
 
     public fun publicFun() {}
 
-    private inner class ZInner {
+    internal inner class ZInner {
         public inline fun test() {
             privateProperty
             privateFun()
@@ -21,7 +21,7 @@ private class Z2 {
             Z2().privateFun()
         }
 
-        inline fun testInternal() {
+        internal inline fun testInternal() {
             privateProperty
             privateFun()
             publicFun()

compiler/testData/diagnostics/tests/inline/nonPublicMember/inNonPublicInnerClass.txt

@@ -1,6 +1,6 @@
 package
 
-private final class Z2 {
+internal final class Z2 {
     public constructor Z2()
     private final val privateProperty: kotlin.Int = 11
     public final val publicProperty: kotlin.Int = 12
@@ -10,7 +10,7 @@ private final class Z2 {
     public final fun publicFun(): kotlin.Unit
     public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
 
-    private final inner class ZInner {
+    internal final inner class ZInner {
         public constructor ZInner()
         public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
         public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int

compiler/testData/diagnostics/tests/inline/privateClass.kt

@@ -0,0 +1,25 @@
+// !DIAGNOSTICS: -EXPOSED_PARAMETER_TYPE
+
+private class S public constructor() {
+    fun a() {
+
+    }
+}
+
+internal inline fun x(s: S, z: () -> Unit) {
+    z()
+    <!PRIVATE_CLASS_MEMBER_FROM_INLINE!>S<!>()
+    s.<!PRIVATE_CLASS_MEMBER_FROM_INLINE!>a<!>()
+    test()
+}
+
+private inline fun x2(s: S, z: () -> Unit) {
+    z()
+    S()
+    s.a()
+    test()
+}
+
+private fun test(): S {
+    return S()
+}

compiler/testData/diagnostics/tests/inline/privateClass.txt

@@ -0,0 +1,12 @@
+package
+
+private fun test(): S
+private inline fun x(/*0*/ s: S, /*1*/ z: () -> kotlin.Unit): kotlin.Unit
+
+private final class S {
+    public constructor S()
+    public final fun a(): kotlin.Unit
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}

compiler/tests/org/jetbrains/kotlin/checkers/DiagnosticsTestGenerated.java

@@ -9225,6 +9225,12 @@ public class DiagnosticsTestGenerated extends AbstractDiagnosticsTest {
                 doTest(fileName);
             }
 
+            @TestMetadata("privateClass.kt")
+            public void testPrivateClass() throws Exception {
+                String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/inline/privateClass.kt");
+                doTest(fileName);
+            }
+
             @TestMetadata("propagation.kt")
             public void testPropagation() throws Exception {
                 String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/inline/propagation.kt");

compiler/tests/org/jetbrains/kotlin/codegen/generated/BlackBoxInlineCodegenTestGenerated.java

@@ -1348,6 +1348,24 @@ public class BlackBoxInlineCodegenTestGenerated extends AbstractBlackBoxInlineCo
             doTestMultiFileWithInlineCheck(fileName);
         }
 
+        @TestMetadata("nestedInPrivateClass.1.kt")
+        public void testNestedInPrivateClass() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/boxInline/private/nestedInPrivateClass.1.kt");
+            doTestMultiFileWithInlineCheck(fileName);
+        }
+
+        @TestMetadata("privateClass.1.kt")
+        public void testPrivateClass() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/boxInline/private/privateClass.1.kt");
+            doTestMultiFileWithInlineCheck(fileName);
+        }
+
+        @TestMetadata("privateClassExtensionLambda.1.kt")
+        public void testPrivateClassExtensionLambda() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/boxInline/private/privateClassExtensionLambda.1.kt");
+            doTestMultiFileWithInlineCheck(fileName);
+        }
+
         @TestMetadata("privateInInlineInMultiFileFacade.1.kt")
         public void testPrivateInInlineInMultiFileFacade() throws Exception {
             String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/boxInline/private/privateInInlineInMultiFileFacade.1.kt");

compiler/tests/org/jetbrains/kotlin/jvm/compiler/CompileKotlinAgainstInlineKotlinTestGenerated.java

@@ -1348,6 +1348,24 @@ public class CompileKotlinAgainstInlineKotlinTestGenerated extends AbstractCompi
             doBoxTestWithInlineCheck(fileName);
         }
 
+        @TestMetadata("nestedInPrivateClass.1.kt")
+        public void testNestedInPrivateClass() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/boxInline/private/nestedInPrivateClass.1.kt");
+            doBoxTestWithInlineCheck(fileName);
+        }
+
+        @TestMetadata("privateClass.1.kt")
+        public void testPrivateClass() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/boxInline/private/privateClass.1.kt");
+            doBoxTestWithInlineCheck(fileName);
+        }
+
+        @TestMetadata("privateClassExtensionLambda.1.kt")
+        public void testPrivateClassExtensionLambda() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/boxInline/private/privateClassExtensionLambda.1.kt");
+            doBoxTestWithInlineCheck(fileName);
+        }
+
         @TestMetadata("privateInInlineInMultiFileFacade.1.kt")
         public void testPrivateInInlineInMultiFileFacade() throws Exception {
             String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/boxInline/private/privateInInlineInMultiFileFacade.1.kt");

core/descriptors/src/org/jetbrains/kotlin/resolve/DescriptorUtils.kt

@@ -106,6 +106,13 @@ val DeclarationDescriptorWithVisibility.isEffectivelyPublicApi: Boolean
         return true
     }
 
+val DeclarationDescriptor.isInsidePrivateClass: Boolean
+    get() {
+        var parent = containingDeclaration as? ClassDescriptor
+        return parent != null && Visibilities.isPrivate(parent.visibility)
+    }
+
+
 fun ClassDescriptor.getSuperClassNotAny(): ClassDescriptor? {
     for (supertype in defaultType.constructor.supertypes) {
         if (!KotlinBuiltIns.isAnyOrNullableAny(supertype)) {