Native: support Objective-C instancetype in more cases in cinterop

Objective-C has a special type, "instancetype". Generally, it is a type
that matches the method receiver type. So, if `Foo.foo` method returns
`instancetype`, then `Foo.foo()` would be of type `Foo`, while
`Bar.foo()` would be of type `Bar` (where `Bar` is a subclass of `Foo`).

Surprisingly, `instancetype` can be used not only as a return type, but
also somewhere inside a return type. cinterop wasn't ready for this.

This commit expands implementation of `instancetype` in cinterop to
cover more cases.

^KT-59597 Fixed

kotlin-native/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/NativeIndex.kt

@@ -260,18 +260,41 @@ data class ObjCMethod(
         val isOptional: Boolean, val isInit: Boolean, val isExplicitlyDesignatedInitializer: Boolean, val isDirect: Boolean
 ) {
 
-    fun returnsInstancetype(): Boolean = returnType is ObjCInstanceType
+    fun containsInstancetype(): Boolean = returnType.containsInstancetype() // Clang doesn't allow parameter types to use instancetype.
 
-    fun getReturnType(container: ObjCClassOrProtocol): Type = if (returnType is ObjCInstanceType) {
-        when (container) {
-            is ObjCClass -> ObjCObjectPointer(container, returnType.nullability, protocols = emptyList())
-            is ObjCProtocol -> ObjCIdType(returnType.nullability, protocols = listOf(container))
-        }
+    fun getReturnType(container: ObjCClassOrProtocol): Type = if (returnType.containsInstancetype()) {
+        returnType.substituteInstancetype(container)
     } else {
+        // Fast path, avoid allocating copies.
         returnType
     }
 }
 
+// Clang seems to allow using instancetype only inside certain kinds of types.
+// The implementation below therefore covers only particular cases, based on the experiments with Clang and common sense.
+private fun Type.containsInstancetype(): Boolean = when (this) {
+    is ObjCInstanceType -> true
+
+    is ObjCBlockPointer -> this.returnType.containsInstancetype()
+    is FunctionType -> this.returnType.containsInstancetype()
+    is PointerType -> this.pointeeType.containsInstancetype()
+
+    else -> false
+}
+
+private fun Type.substituteInstancetype(container: ObjCClassOrProtocol): Type = when (this) {
+    is ObjCInstanceType -> when (container) {
+        is ObjCClass -> ObjCObjectPointer(container, this.nullability, protocols = emptyList())
+        is ObjCProtocol -> ObjCIdType(this.nullability, protocols = listOf(container))
+    }
+
+    is ObjCBlockPointer -> this.copy(returnType = this.returnType.substituteInstancetype(container))
+    is FunctionType -> this.copy(returnType = this.returnType.substituteInstancetype(container))
+    is PointerType -> this.copy(pointeeType = this.pointeeType.substituteInstancetype(container))
+
+    else -> this
+}
+
 data class ObjCProperty(val name: String, val getter: ObjCMethod, val setter: ObjCMethod?) {
     fun getType(container: ObjCClassOrProtocol): Type = getter.getReturnType(container)
 }

kotlin-native/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/ObjCStubs.kt

@@ -388,7 +388,7 @@ internal abstract class ObjCContainerStubBuilder(
         methods -= superMethods
 
         // Add some special methods from super types:
-        methods += superMethods.filter { it.returnsInstancetype() || it.isInit }
+        methods += superMethods.filter { it.containsInstancetype() || it.isInit }
 
         // Add methods from adopted protocols that must be implemented according to Kotlin rules:
         if (container is ObjCClass) {

native/native.tests/testData/CInterop/executable/objcInstancetype/objcInstancetype.def

@@ -0,0 +1,2 @@
+language = Objective-C
+headers = objcInstancetype.h

native/native.tests/testData/CInterop/executable/objcInstancetype/objcInstancetype.h

@@ -0,0 +1,30 @@
+#import "Foundation/NSString.h"
+#import "Foundation/NSObject.h"
+
+@interface Foo : NSObject
+- (instancetype _Nonnull)atReturnType;
+
+// https://youtrack.jetbrains.com/issue/KT-59597/KN-Usage-of-instancetype-in-block-return-type-crashes
++ (instancetype (^ _Nonnull)(void))atBlockReturnType;
+
+- (instancetype (*)(void))atFunctionReturnType;
+- (__strong instancetype*)atPointerType;
+- (__strong instancetype** (^(^ _Nonnull)(void))(void))atComplexType;
+
+//All these declarations are prohibited by Clang:
+//@property (class) instancetype classProperty;
+//@property instancetype instanceProperty;
+//- (void (^)(instancetype))atBlockParameterType;
+//- (void (*)(instancetype))atFunctionParameterType;
+//- (struct { instancetype f; })atStructField;
+//- (instancetype[])atArrayElementType;
+//- (instancetype[42])atConstArrayElementType;
+@end
+
+@protocol Bar
+- (instancetype) protocolMethod;
++ (instancetype _Nonnull) protocolClassMethod;
+@end
+
+@interface Baz : Foo <Bar>
+@end

native/native.tests/testData/CInterop/executable/objcInstancetype/objcInstancetype.kt

@@ -0,0 +1,68 @@
+@file:OptIn(kotlinx.cinterop.ExperimentalForeignApi::class, kotlin.experimental.ExperimentalObjCName::class)
+
+import objcInstancetype.*
+import kotlin.test.*
+import kotlinx.cinterop.*
+
+fun testFoo() {
+    val foo = Foo()
+
+    val atReturnType: Foo = foo.atReturnType()
+    assertEquals(foo, atReturnType)
+
+    val atBlockReturnType: () -> Foo? = Foo.atBlockReturnType()
+    assertNotNull(atBlockReturnType())
+
+    val atFunctionReturnType: CPointer<CFunction<() -> Foo?>>? = foo.atFunctionReturnType()
+    assertNull(atFunctionReturnType!!())
+
+    val atPointerType: CPointer<ObjCObjectVar<Foo?>>? = foo.atPointerType()
+    assertNotNull(atPointerType)
+    assertNull(atPointerType.pointed.value)
+
+    val atComplexType: () -> (() -> CPointer<CPointerVar<ObjCObjectVar<Foo?>>>?)? = foo.atComplexType()
+    assertNull(atComplexType!!())
+}
+
+fun testBar() {
+    val bar: BarProtocol = Baz()
+    val barMeta: BarProtocolMeta = Baz
+
+    val protocolMethod: BarProtocol? = bar.protocolMethod()
+    assertEquals(bar, protocolMethod)
+
+    val protocolClassMethod: BarProtocol = barMeta.protocolClassMethod()
+    assertNotNull(protocolClassMethod)
+}
+
+fun testBaz() {
+    val baz = Baz()
+
+    val atReturnType: Baz = baz.atReturnType()
+    assertEquals(baz, atReturnType)
+
+    val atBlockReturnType: () -> Baz? = Baz.atBlockReturnType()
+    assertNotNull(atBlockReturnType())
+
+    val atFunctionReturnType: CPointer<CFunction<() -> Baz?>>? = baz.atFunctionReturnType()
+    assertNull(atFunctionReturnType!!())
+
+    val atPointerType: CPointer<ObjCObjectVar<Baz?>>? = baz.atPointerType()
+    assertNotNull(atPointerType)
+    assertNull(atPointerType.pointed.value)
+
+    val atComplexType: () -> (() -> CPointer<CPointerVar<ObjCObjectVar<Baz?>>>?)? = baz.atComplexType()
+    assertNull(atComplexType!!())
+
+    val protocolMethod: Baz? = baz.protocolMethod()
+    assertEquals(baz, protocolMethod)
+
+    val protocolClassMethod: Baz = Baz.protocolClassMethod()
+    assertNotNull(protocolClassMethod)
+}
+
+fun main() {
+    testFoo()
+    testBar()
+    testBaz()
+}

native/native.tests/testData/CInterop/executable/objcInstancetype/objcInstancetype.m

@@ -0,0 +1,39 @@
+#import "objcInstancetype.h"
+
+static id returnNil() {
+    return nil;
+}
+
+static Foo* global = nil;
+
+@implementation Foo
+- (instancetype _Nonnull)atReturnType {
+    return self;
+}
+
++ (instancetype (^ _Nonnull)(void))atBlockReturnType {
+    return ^(){ return [self new]; };
+}
+
+- (instancetype (*)(void))atFunctionReturnType {
+    return &returnNil;
+}
+
+- (__strong instancetype*)atPointerType {
+    return &global;
+}
+
+- (__strong Foo** (^(^ _Nonnull)(void))(void))atComplexType {
+    return (__strong Foo** (^(^ _Nonnull)(void))(void))^(){ return nil; };
+}
+@end
+
+@implementation Baz
+- (instancetype) protocolMethod {
+    return self;
+}
+
++ (instancetype _Nonnull) protocolClassMethod {
+    return [Baz new];
+}
+@end

native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/ClassicNativeCInteropExecutableTest.java

@@ -114,6 +114,12 @@ public class ClassicNativeCInteropExecutableTest extends AbstractNativeCInteropE
         runTest("native/native.tests/testData/CInterop/executable/leakMemoryWithRunningThreadUnchecked/");
     }
 
+    @Test
+    @TestMetadata("objcInstancetype")
+    public void testObjcInstancetype() throws Exception {
+        runTest("native/native.tests/testData/CInterop/executable/objcInstancetype/");
+    }
+
     @Test
     @TestMetadata("toKString")
     public void testToKString() throws Exception {

native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/FirNativeCInteropExecutableTest.java

@@ -118,6 +118,12 @@ public class FirNativeCInteropExecutableTest extends AbstractNativeCInteropExecu
         runTest("native/native.tests/testData/CInterop/executable/leakMemoryWithRunningThreadUnchecked/");
     }
 
+    @Test
+    @TestMetadata("objcInstancetype")
+    public void testObjcInstancetype() throws Exception {
+        runTest("native/native.tests/testData/CInterop/executable/objcInstancetype/");
+    }
+
     @Test
     @TestMetadata("toKString")
     public void testToKString() throws Exception {