Interop: implement new indexer in Kotlin

and slightly improve StubGenerator code

experiments/Interop/.idea/modules.xml

@@ -3,10 +3,10 @@
   <component name="ProjectModuleManager">
     <modules>
       <module fileurl="file://$PROJECT_DIR$/Example/Example.iml" filepath="$PROJECT_DIR$/Example/Example.iml" />
+      <module fileurl="file://$PROJECT_DIR$/Indexer/Indexer.iml" filepath="$PROJECT_DIR$/Indexer/Indexer.iml" />
       <module fileurl="file://$PROJECT_DIR$/Interop.iml" filepath="$PROJECT_DIR$/Interop.iml" />
       <module fileurl="file://$PROJECT_DIR$/Runtime/Runtime.iml" filepath="$PROJECT_DIR$/Runtime/Runtime.iml" />
       <module fileurl="file://$PROJECT_DIR$/StubGenerator/StubGenerator.iml" filepath="$PROJECT_DIR$/StubGenerator/StubGenerator.iml" />
-      <module fileurl="file://$PROJECT_DIR$/../kni/indexer/indexer.iml" filepath="$PROJECT_DIR$/../kni/indexer/indexer.iml" />
     </modules>
   </component>
 </project>

experiments/Interop/.idea/runConfigurations/Gen_stubs_for_indexer.xml

@@ -0,0 +1,14 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Gen stubs for indexer" type="JetRunConfigurationType" factoryName="Kotlin">
+    <option name="MAIN_CLASS_NAME" value="org.jetbrains.kotlin.native.interop.gen.jvm.MainKt" />
+    <option name="VM_PARAMETERS" value="-Djava.library.path=$PROJECT_DIR$/Indexer/nativelib" />
+    <option name="PROGRAM_PARAMETERS" value="$PROJECT_DIR$/Indexer/src $PROJECT_DIR$/Indexer/nativelib" />
+    <option name="WORKING_DIRECTORY" value="" />
+    <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
+    <option name="ALTERNATIVE_JRE_PATH" />
+    <option name="PASS_PARENT_ENVS" value="true" />
+    <module name="StubGenerator" />
+    <envs />
+    <method />
+  </configuration>
+</component>

experiments/Interop/.idea/runConfigurations/StubGenerator.xml

@@ -1,7 +1,7 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="StubGenerator" type="JetRunConfigurationType" factoryName="Kotlin">
     <option name="MAIN_CLASS_NAME" value="org.jetbrains.kotlin.native.interop.gen.jvm.MainKt" />
-    <option name="VM_PARAMETERS" value="-Djava.library.path=$PROJECT_DIR$/../kni/dist/build/indexer/native" />
+    <option name="VM_PARAMETERS" value="-Djava.library.path=$PROJECT_DIR$/Indexer/nativelib" />
     <option name="PROGRAM_PARAMETERS" value="$PROJECT_DIR$/Example/src $PROJECT_DIR$/Example/nativelib" />
     <option name="WORKING_DIRECTORY" value="" />
     <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />

experiments/Interop/Indexer/Indexer.iml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="KotlinJavaRuntime" level="project" />
+    <orderEntry type="module" module-name="Runtime" />
+  </component>
+</module>

experiments/Interop/Indexer/native-src/build-callback.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+clang -shared -o../nativelib/libcallback.dylib callback.c -I /System/Library/Frameworks/JavaVM.framework/Headers -pthread
+

experiments/Interop/Indexer/native-src/callback.c

@@ -0,0 +1,51 @@
+#include <pthread.h>
+#include <jni.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+
+static JavaVM *vm = NULL;
+
+static JNIEnv* getCurrentEnv() {
+//    printf("%p\n", pthread_self());
+    JNIEnv* env;
+    assert (vm != NULL);
+    jint res = (*vm)->GetEnv(vm, (void**)&env, JNI_VERSION_1_1);
+    if (res != JNI_OK) {
+	assert(res == JNI_EDETACHED);
+	res = (*vm)->AttachCurrentThreadAsDaemon(vm, (void**)&env, NULL);
+	assert (res == JNI_OK);
+    }
+    return env;
+}
+
+jint JNI_OnLoad(JavaVM *vm_, void *reserved) {
+    vm = vm_;
+    getCurrentEnv();
+    return JNI_VERSION_1_1;
+}
+
+
+void indexDeclarationNativeCallback(void* arg1, void* arg2) {
+    JNIEnv* env = getCurrentEnv();
+
+    static jclass cls = NULL;
+    static jmethodID entryFromNative = NULL;
+
+    if (cls == NULL) {
+        cls = (*env)->FindClass(env, "org/jetbrains/kotlin/native/interop/indexer/indexDeclarationCallback");
+	assert(cls != NULL);
+	entryFromNative = (*env)->GetStaticMethodID(env, cls, "entryFromNative", "(J)V");
+	assert(entryFromNative != NULL);
+    }
+    
+    (*env)->CallStaticVoidMethod(env, cls, entryFromNative, (jlong) arg2);
+    if ((*env)->ExceptionCheck(env)) {
+	(*env)->ExceptionDescribe(env);
+	assert(0);
+    }
+}
+
+JNIEXPORT jlong JNICALL Java_org_jetbrains_kotlin_native_interop_indexer_indexDeclarationCallback_nativeCallbackPtr(JNIEnv *env, jobject obj) {
+    return (jlong) indexDeclarationNativeCallback;
+}

experiments/Interop/Indexer/src/clang/Clang.def

@@ -0,0 +1,14 @@
+libName = clangbridge
+
+headers = clang-c/Index.h
+
+compiler = cc
+
+compilerOpts = -std=c99 -I/usr/local/Cellar/llvm/3.8.1/include  -fPIC
+
+linker = cc
+
+linkerOpts = -fPIC \
+    -L/usr/local/Cellar/llvm/3.8.1/lib -lclang
+
+excludedFunctions =

experiments/Interop/Indexer/src/org/jetbrains/kotlin/native/interop/indexer/Callbacks.kt

@@ -0,0 +1,34 @@
+package org.jetbrains.kotlin.native.interop.indexer
+
+import clang.CXIdxDeclInfo
+import kotlin_native.interop.NativePtr
+
+internal object indexDeclarationCallback {
+
+    private var function: ((CXIdxDeclInfo)->Unit)? = null
+
+    fun setUp(func: (CXIdxDeclInfo)->Unit): NativePtr {
+        assert (function == null)
+        function = func
+        return nativeCallbackPtr
+    }
+
+    fun reset() {
+        assert (function != null)
+        function = null
+    }
+
+    @JvmStatic
+    private fun entryFromNative(info: Long) {
+        function!!(CXIdxDeclInfo(NativePtr.Companion.byValue(info)!!))
+    }
+
+    private val nativeCallbackPtr: NativePtr
+
+    init {
+        System.loadLibrary("callback")
+        nativeCallbackPtr = NativePtr.byValue(nativeCallbackPtr())!!
+    }
+
+    private external fun nativeCallbackPtr(): Long
+}

experiments/Interop/Indexer/src/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt

@@ -0,0 +1,262 @@
+package org.jetbrains.kotlin.native.interop.indexer
+
+import clang.*
+import clang.CXIdxEntityKind.*
+import clang.CXTypeKind.*
+import kotlin_native.interop.*
+import java.io.File
+
+private class StructDeclImpl(spelling: String) : StructDecl(spelling) {
+    override var def: StructDefImpl? = null
+}
+
+private class StructDefImpl(size: Long, decl: StructDecl) : StructDef(size, decl) {
+    override val fields = mutableListOf<Field>()
+}
+
+private class EnumDefImpl(spelling: String, type: PrimitiveType) : EnumDef(spelling, type) {
+    override val values = mutableListOf<EnumValue>()
+}
+
+private class NativeIndexImpl : NativeIndex() {
+
+    val arena = Arena()
+
+    fun clearNativeMem() = arena.clear()
+
+    private data class DeclarationID(val usr: String)
+
+    val structById = mutableMapOf<DeclarationID, StructDeclImpl>()
+
+    override val structs: List<StructDecl>
+        get() = structById.values.toList()
+
+    val enumById = mutableMapOf<DeclarationID, EnumDefImpl>()
+
+    override val enums: List<EnumDef>
+        get() = enumById.values.toList()
+
+    val functionByName = mutableMapOf<String, FunctionDecl>()
+
+    override val functions: List<FunctionDecl>
+        get() = functionByName.values.toList()
+
+    fun getDeclarationId(cursor: CXCursor) = DeclarationID(clang_getCursorUSR(cursor, arena).convertAndDispose())
+
+    fun getStructTypeDecl(type: CXType): StructDeclImpl {
+        assert (type.kind.value == CXType_Record)
+        return getStructDeclAt(clang_getTypeDeclaration(type, arena))
+    }
+
+    fun getStructDeclAt(cursor: CXCursor): StructDeclImpl {
+        val declId = getDeclarationId(cursor)
+
+        val decl = structById[declId]
+        if (decl != null) {
+            return decl
+        }
+
+        val cursorType = clang_getCursorType(cursor, arena)
+        val typeSpelling = clang_getTypeSpelling(cursorType, arena).convertAndDispose()
+
+        val res = StructDeclImpl(typeSpelling)
+        structById[declId] = res
+        return res
+    }
+
+    fun getEnumTypeDef(type: CXType): EnumDefImpl {
+        assert (type.kind.value == CXType_Enum)
+        val declCursor = clang_getTypeDeclaration(type, arena)
+        return getEnumDefAt(declCursor)
+    }
+
+    fun getEnumDefAt(cursor: CXCursor): EnumDefImpl {
+        if (clang_isCursorDefinition(cursor) == 0) {
+            TODO("support enum forward declarations")
+        }
+
+        val declId = getDeclarationId(cursor)
+
+        val enum = enumById[declId]
+        if (enum != null) {
+            return enum
+        }
+
+        val cursorType = clang_getCursorType(cursor, arena)
+        val typeSpelling = clang_getTypeSpelling(cursorType, arena).convertAndDispose()
+
+        val baseType = convertType(clang_getEnumDeclIntegerType(cursor, arena)) as PrimitiveType
+
+        val res = EnumDefImpl(typeSpelling, baseType)
+        enumById[declId] = res
+        return res
+    }
+
+    fun convertType(type: CXType): Type {
+        val kind = type.kind.value
+        return when (kind) {
+            CXType_Unexposed -> {
+                val canonicalType = clang_getCanonicalType(type, arena)
+                if (canonicalType.kind.value != CXType_Unexposed) {
+                    convertType(canonicalType)
+                } else {
+                    throw NotImplementedError()
+                }
+            }
+
+            CXType_Void -> VoidType
+
+            CXType_Char_U, CXType_UChar -> UInt8Type
+            CXType_Char_S, CXType_SChar -> Int8Type
+
+            CXType_UShort -> UInt16Type
+            CXType_Short -> Int16Type
+
+            CXType_UInt -> UInt32Type
+            CXType_Int -> Int32Type
+
+            CXType_ULong -> UIntPtrType
+            CXType_Long -> IntPtrType
+
+            CXType_ULongLong -> UInt64Type
+            CXType_LongLong -> Int64Type
+
+            CXType_Typedef -> {
+                val declaration = clang_getTypeDeclaration(type, arena)
+                val underlying = clang_getTypedefDeclUnderlyingType(declaration, arena)
+                assert (underlying.kind.value != CXType_Invalid)
+                convertType(underlying)
+            }
+
+            CXType_Record -> RecordType(getStructTypeDecl(type))
+            CXType_Enum -> EnumType(getEnumTypeDef(type))
+
+            CXType_Pointer -> PointerType(convertType(clang_getPointeeType(type, arena)))
+
+            CXType_ConstantArray -> {
+                val elemType = convertType(clang_getArrayElementType(type, arena))
+                val length = clang_getArraySize(type)
+                ConstArrayType(elemType, length)
+            }
+
+            CXType_IncompleteArray -> {
+                val elemType = convertType(clang_getArrayElementType(type, arena))
+                IncompleteArrayType(elemType)
+            }
+
+            CXType_FunctionProto -> {
+                if (clang_isFunctionTypeVariadic(type) != 0) {
+                    UnsupportedType
+                } else {
+                    val returnType = convertType(clang_getResultType(type, arena))
+                    val numArgs = clang_getNumArgTypes(type)
+                    val paramTypes = (0..numArgs - 1).map {
+                        convertType(clang_getArgType(type, it, arena))
+                    }
+                    FunctionType(paramTypes, returnType)
+                }
+            }
+
+            else -> UnsupportedType
+        }
+    }
+
+    fun indexDeclaration(info: CXIdxDeclInfo) {
+        val cursor = info.cursor
+        val entityInfo = info.entityInfo.value!!
+        val entityName = entityInfo.name.value?.asCString()?.toString()
+        val kind = entityInfo.kind.value
+
+        when (kind) {
+            CXIdxEntity_Field -> {
+                val name = entityName!!
+                val type = convertType(clang_getCursorType(cursor, arena))
+                val offset = clang_Cursor_getOffsetOfField(cursor)
+
+                val container = info.semanticContainer.value!!
+                val structDef = getStructDeclAt(container.cursor).def!!
+                structDef.fields.add(Field(name, type, offset))
+            }
+
+            CXIdxEntity_Struct, CXIdxEntity_Union -> {
+                val structDecl = getStructDeclAt(cursor)
+                if (clang_isCursorDefinition(cursor) != 0) {
+                    val size = clang_Type_getSizeOf(clang_getCursorType(cursor, arena))
+                    structDecl.def = StructDefImpl(size, structDecl)
+                }
+            }
+
+            CXIdxEntity_Function -> {
+                val name = entityName!!
+                val returnType = convertType(clang_getCursorResultType(cursor, arena))
+                val argNum = clang_Cursor_getNumArguments(cursor)
+                val args = (0 .. argNum - 1).map {
+                    val argCursor = clang_Cursor_getArgument(cursor, it, arena)
+                    val argName = clang_getCursorSpelling(argCursor, arena).convertAndDispose()
+                    val type = convertType(clang_getCursorType(argCursor, arena))
+                    Parameter(argName, type)
+                }
+
+                functionByName[name] = FunctionDecl(name, args, returnType)
+            }
+
+            CXIdxEntity_Enum -> {
+                getEnumDefAt(cursor)
+            }
+
+            CXIdxEntity_EnumConstant -> {
+                val container = info.semanticContainer.value!!
+                val name = entityName!!
+                val value = clang_getEnumConstantDeclValue(info.cursor)
+
+                getEnumDefAt(container.cursor).values.add(EnumValue(name, value))
+            }
+        }
+    }
+
+}
+
+fun CXString.convertAndDispose(): String {
+    try {
+        return clang_getCString(this)!!.asCString().toString()
+    } finally {
+        clang_disposeString(this)
+    }
+}
+
+fun buildNativeIndexImpl(headerFile: File, args: List<String>): NativeIndex {
+    val args1 = args.map { CString.fromString(it)!!.asCharPtr() }.toTypedArray()
+
+    val index = clang_createIndex(0, 0)
+    val indexAction = clang_IndexAction_create(index)
+    val callbacks = malloc(IndexerCallbacks.Companion)
+
+    val res = NativeIndexImpl()
+    try {
+    val indexDeclarationNativeCallback = indexDeclarationCallback.setUp({ res.indexDeclaration(it) })
+
+        try {
+            with(callbacks) {
+                abortQuery.value = null
+                diagnostic.value = null
+                enteredMainFile.value = null
+                ppIncludedFile.value = null
+                importedASTFile.value = null
+                startedTranslationUnit.value = null
+                indexDeclaration.value = indexDeclarationNativeCallback
+                indexEntityReference.value = null
+            }
+
+            clang_indexSourceFile(indexAction, null, callbacks, IndexerCallbacks.Companion.size, 0, headerFile.path,
+                    mallocNativeArrayOf(Int8Box.Companion, *args1)[0], args1.size, null, 0, null, 0)
+
+
+            return res
+
+        } finally {
+            indexDeclarationCallback.reset()
+        }
+    } finally {
+        res.clearNativeMem()
+    }
+}

experiments/Interop/Indexer/src/org/jetbrains/kotlin/native/interop/indexer/NativeIndex.kt

@@ -0,0 +1,74 @@
+package org.jetbrains.kotlin.native.interop.indexer
+
+import java.io.File
+
+fun buildNativeIndex(headerFile: File, args: List<String>): NativeIndex = buildNativeIndexImpl(headerFile, args)
+
+abstract class NativeIndex {
+    abstract val structs: List<StructDecl>
+    abstract val enums: List<EnumDef>
+    abstract val functions: List<FunctionDecl>
+}
+
+
+class Field(val name: String, val type: Type, val offset: Long) {
+    override fun toString(): String = "$type $name at $offset"
+}
+
+abstract class StructDecl(val spelling: String) {
+
+    abstract val def: StructDef?
+
+}
+
+abstract class StructDef(val size: Long, val decl: StructDecl) {
+    abstract val fields: List<Field>
+}
+
+class EnumValue(val name: String, val value: Long)
+
+abstract class EnumDef(val spelling: String, val baseType: PrimitiveType) {
+    abstract val values: List<EnumValue>
+}
+
+class Parameter(val name: String?, val type: Type)
+
+class FunctionDecl(val name: String, val parameters: List<Parameter>, val returnType: Type)
+
+
+
+open class Type
+
+open class PrimitiveType : Type()
+
+object VoidType : Type()
+
+object Int8Type : PrimitiveType()
+object UInt8Type : PrimitiveType()
+
+object Int16Type : PrimitiveType()
+object UInt16Type : PrimitiveType()
+
+object Int32Type : PrimitiveType()
+object UInt32Type : PrimitiveType()
+
+object IntPtrType : PrimitiveType()
+object UIntPtrType : PrimitiveType()
+
+object Int64Type : PrimitiveType()
+object UInt64Type : PrimitiveType()
+
+class RecordType(val decl: StructDecl) : Type()
+
+class EnumType(val def: EnumDef) : Type()
+
+class PointerType(val pointeeType : Type) : Type()
+
+class FunctionType(val parameterTypes: List<Type>, val returnType: Type) : Type()
+
+open class ArrayType(val elemType: Type) : Type()
+class ConstArrayType(elemType: Type, val length: Long) : ArrayType(elemType)
+class IncompleteArrayType(elemType: Type) : ArrayType(elemType)
+
+object UnsupportedType : Type()
+

experiments/Interop/StubGenerator/StubGenerator.iml

@@ -8,6 +8,6 @@
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
     <orderEntry type="library" name="KotlinJavaRuntime" level="project" />
-    <orderEntry type="module" module-name="indexer" />
+    <orderEntry type="module" module-name="Indexer" />
   </component>
 </module>

experiments/Interop/StubGenerator/src/org/jetbrains/kotlin/native/interop/gen/Types.kt

@@ -1,124 +0,0 @@
-package org.jetbrains.kotlin.native.interop.gen
-
-open class CType
-open class DirectlyMappedType(val kotlinType: String) : CType()
-object VoidType : CType()
-object Int8Type : DirectlyMappedType("Byte")
-object UInt8Type : DirectlyMappedType("Byte")
-object Int16Type : DirectlyMappedType("Short")
-object UInt16Type : DirectlyMappedType("Short")
-object Int32Type : DirectlyMappedType("Int")
-object UInt32Type : DirectlyMappedType("Int")
-object Int64Type : DirectlyMappedType("Long")
-object UInt64Type : DirectlyMappedType("Long")
-class RecordType(val name: String) : CType()
-class EnumType(val name: String) : CType()
-class PointerType(val pointeeType : CType) : CType()
-class FunctionPointerType(val parameterTypes: List<CType>, val returnType: CType) : CType()
-open class ArrayType(val elemType: CType) : CType()
-class ConstArrayType(elemType: CType, val length: Int) : ArrayType(elemType)
-class IncompleteArrayType(elemType: CType) : ArrayType(elemType)
-
-
-fun parseType(type: String): CType = TypeParser(type).parse()
-
-class TypeParser(private val type: String) {
-    companion object {
-        private val primitiveType = mapOf(
-                "V"  to VoidType,
-                "C"  to Int8Type,
-                "UC" to UInt8Type,
-                "UB" to UInt8Type,
-                "S"  to Int16Type,
-                "US" to UInt16Type,
-                "I"  to Int32Type,
-                "UI" to UInt32Type,
-                "J"  to Int64Type,
-                "UJ" to UInt64Type
-        )
-    }
-
-    private var at = 0
-
-    private fun at(s: String): Boolean = type.substring(at).startsWith(s)
-
-    private fun expect(s: String) {
-        if (!advance(s)) error("Expecting <$s> (at=$at)")
-    }
-
-    private fun advance(s: String): Boolean {
-        if (at(s)) {
-            at += s.length
-            return true
-        }
-        return false
-    }
-
-    private fun error(s: String): Nothing = throw IllegalStateException(s + ": " + type)
-
-    fun parse(): CType {
-        if (at == type.length) error("No type to parse")
-
-        @Suppress("UNUSED_VARIABLE")
-        val isConst = advance("c")
-
-        for ((string, type) in primitiveType.entries) {
-            if (advance(string)) return type
-        }
-
-        if (advance("R")) {
-            val semicolon = type.indexOf(';', at)
-            if (semicolon < 0) error("L without a matching semicolon")
-            val recordName = type.substring(at, semicolon)
-            expect(recordName)
-            expect(";")
-            return RecordType(recordName)
-        }
-
-        if (advance("E")) { // TODO: copy-pasted!
-            val semicolon = type.indexOf(';', at)
-            if (semicolon < 0) error("E without a matching semicolon")
-            val enumName = type.substring(at, semicolon)
-            expect(enumName)
-            expect(";")
-            return EnumType(enumName)
-        }
-
-        if (advance("*(")) {
-            val paramTypes = mutableListOf<CType>()
-            while (!advance(")")) {
-                if (advance(".")) {
-                    // TODO: support vararg
-                    continue
-                }
-                paramTypes.add(parse())
-            }
-            val returnType = parse()
-            expect(";")
-            return FunctionPointerType(paramTypes, returnType)
-        }
-
-        if (advance("*")) {
-            val pointee = parse()
-            expect(";")
-            return PointerType(pointee)
-        }
-
-        if (advance("[:")) {
-            val lengthEndIndex = type.indexOf(':', at)
-            val length = Integer.parseInt(type.substring(at, lengthEndIndex))
-            at = lengthEndIndex + 1
-            val elemType = parse()
-            expect(";")
-            return ConstArrayType(elemType, length)
-        }
-
-        if (advance("[")) {
-            val elemType = parse()
-            expect(";")
-            return IncompleteArrayType(elemType)
-        }
-
-        throw NotImplementedError("Unsupported type (at=$at): $type")
-    }
-}

experiments/Interop/StubGenerator/src/org/jetbrains/kotlin/native/interop/gen/jvm/StubGenerator.kt

@@ -1,38 +1,46 @@
 package org.jetbrains.kotlin.native.interop.gen.jvm
 
-import org.jetbrains.kni.indexer.NativeIndex
+import org.jetbrains.kotlin.native.interop.indexer.*
-import org.jetbrains.kotlin.native.interop.gen.*
 
 class StubGenerator(
-        val translationUnit: NativeIndex.TranslationUnit,
+        val nativeIndex: NativeIndex,
         val pkgName: String,
         val libName: String,
         val excludedFunctions: Set<String>) {
 
     val forbiddenStructNames = run {
-        val functionNames = translationUnit.functionList.map { it.name }
+        val functionNames = nativeIndex.functions.map { it.name }
-        val fieldNames = translationUnit.structList.flatMap { it.fieldList }.map { it.name }
+        val fieldNames = nativeIndex.structs.mapNotNull { it.def }.flatMap { it.fields }.map { it.name }
         (functionNames + fieldNames).toSet()
     }
 
-    val enums = translationUnit.enumList.map { it.name to it }.toMap()
+    val StructDecl.kotlinName: String
+        get() = if (spelling.startsWith("struct ") || spelling.startsWith("union ")) {
+            spelling.substringAfter(' ')
+        } else {
+            spelling
+        }
 
-    val structs = translationUnit.structList.map { it.name to it }.toMap()
+    val EnumDef.kotlinName: String
+        get() = if (spelling.startsWith("enum ")) {
+            spelling.substringAfter(' ')
+        } else {
+            spelling
+        }
+
+    val RecordType.kotlinName: String
+        get() = decl.kotlinName
+
+    val EnumType.kotlinName: String
+        get() = def.kotlinName
 
 
-    val functionsToBind = translationUnit.functionList.uniqueBy { it.name }.filter { it.name !in excludedFunctions }
+    val functionsToBind = nativeIndex.functions.filter { it.name !in excludedFunctions }
 
     private fun mangleStructName(name: String) = if (name !in forbiddenStructNames) name else (name + "Struct")
 
-    val NativeIndex.CStruct.mangledName: String
+    val StructDecl.mangledName: String
-        get() = mangleStructName(name)
+        get() = mangleStructName(kotlinName)
-
-
-    val NativeIndex.CForwardStruct.mangledName: String
-        get() = mangleStructName(name)
-
-    val EnumType.baseType: DirectlyMappedType
-        get() = (parseType(enums[name]!!.type) as DirectlyMappedType)
 
     private var out: (String) -> Unit = {
         throw IllegalStateException()
@@ -61,8 +69,10 @@ class StubGenerator(
         return withOutput({ oldOut("    $it") }, action)
     }
 
-    fun CType.getStringRepresentation(): String {
+    // TODO: use libclang to implement:
+    fun Type.getStringRepresentation(): String {
         return when (this) {
+
             is VoidType -> "void"
             is Int8Type -> "char"
             is UInt8Type -> "unsigned char"
@@ -70,44 +80,75 @@ class StubGenerator(
             is UInt16Type -> "unsigned short"
             is Int32Type -> "int"
             is UInt32Type -> "unsigned int"
+            is IntPtrType -> "intptr_t"
+            is UIntPtrType -> "uintptr_t"
             is Int64Type -> "int64_t"
             is UInt64Type -> "uint64_t"
-            is PointerType -> pointeeType.getStringRepresentation() + "*"
+
-            is RecordType -> structs[name]?.spelling ?: "void" // FIXME
+            is PointerType -> {
-            is FunctionPointerType -> this.returnType.getStringRepresentation() + " (*)(" +
+                val pointeeType = this.pointeeType
-                    this.parameterTypes.map { it.getStringRepresentation() }.joinToString(", ") + ")"
+                if (pointeeType is FunctionType) {
+                    pointeeType.returnType.getStringRepresentation() + " (*)(" +
+                            pointeeType.parameterTypes.map { it.getStringRepresentation() }.joinToString(", ") + ")"
+                } else {
+                    pointeeType.getStringRepresentation() + "*"
+                }
+            }
+
+            is RecordType -> this.decl.spelling
+
             is ArrayType -> "void*" // TODO
-            is EnumType -> enums["${this.name}"]!!.spelling
+
+            is EnumType -> this.def.spelling
+
             else -> throw kotlin.NotImplementedError()
         }
     }
 
+    val PrimitiveType.kotlinType: String
+        get() = when (this) {
+
+            is Int8Type, is UInt8Type -> "Byte"
+            is Int16Type, is UInt16Type -> "Short"
+            is Int32Type, is UInt32Type -> "Int"
+            is IntPtrType, is UIntPtrType, // TODO: 64-bit specific
+            is Int64Type, is UInt64Type -> "Long"
+
+            else -> throw NotImplementedError()
+        }
+
     class NativeRefType(val typeName: String, val typeExpr: String = typeName)
 
-    fun getKotlinNativeRefType(type: CType): NativeRefType = when (type) {
+    fun getKotlinTypeForRefTo(type: Type): NativeRefType = when (type) {
         is Int8Type, is UInt8Type -> NativeRefType("Int8Box")
         is Int16Type, is UInt16Type -> NativeRefType("Int16Box")
         is Int32Type, is UInt32Type -> NativeRefType("Int32Box")
+        is IntPtrType, is UIntPtrType, // TODO: 64-bit specific
         is Int64Type, is UInt64Type -> NativeRefType("Int64Box")
-        is RecordType -> NativeRefType("${mangleStructName(type.name)}")
+
-        is EnumType -> NativeRefType("${type.name}.ref")
+        is RecordType -> NativeRefType("${mangleStructName(type.kotlinName)}")
+
+        is EnumType -> NativeRefType("${type.kotlinName}.ref")
+
         is PointerType -> {
-            if (type.pointeeType is VoidType) {
+            if (type.pointeeType is VoidType || type.pointeeType is FunctionType) {
                 NativeRefType("NativePtrBox")
             } else {
-                val pointeeRefType = getKotlinNativeRefType(type.pointeeType)
+                val pointeeRefType = getKotlinTypeForRefTo(type.pointeeType)
                 NativeRefType("RefBox<${pointeeRefType.typeName}>", "${pointeeRefType.typeExpr}.ref")
             }
         }
-        is FunctionPointerType -> getKotlinNativeRefType(PointerType(VoidType))
+
         is ConstArrayType -> {
-            val elemRefType = getKotlinNativeRefType(type.elemType)
+            val elemRefType = getKotlinTypeForRefTo(type.elemType)
             NativeRefType("NativeArray<${elemRefType.typeName}>", "array[${type.length}](${elemRefType.typeExpr})")
         }
+
         is IncompleteArrayType -> {
-            val elemRefType = getKotlinNativeRefType(type.elemType)
+            val elemRefType = getKotlinTypeForRefTo(type.elemType)
             NativeRefType("NativeArray<${elemRefType.typeName}>", "array(${elemRefType.typeExpr})")
         }
+
         else -> throw NotImplementedError()
     }
 
@@ -132,10 +173,12 @@ class StubGenerator(
             kotlinType = refType.typeName + "?"
     )
 
-    fun getOutValueBinding(type: CType): OutValueBinding = when (type) {
+    fun getOutValueBinding(type: Type): OutValueBinding = when (type) {
-        is DirectlyMappedType -> OutValueBinding(type.kotlinType)
+
+        is PrimitiveType -> OutValueBinding(type.kotlinType)
+
         is PointerType -> {
-            if (type.pointeeType is VoidType) {
+            if (type.pointeeType is VoidType || type.pointeeType is FunctionType) {
                 OutValueBinding(
                         kotlinType = "NativePtr?",
                         kotlinConv = { "$it.asLong()" },
@@ -149,18 +192,20 @@ class StubGenerator(
                         kotlinJniBridgeType = "Long"
                 )
             } else {
-                outValueRefBinding(getKotlinNativeRefType(type.pointeeType))
+                outValueRefBinding(getKotlinTypeForRefTo(type.pointeeType))
             }
         }
+
         is EnumType -> OutValueBinding(
-                kotlinType = type.name,
+                kotlinType = type.kotlinName,
                 kotlinConv = { "$it.value" },
-                kotlinJniBridgeType = type.baseType.kotlinType
+                kotlinJniBridgeType = type.def.baseType.kotlinType
         )
-        is ArrayType -> outValueRefBinding(getKotlinNativeRefType(type))
+
-        is FunctionPointerType -> getOutValueBinding(PointerType(VoidType))
+        is ArrayType -> outValueRefBinding(getKotlinTypeForRefTo(type))
+
         is RecordType -> {
-            val refType = getKotlinNativeRefType(type)
+            val refType = getKotlinTypeForRefTo(type)
             // pointer will be converted to value in C code
             OutValueBinding(
                     kotlinType = refType.typeName,
@@ -168,61 +213,69 @@ class StubGenerator(
                     kotlinJniBridgeType = "Long"
             )
         }
+
         else -> throw NotImplementedError()
     }
 
-    fun getInValueBinding(type: CType): InValueBinding = when (type) {
+    fun getInValueBinding(type: Type): InValueBinding = when (type) {
+
+        is PrimitiveType -> InValueBinding(type.kotlinType)
+
         is VoidType -> InValueBinding("Unit")
-        is DirectlyMappedType -> InValueBinding(type.kotlinType)
+
         is PointerType -> {
-            if (type.pointeeType is VoidType) {
+            if (type.pointeeType is VoidType || type.pointeeType is FunctionType) {
                 InValueBinding(
                         kotlinJniBridgeType = "Long",
                         conv = { "NativePtr.byValue($it)" },
                         kotlinType = "NativePtr?"
                 )
             } else {
-                inValueRefBinding(getKotlinNativeRefType(type.pointeeType))
+                inValueRefBinding(getKotlinTypeForRefTo(type.pointeeType))
             }
         }
+
         is EnumType -> InValueBinding(
-                kotlinJniBridgeType = type.baseType.kotlinType,
+                kotlinJniBridgeType = type.def.baseType.kotlinType,
-                conv = { "${type.name}.byValue($it)" },
+                conv = { "${type.kotlinName}.byValue($it)" },
-                kotlinType = type.name
+                kotlinType = type.kotlinName
         )
-        is ArrayType -> inValueRefBinding(getKotlinNativeRefType(type))
+
+        is ArrayType -> inValueRefBinding(getKotlinTypeForRefTo(type))
+
         is RecordType -> {
-            // FIXME
+            // TODO: valid only for return values
-            val refType = getKotlinNativeRefType(type)
+            val refType = getKotlinTypeForRefTo(type)
             InValueBinding(
                     kotlinJniBridgeType = "Long",
                     conv = { "NativePtr.byValue($it).asRef(${refType.typeExpr})!!" },
                     kotlinType = refType.typeName
             )
         }
+
         else -> throw NotImplementedError()
     }
 
 
-    private fun <T> Iterable<T>.uniqueBy(key: (T) -> Any): List<T> {
+    private fun generateKotlinStruct(decl: StructDecl) {
-        val found = mutableSetOf<Any>()
+        val def = decl.def
-        return this.filter { found.add(key(it)) }
+        if (def == null) {
-    }
+            generateKotlinForwardStruct(decl)
+            return
+        }
 
-
+        val className = decl.mangledName
-    private fun generateKotlinStruct(s: NativeIndex.CStruct) {
-        val className = s.mangledName
         out("class $className(ptr: NativePtr) : NativeStruct(ptr) {")
         indent {
             out("")
-            out("companion object : Type<$className>(${s.size}, ::$className)")
+            out("companion object : Type<$className>(${def.size}, ::$className)")
             out("")
-            s.fieldList.forEach { field ->
+            def.fields.forEach { field ->
                 try {
                     if (field.offset < 0) throw NotImplementedError();
                     assert(field.offset % 8 == 0L)
                     val offset = field.offset / 8
-                    val fieldRefType = getKotlinNativeRefType(parseType(field.type))
+                    val fieldRefType = getKotlinTypeForRefTo(field.type)
                     out("val ${field.name} by ${fieldRefType.typeExpr} at $offset")
                 } catch (e: Throwable) {
                     println("Warning: cannot generate definition for field $className.${field.name}")
@@ -232,31 +285,30 @@ class StubGenerator(
         out("}")
     }
 
-    private fun generateKotlinForwardStruct(s: NativeIndex.CForwardStruct) {
+    private fun generateKotlinForwardStruct(s: StructDecl) {
         val className = s.mangledName
         out("class $className(ptr: NativePtr) : NativeRef(ptr) {")
         out("    companion object : Type<$className>(::$className)")
         out("}")
     }
 
-    private fun generateKotlinEnum(e: NativeIndex.CEnum) {
+    private fun generateKotlinEnum(e: EnumDef) {
-        val baseType = (parseType(e.type) as DirectlyMappedType)
+        val baseRefType = getKotlinTypeForRefTo(e.baseType)
-        val baseRefType = getKotlinNativeRefType(baseType)
 
-        out("enum class ${e.name}(val value: ${baseType.kotlinType}) {")
+        out("enum class ${e.kotlinName}(val value: ${e.baseType.kotlinType}) {")
         indent {
-            e.valueList.forEach {
+            e.values.forEach {
                 out("${it.name}(${it.value}),")
             }
             out(";")
             out("")
             out("companion object {")
-            out("    fun byValue(value: ${baseType.kotlinType}) = ${e.name}.values().find { it.value == value }!!")
+            out("    fun byValue(value: ${e.baseType.kotlinType}) = ${e.kotlinName}.values().find { it.value == value }!!")
             out("}")
             out("")
             out("class ref(ptr: NativePtr) : NativeRef(ptr) {")
             out("    companion object : TypeWithSize<ref>(${baseRefType.typeExpr}.size, ::ref)")
-            out("    var value: ${e.name}")
+            out("    var value: ${e.kotlinName}")
             out("        get() = byValue(${baseRefType.typeExpr}.byPtr(ptr).value)")
             out("        set(value) { ${baseRefType.typeExpr}.byPtr(ptr).value = value.value }")
             out("}")
@@ -265,16 +317,16 @@ class StubGenerator(
         out("}")
     }
 
-    private fun retValBinding(func: NativeIndex.Function) = getInValueBinding(parseType(func.returnType))
+    private fun retValBinding(func: FunctionDecl) = getInValueBinding(func.returnType)
 
-    private fun paramBindings(func: NativeIndex.Function): Array<OutValueBinding> {
+    private fun paramBindings(func: FunctionDecl): Array<OutValueBinding> {
-        val paramBindings = func.parameterList.map { param ->
+        val paramBindings = func.parameters.map { param ->
-            getOutValueBinding(parseType(param.type))
+            getOutValueBinding(param.type)
         }.toMutableList()
 
-        val retValType = parseType(func.returnType)
+        val retValType = func.returnType
         if (retValType is RecordType) {
-            val retValRefType = getKotlinNativeRefType(retValType)
+            val retValRefType = getKotlinTypeForRefTo(retValType)
             val typeExpr = retValRefType.typeExpr
 
             paramBindings.add(OutValueBinding(
@@ -287,19 +339,24 @@ class StubGenerator(
         return paramBindings.toTypedArray()
     }
 
-    private fun paramNames(func: NativeIndex.Function): Array<String> {
+    private fun paramNames(func: FunctionDecl): Array<String> {
-        val paramNames = func.parameterList.mapIndexed { i: Int, parameter: NativeIndex.Function.Parameter ->
+        val paramNames = func.parameters.mapIndexed { i: Int, parameter: Parameter ->
-            if (parameter.name != "") parameter.name else "arg$i"
+            val name = parameter.name
+            if (name != null && name != "") {
+                name
+            } else {
+                "arg$i"
+            }
         }.toMutableList()
 
-        if (parseType(func.returnType) is RecordType) {
+        if (func.returnType is RecordType) {
             paramNames.add("retValPlacement")
         }
 
         return paramNames.toTypedArray()
     }
 
-    private fun generateKotlinBindingMethod(func: NativeIndex.Function) {
+    private fun generateKotlinBindingMethod(func: FunctionDecl) {
         val paramNames = paramNames(func)
         val paramBindings = paramBindings(func)
         val retValBinding = retValBinding(func)
@@ -337,7 +394,7 @@ class StubGenerator(
         out("}")
     }
 
-    private fun generateKotlinExternalMethod(func: NativeIndex.Function) {
+    private fun generateKotlinExternalMethod(func: FunctionDecl) {
         val paramNames = paramNames(func)
         val paramBindings = paramBindings(func)
         val retValBinding = retValBinding(func)
@@ -368,29 +425,18 @@ class StubGenerator(
             }
         }
 
-        val generatedStructs = mutableSetOf<String>()
+        nativeIndex.structs.forEach { s ->
-
-        translationUnit.structList.uniqueBy { it.name }.forEach { s ->
             try {
                 transaction {
                     generateKotlinStruct(s)
                     out("")
-                    generatedStructs.add(s.name)
                 }
             } catch (e: Throwable) {
-                println("Warning: cannot generate definition for struct ${s.name}")
+                println("Warning: cannot generate definition for struct ${s.kotlinName}")
             }
         }
 
-        translationUnit.forwardStructList
+        nativeIndex.enums.forEach { e ->
-                .uniqueBy { it.name }
-                .filter { it.name !in generatedStructs }
-                .forEach { s ->
-                    generateKotlinForwardStruct(s)
-                    out("")
-                }
-
-        translationUnit.enumList.uniqueBy { it.name }.forEach { e ->
             generateKotlinEnum(e)
             out("")
         }
@@ -421,8 +467,7 @@ class StubGenerator(
         else -> throw NotImplementedError(kotlinJniBridgeType)
     }
 
-
+    fun generateCFile(headerFiles: List<String>) {
-    fun generateCFile(headerFiles: List<String>, translationUnit: NativeIndex.TranslationUnit) {
         out("#include <stdint.h>")
         out("#include <jni.h>")
         headerFiles.forEach {
@@ -432,51 +477,56 @@ class StubGenerator(
 
         functionsToBind.forEach { func ->
             try {
-                val paramNames = paramNames(func)
+                generateCJniFunction(func)
-                val paramBindings = paramBindings(func)
-                val retValBinding = retValBinding(func)
-
-                val args =
-                        if (paramBindings.isEmpty())
-                            ""
-                        else paramBindings
-                                .map { getCJniBridgeType(it.kotlinJniBridgeType) }
-                                .mapIndexed { i, type -> "$type ${paramNames[i]}" }
-                                .joinToString(separator = ", ", prefix = ", ")
-
-                val cReturnType = getCJniBridgeType(retValBinding.kotlinJniBridgeType)
-
-                val params = func.parameterList.mapIndexed { i, parameter ->
-                    val cType = parseType(parameter.type).getStringRepresentation()
-                    if (parseType(parameter.type) is RecordType) {
-                        "*($cType*)${paramNames[i]}" // FIXME
-                    } else {
-                        "($cType)${paramNames[i]}"
-                    }
-                }.joinToString(", ")
-
-                val callExpr = "${func.name}($params)"
-                val funcFullName = if (pkgName.isEmpty()) {
-                    "externals.${func.name}"
-                } else {
-                    "$pkgName.externals.${func.name}"
-                }
-                val jniFuncName = "Java_" + funcFullName.replace("_", "_1").replace('.', '_').replace("$", "_00024")
-
-                out("JNIEXPORT $cReturnType JNICALL $jniFuncName (JNIEnv *env, jobject obj$args) {")
-
-                if (cReturnType == "void") {
-                    out("    $callExpr;")
-                } else if (retValBinding.kotlinType in structs) { // FIXME
-                    out("    *(${structs[retValBinding.kotlinType]!!.spelling}*)retValPlacement = $callExpr;")
-                    out("    return ($cReturnType) retValPlacement;")
-                } else {
-                    out("    return ($cReturnType) ($callExpr);")
-                }
-                out("}")
             } catch (e: Throwable) {
                 System.err.println("Warning: cannot generate C JNI function definition ${func.name}")
             }
         }
     }
+
+    private fun generateCJniFunction(func: FunctionDecl) {
+        val funcReturnType = func.returnType
+        val paramNames = paramNames(func)
+        val paramBindings = paramBindings(func)
+        val retValBinding = retValBinding(func)
+
+        val args =
+                if (paramBindings.isEmpty())
+                    ""
+                else paramBindings
+                        .map { getCJniBridgeType(it.kotlinJniBridgeType) }
+                        .mapIndexed { i, type -> "$type ${paramNames[i]}" }
+                        .joinToString(separator = ", ", prefix = ", ")
+
+        val cReturnType = getCJniBridgeType(retValBinding.kotlinJniBridgeType)
+
+        val params = func.parameters.mapIndexed { i, parameter ->
+            val cType = parameter.type.getStringRepresentation()
+            if (parameter.type is RecordType) {
+                "*($cType*)${paramNames[i]}"
+            } else {
+                "($cType)${paramNames[i]}"
+            }
+        }.joinToString(", ")
+
+        val callExpr = "${func.name}($params)"
+        val funcFullName = if (pkgName.isEmpty()) {
+            "externals.${func.name}"
+        } else {
+            "$pkgName.externals.${func.name}"
+        }
+        val jniFuncName = "Java_" + funcFullName.replace("_", "_1").replace('.', '_').replace("$", "_00024")
+
+        out("JNIEXPORT $cReturnType JNICALL $jniFuncName (JNIEnv *env, jobject obj$args) {")
+
+        if (cReturnType == "void") {
+            out("    $callExpr;")
+        } else if (funcReturnType is RecordType) {
+            out("    *(${funcReturnType.decl.spelling}*)retValPlacement = $callExpr;")
+            out("    return ($cReturnType) retValPlacement;")
+        } else {
+            out("    return ($cReturnType) ($callExpr);")
+        }
+        out("}")
+    }
 }

experiments/Interop/StubGenerator/src/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt

@@ -1,10 +1,7 @@
 package org.jetbrains.kotlin.native.interop.gen.jvm
 
-import org.jetbrains.kni.indexer.IndexerOptions
+import org.jetbrains.kotlin.native.interop.indexer.NativeIndex
-import org.jetbrains.kni.indexer.Language
+import org.jetbrains.kotlin.native.interop.indexer.buildNativeIndex
-import org.jetbrains.kni.indexer.NativeIndex
-import org.jetbrains.kni.indexer.buildNativeIndex
-import org.jetbrains.kotlin.native.interop.gen.jvm.StubGenerator
 import java.io.File
 import java.util.*
 import kotlin.system.exitProcess
@@ -35,9 +32,9 @@ fun main(args: Array<String>) {
 
 
 
-        val translationUnit = buildNativeIndex(headerFiles, compilerOpts)
+        val nativeIndex = buildNativeIndex(headerFiles, compilerOpts)
 
-        val gen = StubGenerator(translationUnit, outKtPkg, libName, excludedFunctions)
+        val gen = StubGenerator(nativeIndex, outKtPkg, libName, excludedFunctions)
 
         File(outKtFile).bufferedWriter().use { out ->
             gen.withOutput({ out.appendln(it) }) {
@@ -50,7 +47,7 @@ fun main(args: Array<String>) {
 
         outCFile.bufferedWriter().use { out ->
             gen.withOutput({ out.appendln(it) }) {
-                gen.generateCFile(headerFiles, translationUnit)
+                gen.generateCFile(headerFiles)
             }
         }
 
@@ -91,12 +88,10 @@ fun main(args: Array<String>) {
 
         outCFile.delete()
         outOFile.delete()
-
     }
-
 }
 
-private fun buildNativeIndex(headerFiles: List<String>, compilerOpts: List<String>): NativeIndex.TranslationUnit {
+private fun buildNativeIndex(headerFiles: List<String>, compilerOpts: List<String>): NativeIndex {
     val tempHeaderFile = createTempFile(suffix = ".h")
     tempHeaderFile.deleteOnExit()
     tempHeaderFile.writer().buffered().use { reader ->
@@ -105,6 +100,7 @@ private fun buildNativeIndex(headerFiles: List<String>, compilerOpts: List<Strin
         }
     }
 
-    val indexerOptions = IndexerOptions(language = Language.CPP, args = compilerOpts)
+    val res = buildNativeIndex(tempHeaderFile, compilerOpts)
-    return buildNativeIndex(tempHeaderFile, indexerOptions)
+    println(res)
+    return res
 }