Implement parsing additional interop header directly from .def file (#557)

INTEROP.md

@@ -124,6 +124,28 @@ excludeDependentModules = true
 When both `excludeDependentModules` and `headerFilter` are used, they are
 applied as intersection.
 
+### Adding custom declarations ###
+
+Sometimes it is required to add custom C declarations to the library before
+generating bindings (e.g. for [macros](#macros)). Instead of creating
+additional header file with these declarations, you can include them directly
+to the end of the `.def` file, after separating line, containing only the
+separator sequence `---`:
+
+```
+headers = errno.h
+
+---
+
+static inline int getErrno() {
+    return errno;
+}
+```
+
+Note that this part of the `.def` file is treated as part of the header file, so
+functions with body should be declared as `static`.
+The declarations are parsed after including the files from `headers` list.
+
 ## Using bindings ##
 
 ### Basic interop types ###
@@ -366,6 +388,24 @@ After that it becomes invalid, so `voidPtr` can't be unwrapped anymore.
 
 See `samples/libcurl` for more details.
 
+### Macros ###
+
+Every C macro that expands to a constant is represented as Kotlin property.
+Other macros are not supported. However they can be exposed manually by
+wrapping with supported declarations. E.g. function-like macro `FOO` can be
+exposed as function `foo` by
+[adding the custom declaration](#adding-custom-declarations) to the library:
+
+```
+headers = library/base.h
+
+---
+
+static inline int foo(int arg) {
+    return FOO(arg);
+}
+```
+
 ### Definition file hints ###
 
 The `.def` file supports several options for adjusting generated bindings.

Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/NativeIndex.kt

@@ -21,6 +21,7 @@ enum class Language {
 }
 
 data class NativeLibrary(val includes: List<String>,
+                         val additionalPreambleLines: List<String>,
                          val compilerArgs: List<String>,
                          val language: Language,
                          val excludeSystemLibs: Boolean, // TODO: drop?

Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Utils.kt

@@ -164,7 +164,7 @@ internal fun List<String>.toNativeStringArray(placement: NativePlacement): CArra
 }
 
 val NativeLibrary.preambleLines: List<String>
-    get() = this.includes.map { "#include <$it>" }
+    get() = this.includes.map { "#include <$it>" } + this.additionalPreambleLines
 
 internal fun Appendable.appendPreamble(library: NativeLibrary) = this.apply {
     library.preambleLines.forEach {
@@ -213,6 +213,7 @@ internal fun NativeLibrary.precompileHeaders(): NativeLibrary {
 
     return this.copy(
             includes = emptyList(),
+            additionalPreambleLines = emptyList(),
             compilerArgs = this.compilerArgs + listOf("-include-pch", precompiledHeader.absolutePath)
     )
 }
@@ -297,6 +298,11 @@ fun List<List<String>>.mapFragmentIsCompilable(originalLibrary: NativeLibrary):
 }
 
 internal interface Indexer {
+    /**
+     * Called when entered main file.
+     */
+    fun enteredMainFile(file: CXFile) {}
+
     /**
      * Called when a file gets #included/#imported.
      */
@@ -316,7 +322,12 @@ internal fun indexTranslationUnit(index: CXIndex, translationUnit: CXTranslation
             val indexerCallbacks = alloc<IndexerCallbacks>().apply {
                 abortQuery = null
                 diagnostic = null
-                enteredMainFile = null
+                enteredMainFile = staticCFunction { clientData, mainFile, reserved ->
+                    @Suppress("NAME_SHADOWING")
+                    val indexer = StableObjPtr.fromValue(clientData!!).get() as Indexer
+                    indexer.enteredMainFile(mainFile!!)
+                    null as CXIdxClientFile?
+                }
                 ppIncludedFile = staticCFunction { clientData, info ->
                     @Suppress("NAME_SHADOWING")
                     val indexer = StableObjPtr.fromValue(clientData!!).get() as Indexer
@@ -402,11 +413,16 @@ internal class ModulesMap(
 internal fun getFilteredHeaders(library: NativeLibrary, index: CXIndex, translationUnit: CXTranslationUnit): Set<CXFile> {
     val result = mutableSetOf<CXFile>()
     val topLevelFiles = mutableListOf<CXFile>()
+    var mainFile: CXFile? = null
 
     indexTranslationUnit(index, translationUnit, 0, object : Indexer {
         val headerToName = mutableMapOf<CXFile, String>()
         // The *name* of the header here is the path relative to the include path element., e.g. `curl/curl.h`.
 
+        override fun enteredMainFile(file: CXFile) {
+            mainFile = file
+        }
+
         override fun ppIncludedFile(info: CXIdxIncludedFileInfo) {
             val includeLocation = clang_indexLoc_getCXSourceLocation(info.hashLoc.readValue())
             val file = info.file!!
@@ -458,5 +474,8 @@ internal fun getFilteredHeaders(library: NativeLibrary, index: CXIndex, translat
         }
     }
 
+
+    result.add(mainFile!!)
+
     return result
 }

Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt

@@ -18,6 +18,7 @@ package org.jetbrains.kotlin.native.interop.gen.jvm
 
 import org.jetbrains.kotlin.native.interop.indexer.*
 import java.io.File
+import java.io.StringReader
 import java.lang.IllegalArgumentException
 import java.util.*
 import kotlin.reflect.KFunction
@@ -234,6 +235,36 @@ private fun loadProperties(file: File?, substitutions: Map<String, String>): Pro
     return result
 }
 
+private fun parseDefFile(file: File?, substitutions: Map<String, String>): Pair<Properties, List<String>> {
+    val properties = Properties()
+
+    if (file == null) {
+        return properties to emptyList()
+    }
+
+    val lines = file.readLines()
+
+    val separator = "---"
+    val separatorIndex = lines.indexOf(separator)
+
+    val propertyLines: List<String>
+    val headerLines: List<String>
+
+    if (separatorIndex != -1) {
+        propertyLines = lines.subList(0, separatorIndex)
+        headerLines = lines.subList(separatorIndex + 1, lines.size)
+    } else {
+        propertyLines = lines
+        headerLines = emptyList()
+    }
+
+    val propertiesReader = StringReader(propertyLines.joinToString(System.lineSeparator()))
+    properties.load(propertiesReader)
+    substitute(properties, substitutions)
+
+    return properties to headerLines
+}
+
 private fun usage() {
     println("""
 Run interop tool with -def <def_file_for_lib>.def
@@ -270,7 +301,7 @@ private fun processLib(konanHome: String,
         return
     }
 
-    val config = loadProperties(defFile, substitutions)
+    val (config, defHeaderLines) = parseDefFile(defFile, substitutions)
 
     val konanFileName = args["-properties"]?.single() ?:
         "${konanHome}/konan/konan.properties"
@@ -329,6 +360,7 @@ private fun processLib(konanHome: String,
 
     val library = NativeLibrary(
             includes = headerFiles,
+            additionalPreambleLines = defHeaderLines,
             compilerArgs = compilerOpts,
             language = language,
             excludeSystemLibs = excludeSystemLibs,

backend.native/tests/interop/basics/4.kt

@@ -1,8 +1,11 @@
 import cstdio.*
 import kotlinx.cinterop.*
 
+val stdout
+    get() = getStdout()
+
 fun main(args: Array<String>) {
-    printf("%s %s %d %d %d %lld %.1f %.1lf\n",
+    fprintf(stdout, "%s %s %d %d %d %lld %.1f %.1lf\n",
             "a", "b".cstr, (-1).toByte(), 2.toShort(), 3, Long.MAX_VALUE, 0.1.toFloat(), 0.2)
 
     memScoped {

backend.native/tests/interop/basics/cstdio.def

@@ -1 +1,7 @@
-headers = stdio.h
+headers = stdio.h
+
+---
+
+static inline FILE* getStdout() {
+    return stdout;
+}