Implement filtering interop headers by globs and by modulemaps

Also do some refactoring

INTEROP.md

@@ -78,6 +78,52 @@ output of config script with `--cflags` flag (maybe without exact paths).
 Output of config script with `--libs` shall be passed as `-linkedArgs`  `kotlinc`
 flag value (quoted) when compiling.
 
+### Selecting library headers
+
+When library headers are imported to C program with `#include` directive,
+all of the headers included by these headers are also included to the program.
+Thus all header dependencies are included in generated stubs as well.
+
+This behaviour is correct but may be very inconvenient for some libraries. So
+it is possible to specify in `.def` file which of the included headers are to
+be imported. The separate declarations from other headers may also be imported
+in case of direct dependencies.
+
+#### Filtering headers by globs
+
+It is possible to filter header by globs. The `headerFilter` property value
+from the `.def` file is treated as space-separated list of globs. If the
+included header matches any of the globs, then declarations from this header
+are included into the bindings.
+
+The globs are applied to the header paths relative to the appropriate include
+path elements, e.g. `time.h` or `curl/curl.h`. So if the library is usually
+included with `#include <SomeLbrary/Header.h>`, then it would probably be
+correct to filter headers with
+```
+headerFilter = SomeLbrary/**
+```
+
+If `headerFilter` is not specified, then all headers are included.
+
+#### Filtering by module maps
+
+Some libraries have proper `module.modulemap` or `module.map` files among its
+headers. For example, macOS and iOS system libraries and frameworks do.
+The [module map file](https://clang.llvm.org/docs/Modules.html#module-map-language)
+describes the correspondence between header files and modules. When the module
+maps are available, the headers from the modules that are not included directly
+can be filtered out using experimental `excludeDependentModules` option of the
+`.def` file:
+```
+headers = OpenGL/gl.h OpenGL/glu.h GLUT/glut.h
+compilerOpts = -framework OpenGL -framework GLUT
+excludeDependentModules = true
+```
+
+When both `excludeDependentModules` and `headerFilter` are used, they are
+applied as intersection.
+
 ## Using bindings ##
 
 ### Basic interop types ###

Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt

@@ -360,45 +360,30 @@ fun buildNativeIndexImpl(library: NativeLibrary): NativeIndex {
 
 private fun indexDeclarations(library: NativeLibrary, nativeIndex: NativeIndexImpl) {
     val index = clang_createIndex(0, 0)!!
-    val indexAction = clang_IndexAction_create(index)!!
     try {
-        val translationUnit = library.parse(index).ensureNoCompileErrors()
+        val translationUnit = library.parse(index, options = CXTranslationUnit_DetailedPreprocessingRecord)
         try {
-            memScoped {
+            translationUnit.ensureNoCompileErrors()
-                val callbacks = alloc<IndexerCallbacks>()
 
-                val nativeIndexPtr = StableObjPtr.create(nativeIndex)
+            val headers = getFilteredHeaders(library, index, translationUnit)
-                val clientData = nativeIndexPtr.value
 
-                try {
+            indexTranslationUnit(index, translationUnit, 0, object : Indexer {
-                    with(callbacks) {
+                override fun indexDeclaration(info: CXIdxDeclInfo) {
-                        abortQuery = null
+                    val file = memScoped {
-                        diagnostic = null
+                        val fileVar = alloc<CXFileVar>()
-                        enteredMainFile = null
+                        clang_indexLoc_getFileLocation(info.loc.readValue(), null, fileVar.ptr, null, null, null)
-                        ppIncludedFile = null
+                        fileVar.value
-                        importedASTFile = null
-                        startedTranslationUnit = null
-                        indexDeclaration = staticCFunction { clientData, info ->
-                            @Suppress("NAME_SHADOWING")
-                            val nativeIndex = StableObjPtr.fromValue(clientData!!).get() as NativeIndexImpl
-                            nativeIndex.indexDeclaration(info!!.pointed)
-                        }
-                        indexEntityReference = null
                     }
 
-                    clang_indexTranslationUnit(indexAction, clientData,
+                    if (file in headers) {
-                            callbacks.ptr, IndexerCallbacks.size.toInt(),
+                        nativeIndex.indexDeclaration(info)
-                            0, translationUnit)
+                    }
-
-                } finally {
-                    nativeIndexPtr.dispose()
                 }
-            }
+            })
         } finally {
             clang_disposeTranslationUnit(translationUnit)
         }
     } finally {
-        clang_IndexAction_dispose(indexAction)
         clang_disposeIndex(index)
     }
 }

Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/MacroConstants.kt

@@ -17,7 +17,7 @@
 package org.jetbrains.kotlin.native.interop.indexer
 
 import clang.*
-import kotlinx.cinterop.CValue
+import kotlinx.cinterop.*
 import java.io.File
 
 /**
@@ -54,9 +54,8 @@ private fun expandMacroConstants(
     try {
         val sourceFile = library.createTempSource()
         val translationUnit = parseTranslationUnit(index, sourceFile, library.compilerArgs, options = 0)
-                .ensureNoCompileErrors()
-
         try {
+            translationUnit.ensureNoCompileErrors()
             return names.mapNotNull {
                 expandMacroAsConstant(library, translationUnit, sourceFile, it, typeConverter)
             }
@@ -214,11 +213,21 @@ private fun collectMacroConstantsNames(library: NativeLibrary): List<String> {
         // Include macros into AST:
         val options = CXTranslationUnit_DetailedPreprocessingRecord
 
-        val translationUnit = library.parse(index, options).ensureNoCompileErrors()
+        val translationUnit = library.parse(index, options)
         try {
+            translationUnit.ensureNoCompileErrors()
+            val headers = getFilteredHeaders(library, index, translationUnit)
+
             visitChildren(translationUnit) { cursor, _ ->
+                val file = memScoped {
+                    val fileVar = alloc<CXFileVar>()
+                    clang_getFileLocation(clang_getCursorLocation(cursor), fileVar.ptr, null, null, null)
+                    fileVar.value
+                }
+
                 if (cursor.kind == CXCursorKind.CXCursor_MacroDefinition &&
                         library.includesDeclaration(cursor) &&
+                        file in headers &&
                         canMacroBeConstant(cursor))
                 {
                     val spelling = getCursorSpelling(cursor)

Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/NativeIndex.kt

@@ -23,7 +23,9 @@ enum class Language {
 data class NativeLibrary(val includes: List<String>,
                          val compilerArgs: List<String>,
                          val language: Language,
-                         val excludeSystemLibs: Boolean = true)
+                         val excludeSystemLibs: Boolean, // TODO: drop?
+                         val excludeDepdendentModules: Boolean,
+                         val headerFilter: (String) -> Boolean)
 
 /**
  * Retrieves the definitions from given C header file using given compiler arguments (e.g. defines).

Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Utils.kt

@@ -18,7 +18,9 @@ package org.jetbrains.kotlin.native.interop.indexer
 
 import clang.*
 import kotlinx.cinterop.*
+import java.io.Closeable
 import java.io.File
+import java.nio.file.Paths
 
 internal val CValue<CXType>.kind: CXTypeKind get() = this.useContents { kind }
 
@@ -198,8 +200,9 @@ internal fun NativeLibrary.precompileHeaders(): NativeLibrary {
     val index = clang_createIndex(excludeDeclarationsFromPCH = 0, displayDiagnostics = 0)!!
     try {
         val options = CXTranslationUnit_ForSerialization
-        val translationUnit = this.parse(index, options).ensureNoCompileErrors()
+        val translationUnit = this.parse(index, options)
         try {
+            translationUnit.ensureNoCompileErrors()
             clang_saveTranslationUnit(translationUnit, precompiledHeader.absolutePath, 0)
         } finally {
             clang_disposeTranslationUnit(translationUnit)
@@ -249,9 +252,8 @@ fun List<List<String>>.mapFragmentIsCompilable(originalLibrary: NativeLibrary):
     try {
         val sourceFile = library.createTempSource()
         val translationUnit = parseTranslationUnit(index, sourceFile, library.compilerArgs, options = 0)
-                .ensureNoCompileErrors()
-
         try {
+            translationUnit.ensureNoCompileErrors()
             while (fragmentsToCheck.isNotEmpty()) {
                 // Combine all fragments to be checked in a single file:
                 sourceFile.bufferedWriter().use { writer ->
@@ -292,4 +294,169 @@ fun List<List<String>>.mapFragmentIsCompilable(originalLibrary: NativeLibrary):
     }
 
     return this.indices.map { it !in indicesOfNonCompilable }
+}
+
+internal interface Indexer {
+    /**
+     * Called when a file gets #included/#imported.
+     */
+    fun ppIncludedFile(info: CXIdxIncludedFileInfo) {}
+
+    /**
+     * Called to index a declaration.
+     */
+    fun indexDeclaration(info: CXIdxDeclInfo) {}
+}
+
+internal fun indexTranslationUnit(index: CXIndex, translationUnit: CXTranslationUnit, options: Int, indexer: Indexer) {
+    val indexerStablePtr = StableObjPtr.create(indexer)
+    try {
+        val clientData = indexerStablePtr.value
+        memScoped {
+            val indexerCallbacks = alloc<IndexerCallbacks>().apply {
+                abortQuery = null
+                diagnostic = null
+                enteredMainFile = null
+                ppIncludedFile = staticCFunction { clientData, info ->
+                    @Suppress("NAME_SHADOWING")
+                    val indexer = StableObjPtr.fromValue(clientData!!).get() as Indexer
+                    indexer.ppIncludedFile(info!!.pointed)
+                    null as CXIdxClientFile?
+                }
+                importedASTFile = null
+                startedTranslationUnit = null
+                indexDeclaration = staticCFunction { clientData, info ->
+                    @Suppress("NAME_SHADOWING")
+                    val nativeIndex = StableObjPtr.fromValue(clientData!!).get() as Indexer
+                    nativeIndex.indexDeclaration(info!!.pointed)
+                }
+                indexEntityReference = null
+            }
+
+            val indexAction = clang_IndexAction_create(index)
+            try {
+                val result = clang_indexTranslationUnit(indexAction, clientData,
+                        indexerCallbacks.ptr, IndexerCallbacks.size.toInt(), options, translationUnit)
+
+                if (result != 0) {
+                    throw Error("clang_indexTranslationUnit returned $result")
+                }
+            } finally {
+                clang_IndexAction_dispose(indexAction)
+            }
+        }
+    } finally {
+        indexerStablePtr.dispose()
+    }
+}
+
+internal class ModulesMap(
+        val library: NativeLibrary,
+        val translationUnit: CXTranslationUnit
+) : Closeable {
+
+    private val index: CXIndex
+    private val translationUnitWithModules: CXTranslationUnit
+
+    init {
+        index = clang_createIndex(0, 0)!!
+        try {
+            translationUnitWithModules =
+                    library
+                            .copy(compilerArgs = library.compilerArgs + "-fmodules")
+                            .parse(index)
+
+            try {
+                translationUnitWithModules.ensureNoCompileErrors()
+            } catch (e: Throwable) {
+                clang_disposeTranslationUnit(translationUnitWithModules)
+                throw e
+            }
+
+        } catch (e: Throwable) {
+            clang_disposeIndex(index)
+            throw e
+        }
+    }
+
+    override fun close() {
+        try {
+            clang_disposeTranslationUnit(translationUnitWithModules)
+        } finally {
+            clang_disposeIndex(index)
+        }
+    }
+
+    data class Module(private val cxModule: CXModule)
+
+    fun getModule(file: CXFile): Module? {
+        // `file` is bound to `translationUnit`, however `translationUnitWithModules` is used to access modules.
+        // Find the corresponding file in `translationUnitWithModules`:
+        val fileInTuWithModules =
+                clang_getFile(translationUnitWithModules, clang_getFileName(file).convertAndDispose())!!
+
+        return clang_getModuleForFile(translationUnitWithModules, fileInTuWithModules)?.let { Module(it) }
+    }
+}
+
+internal fun getFilteredHeaders(library: NativeLibrary, index: CXIndex, translationUnit: CXTranslationUnit): Set<CXFile> {
+    val result = mutableSetOf<CXFile>()
+    val topLevelFiles = mutableListOf<CXFile>()
+
+    indexTranslationUnit(index, translationUnit, 0, object : Indexer {
+        val headerToName = mutableMapOf<CXFile, String>()
+        // The *name* of the header here is the path relative to the include path element., e.g. `curl/curl.h`.
+
+        override fun ppIncludedFile(info: CXIdxIncludedFileInfo) {
+            val includeLocation = clang_indexLoc_getCXSourceLocation(info.hashLoc.readValue())
+            val file = info.file!!
+
+            if (clang_Location_isFromMainFile(includeLocation) != 0) {
+                topLevelFiles.add(file)
+            }
+
+            val name = info.filename!!.toKString()
+            val headerName = if (info.isAngled != 0) {
+                // If the header is included with `#include <$name>`, then `name` is probably
+                // the path relative to the include path element.
+                name
+            } else {
+                // If it is included with `#include "$name"`, then `name` can also be the path relative to the includer.
+                val includerFile = memScoped {
+                    val fileVar = alloc<CXFileVar>()
+                    clang_getFileLocation(includeLocation, fileVar.ptr, null, null, null)
+                    fileVar.value!!
+                }
+                val includerName = headerToName[includerFile] ?: ""
+                val includerPath = clang_getFileName(includerFile).convertAndDispose()
+
+                if (clang_getFile(translationUnit, Paths.get(includerPath).resolveSibling(name).toString()) == file) {
+                    // included file is accessible from the includer by `name` used as relative path, so
+                    // `name` seems to be relative to the includer:
+                    Paths.get(includerName).resolveSibling(name).normalize().toString()
+                } else {
+                    name
+                }
+            }
+
+            headerToName[file] = headerName
+            if (library.headerFilter(headerName)) {
+                result.add(file)
+            }
+        }
+    })
+
+    if (library.excludeDepdendentModules) {
+        ModulesMap(library, translationUnit).use { modulesMap ->
+            val topLevelModules = topLevelFiles.map { modulesMap.getModule(it) }.toSet()
+            result.removeAll {
+                val module = modulesMap.getModule(it)
+                module !in topLevelModules
+            }
+            // Note: if some of the top-level headers don't belong to modules,
+            // then all non-modular headers are included.
+        }
+    }
+
+    return result
 }

Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt

@@ -16,17 +16,14 @@
 
 package org.jetbrains.kotlin.native.interop.gen.jvm
 
-import org.jetbrains.kotlin.native.interop.indexer.Language
+import org.jetbrains.kotlin.native.interop.indexer.*
-import org.jetbrains.kotlin.native.interop.indexer.NativeIndex
-import org.jetbrains.kotlin.native.interop.indexer.NativeLibrary
-import org.jetbrains.kotlin.native.interop.indexer.buildNativeIndex
 import java.io.File
 import java.lang.IllegalArgumentException
 import java.util.*
 import kotlin.reflect.KFunction
 
 fun main(args: Array<String>) {
-    val konanHome = System.getProperty("konan.home")!!
+    val konanHome = File(System.getProperty("konan.home")).absolutePath
 
     // TODO: remove OSX defaults.
     val substitutions = mapOf(
@@ -116,6 +113,10 @@ private fun <T> Collection<T>.atMostOne(): T? {
     }
 }
 
+private fun String.matchesToGlob(glob: String): Boolean =
+        java.nio.file.FileSystems.getDefault()
+                .getPathMatcher("glob:$glob").matches(java.nio.file.Paths.get(this))
+
 private fun Properties.getOsSpecific(name: String, 
     host: String = detectHost()): String? {
 
@@ -292,6 +293,7 @@ private fun processLib(konanHome: String,
     val compiler = "clang"
     val language = Language.C
     val excludeSystemLibs = config.getProperty("excludeSystemLibs")?.toBoolean() ?: false
+    val excludeDependentModules = config.getProperty("excludeDependentModules")?.toBoolean() ?: false
 
     val entryPoint = config.getSpaceSeparated("entryPoint").atMostOne()
     val linkerOpts = 
@@ -312,7 +314,25 @@ private fun processLib(konanHome: String,
 
     val libName = args["-cstubsname"]?.atMostOne() ?: fqParts.joinToString("") + "stubs"
 
-    val library = NativeLibrary(headerFiles, compilerOpts, language, excludeSystemLibs)
+    val headerFilterGlobs = config.getSpaceSeparated("headerFilter")
+
+    val headerFilter = { name: String ->
+        if (headerFilterGlobs.isEmpty()) {
+            true
+        } else {
+            headerFilterGlobs.any { name.matchesToGlob(it) }
+        }
+    }
+
+    val library = NativeLibrary(
+            includes = headerFiles,
+            compilerArgs = compilerOpts,
+            language = language,
+            excludeSystemLibs = excludeSystemLibs,
+            excludeDepdendentModules = excludeDependentModules,
+            headerFilter = headerFilter
+    )
+
     val configuration = InteropConfiguration(
             library = library,
             pkgName = outKtPkg,