Implement filtering interop headers by globs and by modulemaps

Also do some refactoring
2017-04-25 15:42:57 +03:00
parent 94ff4d69be
commit c5495e744e
6 changed files with 271 additions and 42 deletions
@@ -78,6 +78,52 @@ output of config script with `--cflags` flag (maybe without exact paths).
 Output of config script with `--libs` shall be passed as `-linkedArgs`  `kotlinc`
 flag value (quoted) when compiling.

+### Selecting library headers
+
+When library headers are imported to C program with `#include` directive,
+all of the headers included by these headers are also included to the program.
+Thus all header dependencies are included in generated stubs as well.
+
+This behaviour is correct but may be very inconvenient for some libraries. So
+it is possible to specify in `.def` file which of the included headers are to
+be imported. The separate declarations from other headers may also be imported
+in case of direct dependencies.
+
+#### Filtering headers by globs
+
+It is possible to filter header by globs. The `headerFilter` property value
+from the `.def` file is treated as space-separated list of globs. If the
+included header matches any of the globs, then declarations from this header
+are included into the bindings.
+
+The globs are applied to the header paths relative to the appropriate include
+path elements, e.g. `time.h` or `curl/curl.h`. So if the library is usually
+included with `#include <SomeLbrary/Header.h>`, then it would probably be
+correct to filter headers with
+```
+headerFilter = SomeLbrary/**
+```
+
+If `headerFilter` is not specified, then all headers are included.
+
+#### Filtering by module maps
+
+Some libraries have proper `module.modulemap` or `module.map` files among its
+headers. For example, macOS and iOS system libraries and frameworks do.
+The [module map file](https://clang.llvm.org/docs/Modules.html#module-map-language)
+describes the correspondence between header files and modules. When the module
+maps are available, the headers from the modules that are not included directly
+can be filtered out using experimental `excludeDependentModules` option of the
+`.def` file:
+```
+headers = OpenGL/gl.h OpenGL/glu.h GLUT/glut.h
+compilerOpts = -framework OpenGL -framework GLUT
+excludeDependentModules = true
+```
+
+When both `excludeDependentModules` and `headerFilter` are used, they are
+applied as intersection.
+
 ## Using bindings ##

 ### Basic interop types ###
@@ -360,45 +360,30 @@ fun buildNativeIndexImpl(library: NativeLibrary): NativeIndex {

 private fun indexDeclarations(library: NativeLibrary, nativeIndex: NativeIndexImpl) {
    val index = clang_createIndex(0, 0)!!
-    val indexAction = clang_IndexAction_create(index)!!
    try {
-        val translationUnit = library.parse(index).ensureNoCompileErrors()
+        val translationUnit = library.parse(index, options = CXTranslationUnit_DetailedPreprocessingRecord)
        try {
-            memScoped {
-                val callbacks = alloc<IndexerCallbacks>()
+            translationUnit.ensureNoCompileErrors()

-                val nativeIndexPtr = StableObjPtr.create(nativeIndex)
-                val clientData = nativeIndexPtr.value
+            val headers = getFilteredHeaders(library, index, translationUnit)

-                try {
-                    with(callbacks) {
-                        abortQuery = null
-                        diagnostic = null
-                        enteredMainFile = null
-                        ppIncludedFile = null
-                        importedASTFile = null
-                        startedTranslationUnit = null
-                        indexDeclaration = staticCFunction { clientData, info ->
-                            @Suppress("NAME_SHADOWING")
-                            val nativeIndex = StableObjPtr.fromValue(clientData!!).get() as NativeIndexImpl
-                            nativeIndex.indexDeclaration(info!!.pointed)
-                        }
-                        indexEntityReference = null
+            indexTranslationUnit(index, translationUnit, 0, object : Indexer {
+                override fun indexDeclaration(info: CXIdxDeclInfo) {
+                    val file = memScoped {
+                        val fileVar = alloc<CXFileVar>()
+                        clang_indexLoc_getFileLocation(info.loc.readValue(), null, fileVar.ptr, null, null, null)
+                        fileVar.value
                    }

-                    clang_indexTranslationUnit(indexAction, clientData,
-                            callbacks.ptr, IndexerCallbacks.size.toInt(),
-                            0, translationUnit)
-
-                } finally {
-                    nativeIndexPtr.dispose()
+                    if (file in headers) {
+                        nativeIndex.indexDeclaration(info)
+                    }
                }
-            }
+            })
        } finally {
            clang_disposeTranslationUnit(translationUnit)
        }
    } finally {
-        clang_IndexAction_dispose(indexAction)
        clang_disposeIndex(index)
    }
 }
@@ -17,7 +17,7 @@
 package org.jetbrains.kotlin.native.interop.indexer

 import clang.*
-import kotlinx.cinterop.CValue
+import kotlinx.cinterop.*
 import java.io.File

 /**
@@ -54,9 +54,8 @@ private fun expandMacroConstants(
    try {
        val sourceFile = library.createTempSource()
        val translationUnit = parseTranslationUnit(index, sourceFile, library.compilerArgs, options = 0)
-                .ensureNoCompileErrors()
-
        try {
+            translationUnit.ensureNoCompileErrors()
            return names.mapNotNull {
                expandMacroAsConstant(library, translationUnit, sourceFile, it, typeConverter)
            }
@@ -214,11 +213,21 @@ private fun collectMacroConstantsNames(library: NativeLibrary): List<String> {
        // Include macros into AST:
        val options = CXTranslationUnit_DetailedPreprocessingRecord

-        val translationUnit = library.parse(index, options).ensureNoCompileErrors()
+        val translationUnit = library.parse(index, options)
        try {
+            translationUnit.ensureNoCompileErrors()
+            val headers = getFilteredHeaders(library, index, translationUnit)
+
            visitChildren(translationUnit) { cursor, _ ->
+                val file = memScoped {
+                    val fileVar = alloc<CXFileVar>()
+                    clang_getFileLocation(clang_getCursorLocation(cursor), fileVar.ptr, null, null, null)
+                    fileVar.value
+                }
+
                if (cursor.kind == CXCursorKind.CXCursor_MacroDefinition &&
                        library.includesDeclaration(cursor) &&
+                        file in headers &&
                        canMacroBeConstant(cursor))
                {
                    val spelling = getCursorSpelling(cursor)
@@ -23,7 +23,9 @@ enum class Language {
 data class NativeLibrary(val includes: List<String>,
                         val compilerArgs: List<String>,
                         val language: Language,
-                         val excludeSystemLibs: Boolean = true)
+                         val excludeSystemLibs: Boolean, // TODO: drop?
+                         val excludeDepdendentModules: Boolean,
+                         val headerFilter: (String) -> Boolean)

 /**
 * Retrieves the definitions from given C header file using given compiler arguments (e.g. defines).
@@ -18,7 +18,9 @@ package org.jetbrains.kotlin.native.interop.indexer

 import clang.*
 import kotlinx.cinterop.*
+import java.io.Closeable
 import java.io.File
+import java.nio.file.Paths

 internal val CValue<CXType>.kind: CXTypeKind get() = this.useContents { kind }

@@ -198,8 +200,9 @@ internal fun NativeLibrary.precompileHeaders(): NativeLibrary {
    val index = clang_createIndex(excludeDeclarationsFromPCH = 0, displayDiagnostics = 0)!!
    try {
        val options = CXTranslationUnit_ForSerialization
-        val translationUnit = this.parse(index, options).ensureNoCompileErrors()
+        val translationUnit = this.parse(index, options)
        try {
+            translationUnit.ensureNoCompileErrors()
            clang_saveTranslationUnit(translationUnit, precompiledHeader.absolutePath, 0)
        } finally {
            clang_disposeTranslationUnit(translationUnit)
@@ -249,9 +252,8 @@ fun List<List<String>>.mapFragmentIsCompilable(originalLibrary: NativeLibrary):
    try {
        val sourceFile = library.createTempSource()
        val translationUnit = parseTranslationUnit(index, sourceFile, library.compilerArgs, options = 0)
-                .ensureNoCompileErrors()
-
        try {
+            translationUnit.ensureNoCompileErrors()
            while (fragmentsToCheck.isNotEmpty()) {
                // Combine all fragments to be checked in a single file:
                sourceFile.bufferedWriter().use { writer ->
@@ -292,4 +294,169 @@ fun List<List<String>>.mapFragmentIsCompilable(originalLibrary: NativeLibrary):
    }

    return this.indices.map { it !in indicesOfNonCompilable }
+}
+
+internal interface Indexer {
+    /**
+     * Called when a file gets #included/#imported.
+     */
+    fun ppIncludedFile(info: CXIdxIncludedFileInfo) {}
+
+    /**
+     * Called to index a declaration.
+     */
+    fun indexDeclaration(info: CXIdxDeclInfo) {}
+}
+
+internal fun indexTranslationUnit(index: CXIndex, translationUnit: CXTranslationUnit, options: Int, indexer: Indexer) {
+    val indexerStablePtr = StableObjPtr.create(indexer)
+    try {
+        val clientData = indexerStablePtr.value
+        memScoped {
+            val indexerCallbacks = alloc<IndexerCallbacks>().apply {
+                abortQuery = null
+                diagnostic = null
+                enteredMainFile = null
+                ppIncludedFile = staticCFunction { clientData, info ->
+                    @Suppress("NAME_SHADOWING")
+                    val indexer = StableObjPtr.fromValue(clientData!!).get() as Indexer
+                    indexer.ppIncludedFile(info!!.pointed)
+                    null as CXIdxClientFile?
+                }
+                importedASTFile = null
+                startedTranslationUnit = null
+                indexDeclaration = staticCFunction { clientData, info ->
+                    @Suppress("NAME_SHADOWING")
+                    val nativeIndex = StableObjPtr.fromValue(clientData!!).get() as Indexer
+                    nativeIndex.indexDeclaration(info!!.pointed)
+                }
+                indexEntityReference = null
+            }
+
+            val indexAction = clang_IndexAction_create(index)
+            try {
+                val result = clang_indexTranslationUnit(indexAction, clientData,
+                        indexerCallbacks.ptr, IndexerCallbacks.size.toInt(), options, translationUnit)
+
+                if (result != 0) {
+                    throw Error("clang_indexTranslationUnit returned $result")
+                }
+            } finally {
+                clang_IndexAction_dispose(indexAction)
+            }
+        }
+    } finally {
+        indexerStablePtr.dispose()
+    }
+}
+
+internal class ModulesMap(
+        val library: NativeLibrary,
+        val translationUnit: CXTranslationUnit
+) : Closeable {
+
+    private val index: CXIndex
+    private val translationUnitWithModules: CXTranslationUnit
+
+    init {
+        index = clang_createIndex(0, 0)!!
+        try {
+            translationUnitWithModules =
+                    library
+                            .copy(compilerArgs = library.compilerArgs + "-fmodules")
+                            .parse(index)
+
+            try {
+                translationUnitWithModules.ensureNoCompileErrors()
+            } catch (e: Throwable) {
+                clang_disposeTranslationUnit(translationUnitWithModules)
+                throw e
+            }
+
+        } catch (e: Throwable) {
+            clang_disposeIndex(index)
+            throw e
+        }
+    }
+
+    override fun close() {
+        try {
+            clang_disposeTranslationUnit(translationUnitWithModules)
+        } finally {
+            clang_disposeIndex(index)
+        }
+    }
+
+    data class Module(private val cxModule: CXModule)
+
+    fun getModule(file: CXFile): Module? {
+        // `file` is bound to `translationUnit`, however `translationUnitWithModules` is used to access modules.
+        // Find the corresponding file in `translationUnitWithModules`:
+        val fileInTuWithModules =
+                clang_getFile(translationUnitWithModules, clang_getFileName(file).convertAndDispose())!!
+
+        return clang_getModuleForFile(translationUnitWithModules, fileInTuWithModules)?.let { Module(it) }
+    }
+}
+
+internal fun getFilteredHeaders(library: NativeLibrary, index: CXIndex, translationUnit: CXTranslationUnit): Set<CXFile> {
+    val result = mutableSetOf<CXFile>()
+    val topLevelFiles = mutableListOf<CXFile>()
+
+    indexTranslationUnit(index, translationUnit, 0, object : Indexer {
+        val headerToName = mutableMapOf<CXFile, String>()
+        // The *name* of the header here is the path relative to the include path element., e.g. `curl/curl.h`.
+
+        override fun ppIncludedFile(info: CXIdxIncludedFileInfo) {
+            val includeLocation = clang_indexLoc_getCXSourceLocation(info.hashLoc.readValue())
+            val file = info.file!!
+
+            if (clang_Location_isFromMainFile(includeLocation) != 0) {
+                topLevelFiles.add(file)
+            }
+
+            val name = info.filename!!.toKString()
+            val headerName = if (info.isAngled != 0) {
+                // If the header is included with `#include <$name>`, then `name` is probably
+                // the path relative to the include path element.
+                name
+            } else {
+                // If it is included with `#include "$name"`, then `name` can also be the path relative to the includer.
+                val includerFile = memScoped {
+                    val fileVar = alloc<CXFileVar>()
+                    clang_getFileLocation(includeLocation, fileVar.ptr, null, null, null)
+                    fileVar.value!!
+                }
+                val includerName = headerToName[includerFile] ?: ""
+                val includerPath = clang_getFileName(includerFile).convertAndDispose()
+
+                if (clang_getFile(translationUnit, Paths.get(includerPath).resolveSibling(name).toString()) == file) {
+                    // included file is accessible from the includer by `name` used as relative path, so
+                    // `name` seems to be relative to the includer:
+                    Paths.get(includerName).resolveSibling(name).normalize().toString()
+                } else {
+                    name
+                }
+            }
+
+            headerToName[file] = headerName
+            if (library.headerFilter(headerName)) {
+                result.add(file)
+            }
+        }
+    })
+
+    if (library.excludeDepdendentModules) {
+        ModulesMap(library, translationUnit).use { modulesMap ->
+            val topLevelModules = topLevelFiles.map { modulesMap.getModule(it) }.toSet()
+            result.removeAll {
+                val module = modulesMap.getModule(it)
+                module !in topLevelModules
+            }
+            // Note: if some of the top-level headers don't belong to modules,
+            // then all non-modular headers are included.
+        }
+    }
+
+    return result
 }
@@ -16,17 +16,14 @@

 package org.jetbrains.kotlin.native.interop.gen.jvm

-import org.jetbrains.kotlin.native.interop.indexer.Language
-import org.jetbrains.kotlin.native.interop.indexer.NativeIndex
-import org.jetbrains.kotlin.native.interop.indexer.NativeLibrary
-import org.jetbrains.kotlin.native.interop.indexer.buildNativeIndex
+import org.jetbrains.kotlin.native.interop.indexer.*
 import java.io.File
 import java.lang.IllegalArgumentException
 import java.util.*
 import kotlin.reflect.KFunction

 fun main(args: Array<String>) {
-    val konanHome = System.getProperty("konan.home")!!
+    val konanHome = File(System.getProperty("konan.home")).absolutePath

    // TODO: remove OSX defaults.
    val substitutions = mapOf(
@@ -116,6 +113,10 @@ private fun <T> Collection<T>.atMostOne(): T? {
    }
 }

+private fun String.matchesToGlob(glob: String): Boolean =
+        java.nio.file.FileSystems.getDefault()
+                .getPathMatcher("glob:$glob").matches(java.nio.file.Paths.get(this))
+
 private fun Properties.getOsSpecific(name: String, 
    host: String = detectHost()): String? {

@@ -292,6 +293,7 @@ private fun processLib(konanHome: String,
    val compiler = "clang"
    val language = Language.C
    val excludeSystemLibs = config.getProperty("excludeSystemLibs")?.toBoolean() ?: false
+    val excludeDependentModules = config.getProperty("excludeDependentModules")?.toBoolean() ?: false

    val entryPoint = config.getSpaceSeparated("entryPoint").atMostOne()
    val linkerOpts = 
@@ -312,7 +314,25 @@ private fun processLib(konanHome: String,

    val libName = args["-cstubsname"]?.atMostOne() ?: fqParts.joinToString("") + "stubs"

-    val library = NativeLibrary(headerFiles, compilerOpts, language, excludeSystemLibs)
+    val headerFilterGlobs = config.getSpaceSeparated("headerFilter")
+
+    val headerFilter = { name: String ->
+        if (headerFilterGlobs.isEmpty()) {
+            true
+        } else {
+            headerFilterGlobs.any { name.matchesToGlob(it) }
+        }
+    }
+
+    val library = NativeLibrary(
+            includes = headerFiles,
+            compilerArgs = compilerOpts,
+            language = language,
+            excludeSystemLibs = excludeSystemLibs,
+            excludeDepdendentModules = excludeDependentModules,
+            headerFilter = headerFilter
+    )
+
    val configuration = InteropConfiguration(
            library = library,
            pkgName = outKtPkg,