From c5495e744e365efb9c03e496189792d71411f2d9 Mon Sep 17 00:00:00 2001 From: Svyatoslav Scherbina Date: Tue, 25 Apr 2017 15:42:57 +0300 Subject: [PATCH] Implement filtering interop headers by globs and by modulemaps Also do some refactoring --- INTEROP.md | 46 +++++ .../kotlin/native/interop/indexer/Indexer.kt | 41 ++--- .../native/interop/indexer/MacroConstants.kt | 17 +- .../native/interop/indexer/NativeIndex.kt | 4 +- .../kotlin/native/interop/indexer/Utils.kt | 173 +++++++++++++++++- .../kotlin/native/interop/gen/jvm/main.kt | 32 +++- 6 files changed, 271 insertions(+), 42 deletions(-) diff --git a/INTEROP.md b/INTEROP.md index 449d88162c1..5477685b12d 100644 --- a/INTEROP.md +++ b/INTEROP.md @@ -78,6 +78,52 @@ output of config script with `--cflags` flag (maybe without exact paths). Output of config script with `--libs` shall be passed as `-linkedArgs` `kotlinc` flag value (quoted) when compiling. +### Selecting library headers + +When library headers are imported to C program with `#include` directive, +all of the headers included by these headers are also included to the program. +Thus all header dependencies are included in generated stubs as well. + +This behaviour is correct but may be very inconvenient for some libraries. So +it is possible to specify in `.def` file which of the included headers are to +be imported. The separate declarations from other headers may also be imported +in case of direct dependencies. + +#### Filtering headers by globs + +It is possible to filter header by globs. The `headerFilter` property value +from the `.def` file is treated as space-separated list of globs. If the +included header matches any of the globs, then declarations from this header +are included into the bindings. + +The globs are applied to the header paths relative to the appropriate include +path elements, e.g. `time.h` or `curl/curl.h`. So if the library is usually +included with `#include `, then it would probably be +correct to filter headers with +``` +headerFilter = SomeLbrary/** +``` + +If `headerFilter` is not specified, then all headers are included. + +#### Filtering by module maps + +Some libraries have proper `module.modulemap` or `module.map` files among its +headers. For example, macOS and iOS system libraries and frameworks do. +The [module map file](https://clang.llvm.org/docs/Modules.html#module-map-language) +describes the correspondence between header files and modules. When the module +maps are available, the headers from the modules that are not included directly +can be filtered out using experimental `excludeDependentModules` option of the +`.def` file: +``` +headers = OpenGL/gl.h OpenGL/glu.h GLUT/glut.h +compilerOpts = -framework OpenGL -framework GLUT +excludeDependentModules = true +``` + +When both `excludeDependentModules` and `headerFilter` are used, they are +applied as intersection. + ## Using bindings ## ### Basic interop types ### diff --git a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt index 6286afeca2b..4b8043f8634 100644 --- a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt +++ b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt @@ -360,45 +360,30 @@ fun buildNativeIndexImpl(library: NativeLibrary): NativeIndex { private fun indexDeclarations(library: NativeLibrary, nativeIndex: NativeIndexImpl) { val index = clang_createIndex(0, 0)!! - val indexAction = clang_IndexAction_create(index)!! try { - val translationUnit = library.parse(index).ensureNoCompileErrors() + val translationUnit = library.parse(index, options = CXTranslationUnit_DetailedPreprocessingRecord) try { - memScoped { - val callbacks = alloc() + translationUnit.ensureNoCompileErrors() - val nativeIndexPtr = StableObjPtr.create(nativeIndex) - val clientData = nativeIndexPtr.value + val headers = getFilteredHeaders(library, index, translationUnit) - try { - with(callbacks) { - abortQuery = null - diagnostic = null - enteredMainFile = null - ppIncludedFile = null - importedASTFile = null - startedTranslationUnit = null - indexDeclaration = staticCFunction { clientData, info -> - @Suppress("NAME_SHADOWING") - val nativeIndex = StableObjPtr.fromValue(clientData!!).get() as NativeIndexImpl - nativeIndex.indexDeclaration(info!!.pointed) - } - indexEntityReference = null + indexTranslationUnit(index, translationUnit, 0, object : Indexer { + override fun indexDeclaration(info: CXIdxDeclInfo) { + val file = memScoped { + val fileVar = alloc() + clang_indexLoc_getFileLocation(info.loc.readValue(), null, fileVar.ptr, null, null, null) + fileVar.value } - clang_indexTranslationUnit(indexAction, clientData, - callbacks.ptr, IndexerCallbacks.size.toInt(), - 0, translationUnit) - - } finally { - nativeIndexPtr.dispose() + if (file in headers) { + nativeIndex.indexDeclaration(info) + } } - } + }) } finally { clang_disposeTranslationUnit(translationUnit) } } finally { - clang_IndexAction_dispose(indexAction) clang_disposeIndex(index) } } diff --git a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/MacroConstants.kt b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/MacroConstants.kt index 3f78bbd8991..bca8f42c865 100644 --- a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/MacroConstants.kt +++ b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/MacroConstants.kt @@ -17,7 +17,7 @@ package org.jetbrains.kotlin.native.interop.indexer import clang.* -import kotlinx.cinterop.CValue +import kotlinx.cinterop.* import java.io.File /** @@ -54,9 +54,8 @@ private fun expandMacroConstants( try { val sourceFile = library.createTempSource() val translationUnit = parseTranslationUnit(index, sourceFile, library.compilerArgs, options = 0) - .ensureNoCompileErrors() - try { + translationUnit.ensureNoCompileErrors() return names.mapNotNull { expandMacroAsConstant(library, translationUnit, sourceFile, it, typeConverter) } @@ -214,11 +213,21 @@ private fun collectMacroConstantsNames(library: NativeLibrary): List { // Include macros into AST: val options = CXTranslationUnit_DetailedPreprocessingRecord - val translationUnit = library.parse(index, options).ensureNoCompileErrors() + val translationUnit = library.parse(index, options) try { + translationUnit.ensureNoCompileErrors() + val headers = getFilteredHeaders(library, index, translationUnit) + visitChildren(translationUnit) { cursor, _ -> + val file = memScoped { + val fileVar = alloc() + clang_getFileLocation(clang_getCursorLocation(cursor), fileVar.ptr, null, null, null) + fileVar.value + } + if (cursor.kind == CXCursorKind.CXCursor_MacroDefinition && library.includesDeclaration(cursor) && + file in headers && canMacroBeConstant(cursor)) { val spelling = getCursorSpelling(cursor) diff --git a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/NativeIndex.kt b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/NativeIndex.kt index 5da08cbe509..f2728e85e1b 100644 --- a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/NativeIndex.kt +++ b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/NativeIndex.kt @@ -23,7 +23,9 @@ enum class Language { data class NativeLibrary(val includes: List, val compilerArgs: List, val language: Language, - val excludeSystemLibs: Boolean = true) + val excludeSystemLibs: Boolean, // TODO: drop? + val excludeDepdendentModules: Boolean, + val headerFilter: (String) -> Boolean) /** * Retrieves the definitions from given C header file using given compiler arguments (e.g. defines). diff --git a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Utils.kt b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Utils.kt index 2d1a368d229..3e0e477e3c1 100644 --- a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Utils.kt +++ b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Utils.kt @@ -18,7 +18,9 @@ package org.jetbrains.kotlin.native.interop.indexer import clang.* import kotlinx.cinterop.* +import java.io.Closeable import java.io.File +import java.nio.file.Paths internal val CValue.kind: CXTypeKind get() = this.useContents { kind } @@ -198,8 +200,9 @@ internal fun NativeLibrary.precompileHeaders(): NativeLibrary { val index = clang_createIndex(excludeDeclarationsFromPCH = 0, displayDiagnostics = 0)!! try { val options = CXTranslationUnit_ForSerialization - val translationUnit = this.parse(index, options).ensureNoCompileErrors() + val translationUnit = this.parse(index, options) try { + translationUnit.ensureNoCompileErrors() clang_saveTranslationUnit(translationUnit, precompiledHeader.absolutePath, 0) } finally { clang_disposeTranslationUnit(translationUnit) @@ -249,9 +252,8 @@ fun List>.mapFragmentIsCompilable(originalLibrary: NativeLibrary): try { val sourceFile = library.createTempSource() val translationUnit = parseTranslationUnit(index, sourceFile, library.compilerArgs, options = 0) - .ensureNoCompileErrors() - try { + translationUnit.ensureNoCompileErrors() while (fragmentsToCheck.isNotEmpty()) { // Combine all fragments to be checked in a single file: sourceFile.bufferedWriter().use { writer -> @@ -292,4 +294,169 @@ fun List>.mapFragmentIsCompilable(originalLibrary: NativeLibrary): } return this.indices.map { it !in indicesOfNonCompilable } +} + +internal interface Indexer { + /** + * Called when a file gets #included/#imported. + */ + fun ppIncludedFile(info: CXIdxIncludedFileInfo) {} + + /** + * Called to index a declaration. + */ + fun indexDeclaration(info: CXIdxDeclInfo) {} +} + +internal fun indexTranslationUnit(index: CXIndex, translationUnit: CXTranslationUnit, options: Int, indexer: Indexer) { + val indexerStablePtr = StableObjPtr.create(indexer) + try { + val clientData = indexerStablePtr.value + memScoped { + val indexerCallbacks = alloc().apply { + abortQuery = null + diagnostic = null + enteredMainFile = null + ppIncludedFile = staticCFunction { clientData, info -> + @Suppress("NAME_SHADOWING") + val indexer = StableObjPtr.fromValue(clientData!!).get() as Indexer + indexer.ppIncludedFile(info!!.pointed) + null as CXIdxClientFile? + } + importedASTFile = null + startedTranslationUnit = null + indexDeclaration = staticCFunction { clientData, info -> + @Suppress("NAME_SHADOWING") + val nativeIndex = StableObjPtr.fromValue(clientData!!).get() as Indexer + nativeIndex.indexDeclaration(info!!.pointed) + } + indexEntityReference = null + } + + val indexAction = clang_IndexAction_create(index) + try { + val result = clang_indexTranslationUnit(indexAction, clientData, + indexerCallbacks.ptr, IndexerCallbacks.size.toInt(), options, translationUnit) + + if (result != 0) { + throw Error("clang_indexTranslationUnit returned $result") + } + } finally { + clang_IndexAction_dispose(indexAction) + } + } + } finally { + indexerStablePtr.dispose() + } +} + +internal class ModulesMap( + val library: NativeLibrary, + val translationUnit: CXTranslationUnit +) : Closeable { + + private val index: CXIndex + private val translationUnitWithModules: CXTranslationUnit + + init { + index = clang_createIndex(0, 0)!! + try { + translationUnitWithModules = + library + .copy(compilerArgs = library.compilerArgs + "-fmodules") + .parse(index) + + try { + translationUnitWithModules.ensureNoCompileErrors() + } catch (e: Throwable) { + clang_disposeTranslationUnit(translationUnitWithModules) + throw e + } + + } catch (e: Throwable) { + clang_disposeIndex(index) + throw e + } + } + + override fun close() { + try { + clang_disposeTranslationUnit(translationUnitWithModules) + } finally { + clang_disposeIndex(index) + } + } + + data class Module(private val cxModule: CXModule) + + fun getModule(file: CXFile): Module? { + // `file` is bound to `translationUnit`, however `translationUnitWithModules` is used to access modules. + // Find the corresponding file in `translationUnitWithModules`: + val fileInTuWithModules = + clang_getFile(translationUnitWithModules, clang_getFileName(file).convertAndDispose())!! + + return clang_getModuleForFile(translationUnitWithModules, fileInTuWithModules)?.let { Module(it) } + } +} + +internal fun getFilteredHeaders(library: NativeLibrary, index: CXIndex, translationUnit: CXTranslationUnit): Set { + val result = mutableSetOf() + val topLevelFiles = mutableListOf() + + indexTranslationUnit(index, translationUnit, 0, object : Indexer { + val headerToName = mutableMapOf() + // The *name* of the header here is the path relative to the include path element., e.g. `curl/curl.h`. + + override fun ppIncludedFile(info: CXIdxIncludedFileInfo) { + val includeLocation = clang_indexLoc_getCXSourceLocation(info.hashLoc.readValue()) + val file = info.file!! + + if (clang_Location_isFromMainFile(includeLocation) != 0) { + topLevelFiles.add(file) + } + + val name = info.filename!!.toKString() + val headerName = if (info.isAngled != 0) { + // If the header is included with `#include <$name>`, then `name` is probably + // the path relative to the include path element. + name + } else { + // If it is included with `#include "$name"`, then `name` can also be the path relative to the includer. + val includerFile = memScoped { + val fileVar = alloc() + clang_getFileLocation(includeLocation, fileVar.ptr, null, null, null) + fileVar.value!! + } + val includerName = headerToName[includerFile] ?: "" + val includerPath = clang_getFileName(includerFile).convertAndDispose() + + if (clang_getFile(translationUnit, Paths.get(includerPath).resolveSibling(name).toString()) == file) { + // included file is accessible from the includer by `name` used as relative path, so + // `name` seems to be relative to the includer: + Paths.get(includerName).resolveSibling(name).normalize().toString() + } else { + name + } + } + + headerToName[file] = headerName + if (library.headerFilter(headerName)) { + result.add(file) + } + } + }) + + if (library.excludeDepdendentModules) { + ModulesMap(library, translationUnit).use { modulesMap -> + val topLevelModules = topLevelFiles.map { modulesMap.getModule(it) }.toSet() + result.removeAll { + val module = modulesMap.getModule(it) + module !in topLevelModules + } + // Note: if some of the top-level headers don't belong to modules, + // then all non-modular headers are included. + } + } + + return result } \ No newline at end of file diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt index 146a8ad7f69..97c26f44230 100644 --- a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt @@ -16,17 +16,14 @@ package org.jetbrains.kotlin.native.interop.gen.jvm -import org.jetbrains.kotlin.native.interop.indexer.Language -import org.jetbrains.kotlin.native.interop.indexer.NativeIndex -import org.jetbrains.kotlin.native.interop.indexer.NativeLibrary -import org.jetbrains.kotlin.native.interop.indexer.buildNativeIndex +import org.jetbrains.kotlin.native.interop.indexer.* import java.io.File import java.lang.IllegalArgumentException import java.util.* import kotlin.reflect.KFunction fun main(args: Array) { - val konanHome = System.getProperty("konan.home")!! + val konanHome = File(System.getProperty("konan.home")).absolutePath // TODO: remove OSX defaults. val substitutions = mapOf( @@ -116,6 +113,10 @@ private fun Collection.atMostOne(): T? { } } +private fun String.matchesToGlob(glob: String): Boolean = + java.nio.file.FileSystems.getDefault() + .getPathMatcher("glob:$glob").matches(java.nio.file.Paths.get(this)) + private fun Properties.getOsSpecific(name: String, host: String = detectHost()): String? { @@ -292,6 +293,7 @@ private fun processLib(konanHome: String, val compiler = "clang" val language = Language.C val excludeSystemLibs = config.getProperty("excludeSystemLibs")?.toBoolean() ?: false + val excludeDependentModules = config.getProperty("excludeDependentModules")?.toBoolean() ?: false val entryPoint = config.getSpaceSeparated("entryPoint").atMostOne() val linkerOpts = @@ -312,7 +314,25 @@ private fun processLib(konanHome: String, val libName = args["-cstubsname"]?.atMostOne() ?: fqParts.joinToString("") + "stubs" - val library = NativeLibrary(headerFiles, compilerOpts, language, excludeSystemLibs) + val headerFilterGlobs = config.getSpaceSeparated("headerFilter") + + val headerFilter = { name: String -> + if (headerFilterGlobs.isEmpty()) { + true + } else { + headerFilterGlobs.any { name.matchesToGlob(it) } + } + } + + val library = NativeLibrary( + includes = headerFiles, + compilerArgs = compilerOpts, + language = language, + excludeSystemLibs = excludeSystemLibs, + excludeDepdendentModules = excludeDependentModules, + headerFilter = headerFilter + ) + val configuration = InteropConfiguration( library = library, pkgName = outKtPkg,