Support importing modules in cinterop

Modules can be imported by using 'modules =' in .def file
2019-02-01 11:34:13 +03:00
parent 67b06b5f2a
commit 1f60745f4b
9 changed files with 290 additions and 72 deletions
@@ -16,14 +16,14 @@

 package org.jetbrains.kotlin.native.interop.indexer

-import java.nio.file.Paths
+import java.io.File

 class HeaderToIdMapper(sysRoot: String) {
    private val headerPathToId = mutableMapOf<String, HeaderId>()
-    private val sysRoot = Paths.get(sysRoot).normalize()
+    private val sysRoot = File(sysRoot).canonicalFile.toPath()

    internal fun getHeaderId(filePath: String) = headerPathToId.getOrPut(filePath) {
-        val path = Paths.get(filePath)
+        val path = File(filePath).canonicalFile.toPath()
        val headerIdValue = if (path.startsWith(sysRoot)) {
            val relative = sysRoot.relativize(path)
            relative.toString()
@@ -99,7 +99,7 @@ internal class NativeIndexImpl(val library: NativeLibrary, val verbose: Boolean
                all[key] = value

                val headerId = getHeaderId(getContainingFile(cursor))
-                if (!library.headerInclusionPolicy.excludeAll(headerId)) {
+                if (!library.headerExclusionPolicy.excludeAll(headerId)) {
                    // This declaration is used, and thus should be included:
                    included.add(value)
                }
@@ -917,8 +917,7 @@ fun buildNativeIndexImpl(library: NativeLibrary, verbose: Boolean): NativeIndex
 }

 private fun indexDeclarations(nativeIndex: NativeIndexImpl) {
-    val index = clang_createIndex(0, 0)!!
-    try {
+    withIndex { index ->
        val translationUnit = nativeIndex.library.parse(index, options = CXTranslationUnit_DetailedPreprocessingRecord)
        try {
            translationUnit.ensureNoCompileErrors()
@@ -959,7 +958,5 @@ private fun indexDeclarations(nativeIndex: NativeIndexImpl) {
        } finally {
            clang_disposeTranslationUnit(translationUnit)
        }
-    } finally {
-        clang_disposeIndex(index)
    }
 }
@@ -50,8 +50,7 @@ private fun expandMacros(
    // so precompile library headers to significantly speed up the parsing:
    val library = originalLibrary.precompileHeaders()

-    val index = clang_createIndex(excludeDeclarationsFromPCH = 1, displayDiagnostics = 0)!!
-    try {
+    withIndex(excludeDeclarationsFromPCH = true) { index ->
        val sourceFile = library.createTempSource()
        // We disable implicit function declaration to filter out cases when a macro is expanded as a function
        // or function-like construction (e.g. #define FOO throw()) but such a function is undeclared.
@@ -65,9 +64,6 @@ private fun expandMacros(
        } finally {
            clang_disposeTranslationUnit(translationUnit)
        }
-
-    } finally {
-        clang_disposeIndex(index)
    }
 }

@@ -0,0 +1,114 @@
+package org.jetbrains.kotlin.native.interop.indexer
+
+import clang.*
+import kotlinx.cinterop.*
+
+data class ModulesInfo(val topLevelHeaders: List<String>, val ownHeaders: Set<String>)
+
+fun getModulesInfo(compilation: Compilation, modules: List<String>): ModulesInfo {
+    if (modules.isEmpty()) return ModulesInfo(emptyList(), emptySet())
+
+    withIndex { index ->
+
+        val ownHeaders = mutableSetOf<String>()
+        val topLevelHeaders = linkedSetOf<String>()
+
+        getModulesASTFiles(index, compilation, modules).forEach {
+            val moduleTranslationUnit = clang_createTranslationUnit(index, it)!!
+            try {
+                val modulesHeaders = getModulesHeaders(index, moduleTranslationUnit, modules.toSet(), topLevelHeaders)
+                modulesHeaders.mapTo(ownHeaders) { it.canonicalPath }
+            } finally {
+                clang_disposeTranslationUnit(moduleTranslationUnit)
+            }
+        }
+
+        return ModulesInfo(topLevelHeaders.toList(), ownHeaders)
+    }
+}
+
+private fun getModulesASTFiles(index: CXIndex, compilation: Compilation, modules: List<String>): List<String> {
+    val compilationWithImports = object : Compilation by compilation {
+        override val compilerArgs = compilation.compilerArgs + "-fmodules"
+        override val additionalPreambleLines = modules.map { "@import $it;" } + compilation.additionalPreambleLines
+    }
+
+    val result = linkedSetOf<String>()
+
+    val translationUnit = compilationWithImports.parse(
+            index,
+            options = CXTranslationUnit_DetailedPreprocessingRecord
+    )
+    try {
+        translationUnit.ensureNoCompileErrors()
+
+        indexTranslationUnit(index, translationUnit, 0, object : Indexer {
+            override fun importedASTFile(info: CXIdxImportedASTFileInfo) {
+                result += info.file!!.canonicalPath
+            }
+        })
+    } finally {
+        clang_disposeTranslationUnit(translationUnit)
+    }
+    return result.toList()
+}
+
+private fun getModulesHeaders(
+        index: CXIndex,
+        translationUnit: CXTranslationUnit,
+        modules: Set<String>,
+        topLevelHeaders: LinkedHashSet<String>
+): Set<CXFile> {
+    val nonModularIncludes = mutableMapOf<CXFile, MutableSet<CXFile>>()
+    val result = mutableSetOf<CXFile>()
+
+    indexTranslationUnit(index, translationUnit, 0, object : Indexer {
+        override fun ppIncludedFile(info: CXIdxIncludedFileInfo) {
+            val file = info.file!!
+            val includer = clang_indexLoc_getCXSourceLocation(info.hashLoc.readValue()).getContainingFile()
+
+            if (includer == null) {
+                // i.e. the header is included by the module itself.
+                topLevelHeaders += file.path
+            }
+
+            val module = clang_getModuleForFile(translationUnit, file)
+
+            if (module != null) {
+                val moduleWithParents = generateSequence(module, { clang_Module_getParent(it) }).map {
+                    clang_Module_getFullName(it).convertAndDispose()
+                }
+
+                if (moduleWithParents.any { it in modules }) {
+                    result += file
+                }
+            } else if (includer != null) {
+                nonModularIncludes.getOrPut(includer, { mutableSetOf() }) += file
+            }
+        }
+    })
+
+
+    // There are cases when non-modular includes should also be considered as a part of module. For example:
+    // 1. Some module maps are broken,
+    //    e.g. system header `IOKit/hid/IOHIDProperties.h` isn't included to framework module map at all.
+    // 2. Textual headers are reported as non-modular by libclang.
+    //
+    // Find and include non-modular headers too:
+    result += findReachable(roots = result, arcs = nonModularIncludes)
+
+    return result
+}
+
+private fun <T> findReachable(roots: Set<T>, arcs: Map<T, Set<T>>): Set<T> {
+    val visited = mutableSetOf<T>()
+
+    fun dfs(vertex: T) {
+        if (!visited.add(vertex)) return
+        arcs[vertex].orEmpty().forEach { dfs(it) }
+    }
+
+    roots.forEach { dfs(it) }
+
+    return visited
+}
@@ -29,7 +29,9 @@ interface HeaderInclusionPolicy {
     * or `null` for builtin declarations.
     */
    fun excludeUnused(headerName: String?): Boolean
+}

+interface HeaderExclusionPolicy {
    /**
     * Whether all declarations from this header should be excluded.
     *
@@ -37,18 +39,34 @@ interface HeaderInclusionPolicy {
     * but not included into the root collections.
     */
    fun excludeAll(headerId: HeaderId): Boolean
-
-    // TODO: these methods should probably be combined into the only one, but it would require some refactoring.
 }

-data class NativeLibrary(val includes: List<String>,
-                         val additionalPreambleLines: List<String>,
-                         val compilerArgs: List<String>,
+sealed class NativeLibraryHeaderFilter {
+    class NameBased(
+            val policy: HeaderInclusionPolicy,
+            val excludeDepdendentModules: Boolean
+    ) : NativeLibraryHeaderFilter()
+
+    class Predefined(val headers: Set<String>) : NativeLibraryHeaderFilter()
+}
+
+interface Compilation {
+    val includes: List<String>
+    val additionalPreambleLines: List<String>
+    val compilerArgs: List<String>
+    val language: Language
+}
+
+// TODO: consider replacing inheritance by composition and turning Compilation into final class.
+
+data class NativeLibrary(override val includes: List<String>,
+                         override val additionalPreambleLines: List<String>,
+                         override val compilerArgs: List<String>,
                         val headerToIdMapper: HeaderToIdMapper,
-                         val language: Language,
+                         override val language: Language,
                         val excludeSystemLibs: Boolean, // TODO: drop?
-                         val excludeDepdendentModules: Boolean,
-                         val headerInclusionPolicy: HeaderInclusionPolicy)
+                         val headerExclusionPolicy: HeaderExclusionPolicy,
+                         val headerFilter: NativeLibraryHeaderFilter) : Compilation

 /**
 * Retrieves the definitions from given C header file using given compiler arguments (e.g. defines).
@@ -87,6 +87,23 @@ internal fun convertUnqualifiedPrimitiveType(type: CValue<CXType>): Type = when
    else -> UnsupportedType
 }

+internal inline fun <R> withIndex(
+        excludeDeclarationsFromPCH: Boolean = false,
+        displayDiagnostics: Boolean = false,
+        block: (index: CXIndex) -> R
+): R {
+    val index = clang_createIndex(
+            excludeDeclarationsFromPCH = if (excludeDeclarationsFromPCH) 1 else 0,
+            displayDiagnostics = if (displayDiagnostics) 1 else 0
+    )!!
+
+    return try {
+        block(index)
+    } finally {
+        clang_disposeIndex(index)
+    }
+}
+
 internal fun parseTranslationUnit(
        index: CXIndex,
        sourceFile: File,
@@ -107,7 +124,7 @@ internal fun parseTranslationUnit(
    }
 }

-internal fun NativeLibrary.parse(index: CXIndex, options: Int = 0): CXTranslationUnit =
+internal fun Compilation.parse(index: CXIndex, options: Int = 0): CXTranslationUnit =
        parseTranslationUnit(index, this.createTempSource(), this.compilerArgs, options)

 internal data class Diagnostic(val severity: CXDiagnosticSeverity, val format: String,
@@ -218,11 +235,11 @@ internal fun List<String>.toNativeStringArray(scope: AutofreeScope): CArrayPoint
    }
 }

-val NativeLibrary.preambleLines: List<String>
+val Compilation.preambleLines: List<String>
    get() = this.includes.map { "#include <$it>" } + this.additionalPreambleLines

-internal fun Appendable.appendPreamble(library: NativeLibrary) = this.apply {
-    library.preambleLines.forEach {
+internal fun Appendable.appendPreamble(compilation: Compilation) = this.apply {
+    compilation.preambleLines.forEach {
        this.appendln(it)
    }
 }
@@ -230,7 +247,7 @@ internal fun Appendable.appendPreamble(library: NativeLibrary) = this.apply {
 /**
 * Creates temporary source file which includes the library.
 */
-internal fun NativeLibrary.createTempSource(): File {
+internal fun Compilation.createTempSource(): File {
    val result = createTempFile(suffix = ".${language.sourceFileExtension}")
    result.deleteOnExit()

@@ -249,8 +266,7 @@ internal fun NativeLibrary.createTempSource(): File {
 internal fun NativeLibrary.precompileHeaders(): NativeLibrary {
    val precompiledHeader = createTempFile(suffix = ".pch").apply { this.deleteOnExit() }

-    val index = clang_createIndex(excludeDeclarationsFromPCH = 0, displayDiagnostics = 0)!!
-    try {
+    withIndex { index ->
        val options = CXTranslationUnit_ForSerialization
        val translationUnit = this.parse(index, options)
        try {
@@ -259,8 +275,6 @@ internal fun NativeLibrary.precompileHeaders(): NativeLibrary {
        } finally {
            clang_disposeTranslationUnit(translationUnit)
        }
-    } finally {
-        clang_disposeIndex(index)
    }

    return this.copy(
@@ -301,8 +315,7 @@ fun List<List<String>>.mapFragmentIsCompilable(originalLibrary: NativeLibrary):

    val fragmentsToCheck = this.withIndex().toMutableList()

-    val index = clang_createIndex(excludeDeclarationsFromPCH = 1, displayDiagnostics = 0)!!
-    try {
+    withIndex(excludeDeclarationsFromPCH = true) { index ->
        val sourceFile = library.createTempSource()
        val translationUnit = parseTranslationUnit(index, sourceFile, library.compilerArgs, options = 0)
        try {
@@ -342,8 +355,6 @@ fun List<List<String>>.mapFragmentIsCompilable(originalLibrary: NativeLibrary):
        } finally {
            clang_disposeTranslationUnit(translationUnit)
        }
-    } finally {
-        clang_disposeIndex(index)
    }

    return this.indices.map { it !in indicesOfNonCompilable }
@@ -360,6 +371,11 @@ internal interface Indexer {
     */
    fun ppIncludedFile(info: CXIdxIncludedFileInfo) {}

+    /**
+     * Called when a AST file (PCH or module) gets imported.
+     */
+    fun importedASTFile(info: CXIdxImportedASTFileInfo) {}
+
    /**
     * Called to index a declaration.
     */
@@ -390,7 +406,14 @@ internal fun indexTranslationUnit(index: CXIndex, translationUnit: CXTranslation
                    @Suppress("USELESS_CAST")
                    null as CXIdxClientFile?
                }
-                importedASTFile = null
+                importedASTFile = staticCFunction { clientData, info ->
+                    @Suppress("NAME_SHADOWING")
+                    val indexer = clientData!!.asStableRef<Indexer>().get()
+                    indexer.importedASTFile(info!!.pointed)
+                    // We must ensure only interop types exist in function signature.
+                    @Suppress("USELESS_CAST")
+                    null as CXIdxClientFile?
+                }
                startedTranslationUnit = null
                indexDeclaration = staticCFunction { clientData, info ->
                    @Suppress("NAME_SHADOWING")
@@ -418,7 +441,7 @@ internal fun indexTranslationUnit(index: CXIndex, translationUnit: CXTranslation
 }

 internal class ModulesMap(
-        val library: NativeLibrary,
+        val compilation: Compilation,
        val translationUnit: CXTranslationUnit
 ) : Closeable {

@@ -428,10 +451,9 @@ internal class ModulesMap(
    init {
        index = clang_createIndex(0, 0)!!
        try {
-            translationUnitWithModules =
-                    library
-                            .copy(compilerArgs = library.compilerArgs + "-fmodules")
-                            .parse(index)
+            translationUnitWithModules = object : Compilation by compilation {
+                override val compilerArgs = compilation.compilerArgs + "-fmodules"
+            }.parse(index)

            try {
                translationUnitWithModules.ensureNoCompileErrors()
@@ -466,15 +488,12 @@ internal class ModulesMap(
    }
 }

-fun HeaderInclusionPolicy.includeAll(headerName: String?, headerId: HeaderId): Boolean =
-        !this.excludeUnused(headerName) && !this.excludeAll(headerId)
-
 internal fun getHeaderId(library: NativeLibrary, header: CXFile?): HeaderId {
    if (header == null) {
        return HeaderId("builtins")
    }

-    val filePath = clang_getFileName(header).convertAndDispose()
+    val filePath = header.path
    return library.headerToIdMapper.getHeaderId(filePath)
 }

@@ -493,6 +512,30 @@ internal fun getHeaders(
 ): NativeLibraryHeaders<CXFile?> {
    val ownHeaders = mutableSetOf<CXFile?>()
    val allHeaders = mutableSetOf<CXFile?>(null)
+
+    val filter = library.headerFilter
+
+    when (filter) {
+        is NativeLibraryHeaderFilter.NameBased ->
+            filterHeadersByName(library, filter, index, translationUnit, ownHeaders, allHeaders)
+
+        is NativeLibraryHeaderFilter.Predefined ->
+            filterHeadersByPredefined(filter, index, translationUnit, ownHeaders, allHeaders)
+    }
+
+    ownHeaders.removeAll { library.headerExclusionPolicy.excludeAll(getHeaderId(library, it)) }
+
+    return NativeLibraryHeaders(ownHeaders, allHeaders - ownHeaders)
+}
+
+private fun filterHeadersByName(
+        compilation: Compilation,
+        filter: NativeLibraryHeaderFilter.NameBased,
+        index: CXIndex,
+        translationUnit: CXTranslationUnit,
+        ownHeaders: MutableSet<CXFile?>,
+        allHeaders: MutableSet<CXFile?>
+) {
    val topLevelFiles = mutableListOf<CXFile>()
    var mainFile: CXFile? = null

@@ -524,7 +567,7 @@ internal fun getHeaders(
                // If it is included with `#include "$name"`, then `name` can also be the path relative to the includer.
                val includerFile = includeLocation.getContainingFile()!!
                val includerName = headerToName[includerFile] ?: ""
-                val includerPath = clang_getFileName(includerFile).convertAndDispose()
+                val includerPath = includerFile.path

                if (clang_getFile(translationUnit, Paths.get(includerPath).resolveSibling(name).toString()) == file) {
                    // included file is accessible from the includer by `name` used as relative path, so
@@ -536,14 +579,14 @@ internal fun getHeaders(
            }

            headerToName[file] = headerName
-            if (library.headerInclusionPolicy.includeAll(headerName, getHeaderId(library, file))) {
+            if (!filter.policy.excludeUnused(headerName)) {
                ownHeaders.add(file)
            }
        }
    })

-    if (library.excludeDepdendentModules) {
-        ModulesMap(library, translationUnit).use { modulesMap ->
+    if (filter.excludeDepdendentModules) {
+        ModulesMap(compilation, translationUnit).use { modulesMap ->
            val topLevelModules = topLevelFiles.map { modulesMap.getModule(it) }.toSet()
            ownHeaders.removeAll {
                val module = modulesMap.getModule(it!!)
@@ -553,25 +596,41 @@ internal fun getHeaders(
            // then all non-modular headers are included.
        }
    } else {
-        if (library.headerInclusionPolicy.includeAll(headerName = null, headerId = getHeaderId(library, null))) {
+        if (!filter.policy.excludeUnused(headerName = null)) {
            // Builtins.
            ownHeaders.add(null)
        }
    }

    ownHeaders.add(mainFile!!)
+}

-    return NativeLibraryHeaders(ownHeaders, allHeaders - ownHeaders)
+private fun filterHeadersByPredefined(
+        filter: NativeLibraryHeaderFilter.Predefined,
+        index: CXIndex,
+        translationUnit: CXTranslationUnit,
+        ownHeaders: MutableSet<CXFile?>,
+        allHeaders: MutableSet<CXFile?>
+) {
+    // Note: suboptimal but simple.
+    indexTranslationUnit(index, translationUnit, 0, object : Indexer {
+        override fun ppIncludedFile(info: CXIdxIncludedFileInfo) {
+            val file = info.file
+            allHeaders += file
+            if (file?.canonicalPath in filter.headers) {
+                ownHeaders += file
+            }
+        }
+    })
 }

 fun NativeLibrary.getHeaderPaths(): NativeLibraryHeaders<String> {
-    val index = clang_createIndex(0, 0)!!
-    try {
+    withIndex { index ->
        val translationUnit =
                this.parse(index, options = CXTranslationUnit_DetailedPreprocessingRecord).ensureNoCompileErrors()
        try {

-            fun getPath(file: CXFile?) = if (file == null) "<builtins>" else clang_getFileName(file).convertAndDispose()
+            fun getPath(file: CXFile?) = if (file == null) "<builtins>" else file.canonicalPath

            val headers = getHeaders(this, index, translationUnit)
            return NativeLibraryHeaders(
@@ -581,8 +640,6 @@ fun NativeLibrary.getHeaderPaths(): NativeLibraryHeaders<String> {
        } finally {
            clang_disposeTranslationUnit(translationUnit)
        }
-    } finally {
-        clang_disposeIndex(index)
    }
 }

@@ -618,6 +675,9 @@ internal fun getContainingFile(cursor: CValue<CXCursor>): CXFile? {
    return clang_getCursorLocation(cursor).getContainingFile()
 }

+internal val CXFile.path: String get() = clang_getFileName(this).convertAndDispose()
+internal val CXFile.canonicalPath: String get() = File(this.path).canonicalPath
+
 private fun createVfsOverlayFileContents(virtualPathToReal: Map<Path, Path>): ByteArray {
    val overlay = clang_VirtualFileOverlay_create(0)

@@ -16,9 +16,7 @@

 package org.jetbrains.kotlin.native.interop.gen

-import org.jetbrains.kotlin.native.interop.indexer.HeaderId
-import org.jetbrains.kotlin.native.interop.indexer.HeaderInclusionPolicy
-import org.jetbrains.kotlin.native.interop.indexer.Location
+import org.jetbrains.kotlin.native.interop.indexer.*

 interface Imports {
    fun getPackage(location: Location): String?
@@ -31,10 +29,7 @@ class ImportsImpl(internal val headerIdToPackage: Map<HeaderId, String>) : Impor

 }

-class HeadersInclusionPolicyImpl(
-        private val nameGlobs: List<String>,
-        private val importsImpl: ImportsImpl
-) : HeaderInclusionPolicy {
+class HeaderInclusionPolicyImpl(private val nameGlobs: List<String>) : HeaderInclusionPolicy {

    override fun excludeUnused(headerName: String?): Boolean {
        if (nameGlobs.isEmpty()) {
@@ -48,6 +43,11 @@ class HeadersInclusionPolicyImpl(

        return nameGlobs.all { !headerName.matchesToGlob(it) }
    }
+}
+
+class HeaderExclusionPolicyImpl(
+        private val importsImpl: ImportsImpl
+) : HeaderExclusionPolicy {

    override fun excludeAll(headerId: HeaderId): Boolean {
        return headerId in importsImpl.headerIdToPackage
@@ -337,20 +337,49 @@ internal fun buildNativeLibrary(
        })
    }

-    val excludeSystemLibs = def.config.excludeSystemLibs
-    val excludeDependentModules = def.config.excludeDependentModules
+    val compilation = object : Compilation {
+        override val includes = headerFiles
+        override val additionalPreambleLines = def.defHeaderLines
+        override val compilerArgs = compilerOpts + tool.platformCompilerOpts
+        override val language = language
+    }

-    val headerFilterGlobs = def.config.headerFilter
-    val headerInclusionPolicy = HeadersInclusionPolicyImpl(headerFilterGlobs, imports)
+    val headerFilter: NativeLibraryHeaderFilter
+    val includes: List<String>
+
+    val modules = def.config.modules
+
+    if (modules.isEmpty()) {
+        val excludeDependentModules = def.config.excludeDependentModules
+
+        val headerFilterGlobs = def.config.headerFilter
+        val headerInclusionPolicy = HeaderInclusionPolicyImpl(headerFilterGlobs)
+
+        headerFilter = NativeLibraryHeaderFilter.NameBased(headerInclusionPolicy, excludeDependentModules)
+        includes = headerFiles
+    } else {
+        require(language == Language.OBJECTIVE_C) { "cinterop supports 'modules' only when 'language = Objective-C'" }
+        require(headerFiles.isEmpty()) { "cinterop doesn't support having headers and modules specified at the same time" }
+        require(def.config.headerFilter.isEmpty()) { "cinterop doesn't support 'headerFilter' with 'modules'" }
+
+        val modulesInfo = getModulesInfo(compilation, modules)
+
+        headerFilter = NativeLibraryHeaderFilter.Predefined(modulesInfo.ownHeaders)
+        includes = modulesInfo.topLevelHeaders
+    }
+
+    val excludeSystemLibs = def.config.excludeSystemLibs
+
+    val headerExclusionPolicy = HeaderExclusionPolicyImpl(imports)

    return NativeLibrary(
-            includes = headerFiles,
-            additionalPreambleLines = def.defHeaderLines,
-            compilerArgs = compilerOpts + tool.platformCompilerOpts,
+            includes = includes,
+            additionalPreambleLines = compilation.additionalPreambleLines,
+            compilerArgs = compilation.compilerArgs,
            headerToIdMapper = HeaderToIdMapper(sysRoot = tool.sysRoot),
-            language = language,
+            language = compilation.language,
            excludeSystemLibs = excludeSystemLibs,
-            excludeDepdendentModules = excludeDependentModules,
-            headerInclusionPolicy = headerInclusionPolicy
+            headerExclusionPolicy = headerExclusionPolicy,
+            headerFilter = headerFilter
    )
 }