azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support importing modules in cinterop

Browse Source 
Modules can be imported by using 'modules =' in .def file
This commit is contained in:
Svyatoslav Scherbina
2019-02-01 11:34:13 +03:00
committed by SvyatoslavScherbina
parent 67b06b5f2a
commit 1f60745f4b
9 changed files with 290 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/HeaderToIdMapper.kt
+3 -3
View File
@@ -16,14 +16,14 @@
package org.jetbrains.kotlin.native.interop.indexer
import java.nio.file.Paths
import java.io.File
class HeaderToIdMapper(sysRoot: String) {
private val headerPathToId = mutableMapOf<String, HeaderId>()
private val sysRoot = Paths.get(sysRoot).normalize()
private val sysRoot = File(sysRoot).canonicalFile.toPath()
internal fun getHeaderId(filePath: String) = headerPathToId.getOrPut(filePath) {
val path = Paths.get(filePath)
val path = File(filePath).canonicalFile.toPath()
val headerIdValue = if (path.startsWith(sysRoot)) {
val relative = sysRoot.relativize(path)
relative.toString()
Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt
+2 -5
View File
@@ -99,7 +99,7 @@ internal class NativeIndexImpl(val library: NativeLibrary, val verbose: Boolean
all[key] = value
val headerId = getHeaderId(getContainingFile(cursor))
if (!library.headerInclusionPolicy.excludeAll(headerId)) {
if (!library.headerExclusionPolicy.excludeAll(headerId)) {
// This declaration is used, and thus should be included:
included.add(value)
}
@@ -917,8 +917,7 @@ fun buildNativeIndexImpl(library: NativeLibrary, verbose: Boolean): NativeIndex
}
private fun indexDeclarations(nativeIndex: NativeIndexImpl) {
val index = clang_createIndex(0, 0)!!
try {
withIndex { index ->
val translationUnit = nativeIndex.library.parse(index, options = CXTranslationUnit_DetailedPreprocessingRecord)
try {
translationUnit.ensureNoCompileErrors()
@@ -959,7 +958,5 @@ private fun indexDeclarations(nativeIndex: NativeIndexImpl) {
} finally {
clang_disposeTranslationUnit(translationUnit)
}
} finally {
clang_disposeIndex(index)
}
}
Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/MacroConstants.kt
+1 -5
View File
@@ -50,8 +50,7 @@ private fun expandMacros(
// so precompile library headers to significantly speed up the parsing:
val library = originalLibrary.precompileHeaders()
val index = clang_createIndex(excludeDeclarationsFromPCH = 1, displayDiagnostics = 0)!!
try {
withIndex(excludeDeclarationsFromPCH = true) { index ->
val sourceFile = library.createTempSource()
// We disable implicit function declaration to filter out cases when a macro is expanded as a function
// or function-like construction (e.g. #define FOO throw()) but such a function is undeclared.
@@ -65,9 +64,6 @@ private fun expandMacros(
} finally {
clang_disposeTranslationUnit(translationUnit)
}
} finally {
clang_disposeIndex(index)
}
}
Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/ModuleSupport.kt
+114
View File
@@ -0,0 +1,114 @@
package org.jetbrains.kotlin.native.interop.indexer
import clang.*
import kotlinx.cinterop.*
data class ModulesInfo(val topLevelHeaders: List<String>, val ownHeaders: Set<String>)
fun getModulesInfo(compilation: Compilation, modules: List<String>): ModulesInfo {
if (modules.isEmpty()) return ModulesInfo(emptyList(), emptySet())
withIndex { index ->
val ownHeaders = mutableSetOf<String>()
val topLevelHeaders = linkedSetOf<String>()
getModulesASTFiles(index, compilation, modules).forEach {
val moduleTranslationUnit = clang_createTranslationUnit(index, it)!!
try {
val modulesHeaders = getModulesHeaders(index, moduleTranslationUnit, modules.toSet(), topLevelHeaders)
modulesHeaders.mapTo(ownHeaders) { it.canonicalPath }
} finally {
clang_disposeTranslationUnit(moduleTranslationUnit)
}
}
return ModulesInfo(topLevelHeaders.toList(), ownHeaders)
}
}
private fun getModulesASTFiles(index: CXIndex, compilation: Compilation, modules: List<String>): List<String> {
val compilationWithImports = object : Compilation by compilation {
override val compilerArgs = compilation.compilerArgs + "-fmodules"
override val additionalPreambleLines = modules.map { "@import $it;" } + compilation.additionalPreambleLines
}
val result = linkedSetOf<String>()
val translationUnit = compilationWithImports.parse(
index,
options = CXTranslationUnit_DetailedPreprocessingRecord
)
try {
translationUnit.ensureNoCompileErrors()
indexTranslationUnit(index, translationUnit, 0, object : Indexer {
override fun importedASTFile(info: CXIdxImportedASTFileInfo) {
result += info.file!!.canonicalPath
}
})
} finally {
clang_disposeTranslationUnit(translationUnit)
}
return result.toList()
}
private fun getModulesHeaders(
index: CXIndex,
translationUnit: CXTranslationUnit,
modules: Set<String>,
topLevelHeaders: LinkedHashSet<String>
): Set<CXFile> {
val nonModularIncludes = mutableMapOf<CXFile, MutableSet<CXFile>>()
val result = mutableSetOf<CXFile>()
indexTranslationUnit(index, translationUnit, 0, object : Indexer {
override fun ppIncludedFile(info: CXIdxIncludedFileInfo) {
val file = info.file!!
val includer = clang_indexLoc_getCXSourceLocation(info.hashLoc.readValue()).getContainingFile()
if (includer == null) {
// i.e. the header is included by the module itself.
topLevelHeaders += file.path
}
val module = clang_getModuleForFile(translationUnit, file)
if (module != null) {
val moduleWithParents = generateSequence(module, { clang_Module_getParent(it) }).map {
clang_Module_getFullName(it).convertAndDispose()
}
if (moduleWithParents.any { it in modules }) {
result += file
}
} else if (includer != null) {
nonModularIncludes.getOrPut(includer, { mutableSetOf() }) += file
}
}
})
// There are cases when non-modular includes should also be considered as a part of module. For example:
// 1. Some module maps are broken,
// e.g. system header `IOKit/hid/IOHIDProperties.h` isn't included to framework module map at all.
// 2. Textual headers are reported as non-modular by libclang.
//
// Find and include non-modular headers too:
result += findReachable(roots = result, arcs = nonModularIncludes)
return result
}
private fun <T> findReachable(roots: Set<T>, arcs: Map<T, Set<T>>): Set<T> {
val visited = mutableSetOf<T>()
fun dfs(vertex: T) {
if (!visited.add(vertex)) return
arcs[vertex].orEmpty().forEach { dfs(it) }
}
roots.forEach { dfs(it) }
return visited
}
Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/NativeIndex.kt
+26 -8
View File
@@ -29,7 +29,9 @@ interface HeaderInclusionPolicy {
* or `null` for builtin declarations.
*/
fun excludeUnused(headerName: String?): Boolean
}
interface HeaderExclusionPolicy {
/**
* Whether all declarations from this header should be excluded.
*
@@ -37,18 +39,34 @@ interface HeaderInclusionPolicy {
* but not included into the root collections.
*/
fun excludeAll(headerId: HeaderId): Boolean
// TODO: these methods should probably be combined into the only one, but it would require some refactoring.
}
data class NativeLibrary(val includes: List<String>,
val additionalPreambleLines: List<String>,
val compilerArgs: List<String>,
sealed class NativeLibraryHeaderFilter {
class NameBased(
val policy: HeaderInclusionPolicy,
val excludeDepdendentModules: Boolean
) : NativeLibraryHeaderFilter()
class Predefined(val headers: Set<String>) : NativeLibraryHeaderFilter()
}
interface Compilation {
val includes: List<String>
val additionalPreambleLines: List<String>
val compilerArgs: List<String>
val language: Language
}
// TODO: consider replacing inheritance by composition and turning Compilation into final class.
data class NativeLibrary(override val includes: List<String>,
override val additionalPreambleLines: List<String>,
override val compilerArgs: List<String>,
val headerToIdMapper: HeaderToIdMapper,
val language: Language,
override val language: Language,
val excludeSystemLibs: Boolean, // TODO: drop?
val excludeDepdendentModules: Boolean,
val headerInclusionPolicy: HeaderInclusionPolicy)
val headerExclusionPolicy: HeaderExclusionPolicy,
val headerFilter: NativeLibraryHeaderFilter) : Compilation
/**
* Retrieves the definitions from given C header file using given compiler arguments (e.g. defines).
Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Utils.kt
+94 -34
View File
@@ -87,6 +87,23 @@ internal fun convertUnqualifiedPrimitiveType(type: CValue<CXType>): Type = when
else -> UnsupportedType
}
internal inline fun <R> withIndex(
excludeDeclarationsFromPCH: Boolean = false,
displayDiagnostics: Boolean = false,
block: (index: CXIndex) -> R
): R {
val index = clang_createIndex(
excludeDeclarationsFromPCH = if (excludeDeclarationsFromPCH) 1 else 0,
displayDiagnostics = if (displayDiagnostics) 1 else 0
)!!
return try {
block(index)
} finally {
clang_disposeIndex(index)
}
}
internal fun parseTranslationUnit(
index: CXIndex,
sourceFile: File,
@@ -107,7 +124,7 @@ internal fun parseTranslationUnit(
}
}
internal fun NativeLibrary.parse(index: CXIndex, options: Int = 0): CXTranslationUnit =
internal fun Compilation.parse(index: CXIndex, options: Int = 0): CXTranslationUnit =
parseTranslationUnit(index, this.createTempSource(), this.compilerArgs, options)
internal data class Diagnostic(val severity: CXDiagnosticSeverity, val format: String,
@@ -218,11 +235,11 @@ internal fun List<String>.toNativeStringArray(scope: AutofreeScope): CArrayPoint
}
}
val NativeLibrary.preambleLines: List<String>
val Compilation.preambleLines: List<String>
get() = this.includes.map { "#include <$it>" } + this.additionalPreambleLines
internal fun Appendable.appendPreamble(library: NativeLibrary) = this.apply {
library.preambleLines.forEach {
internal fun Appendable.appendPreamble(compilation: Compilation) = this.apply {
compilation.preambleLines.forEach {
this.appendln(it)
}
}
@@ -230,7 +247,7 @@ internal fun Appendable.appendPreamble(library: NativeLibrary) = this.apply {
/**
* Creates temporary source file which includes the library.
*/
internal fun NativeLibrary.createTempSource(): File {
internal fun Compilation.createTempSource(): File {
val result = createTempFile(suffix = ".${language.sourceFileExtension}")
result.deleteOnExit()
@@ -249,8 +266,7 @@ internal fun NativeLibrary.createTempSource(): File {
internal fun NativeLibrary.precompileHeaders(): NativeLibrary {
val precompiledHeader = createTempFile(suffix = ".pch").apply { this.deleteOnExit() }
val index = clang_createIndex(excludeDeclarationsFromPCH = 0, displayDiagnostics = 0)!!
try {
withIndex { index ->
val options = CXTranslationUnit_ForSerialization
val translationUnit = this.parse(index, options)
try {
@@ -259,8 +275,6 @@ internal fun NativeLibrary.precompileHeaders(): NativeLibrary {
} finally {
clang_disposeTranslationUnit(translationUnit)
}
} finally {
clang_disposeIndex(index)
}
return this.copy(
@@ -301,8 +315,7 @@ fun List<List<String>>.mapFragmentIsCompilable(originalLibrary: NativeLibrary):
val fragmentsToCheck = this.withIndex().toMutableList()
val index = clang_createIndex(excludeDeclarationsFromPCH = 1, displayDiagnostics = 0)!!
try {
withIndex(excludeDeclarationsFromPCH = true) { index ->
val sourceFile = library.createTempSource()
val translationUnit = parseTranslationUnit(index, sourceFile, library.compilerArgs, options = 0)
try {
@@ -342,8 +355,6 @@ fun List<List<String>>.mapFragmentIsCompilable(originalLibrary: NativeLibrary):
} finally {
clang_disposeTranslationUnit(translationUnit)
}
} finally {
clang_disposeIndex(index)
}
return this.indices.map { it !in indicesOfNonCompilable }
@@ -360,6 +371,11 @@ internal interface Indexer {
*/
fun ppIncludedFile(info: CXIdxIncludedFileInfo) {}
/**
* Called when a AST file (PCH or module) gets imported.
*/
fun importedASTFile(info: CXIdxImportedASTFileInfo) {}
/**
* Called to index a declaration.
*/
@@ -390,7 +406,14 @@ internal fun indexTranslationUnit(index: CXIndex, translationUnit: CXTranslation
@Suppress("USELESS_CAST")
null as CXIdxClientFile?
}
importedASTFile = null
importedASTFile = staticCFunction { clientData, info ->
@Suppress("NAME_SHADOWING")
val indexer = clientData!!.asStableRef<Indexer>().get()
indexer.importedASTFile(info!!.pointed)
// We must ensure only interop types exist in function signature.
@Suppress("USELESS_CAST")
null as CXIdxClientFile?
}
startedTranslationUnit = null
indexDeclaration = staticCFunction { clientData, info ->
@Suppress("NAME_SHADOWING")
@@ -418,7 +441,7 @@ internal fun indexTranslationUnit(index: CXIndex, translationUnit: CXTranslation
}
internal class ModulesMap(
val library: NativeLibrary,
val compilation: Compilation,
val translationUnit: CXTranslationUnit
) : Closeable {
@@ -428,10 +451,9 @@ internal class ModulesMap(
init {
index = clang_createIndex(0, 0)!!
try {
translationUnitWithModules =
library
.copy(compilerArgs = library.compilerArgs + "-fmodules")
.parse(index)
translationUnitWithModules = object : Compilation by compilation {
override val compilerArgs = compilation.compilerArgs + "-fmodules"
}.parse(index)
try {
translationUnitWithModules.ensureNoCompileErrors()
@@ -466,15 +488,12 @@ internal class ModulesMap(
}
}
fun HeaderInclusionPolicy.includeAll(headerName: String?, headerId: HeaderId): Boolean =
!this.excludeUnused(headerName) && !this.excludeAll(headerId)
internal fun getHeaderId(library: NativeLibrary, header: CXFile?): HeaderId {
if (header == null) {
return HeaderId("builtins")
}
val filePath = clang_getFileName(header).convertAndDispose()
val filePath = header.path
return library.headerToIdMapper.getHeaderId(filePath)
}
@@ -493,6 +512,30 @@ internal fun getHeaders(
): NativeLibraryHeaders<CXFile?> {
val ownHeaders = mutableSetOf<CXFile?>()
val allHeaders = mutableSetOf<CXFile?>(null)
val filter = library.headerFilter
when (filter) {
is NativeLibraryHeaderFilter.NameBased ->
filterHeadersByName(library, filter, index, translationUnit, ownHeaders, allHeaders)
is NativeLibraryHeaderFilter.Predefined ->
filterHeadersByPredefined(filter, index, translationUnit, ownHeaders, allHeaders)
}
ownHeaders.removeAll { library.headerExclusionPolicy.excludeAll(getHeaderId(library, it)) }
return NativeLibraryHeaders(ownHeaders, allHeaders - ownHeaders)
}
private fun filterHeadersByName(
compilation: Compilation,
filter: NativeLibraryHeaderFilter.NameBased,
index: CXIndex,
translationUnit: CXTranslationUnit,
ownHeaders: MutableSet<CXFile?>,
allHeaders: MutableSet<CXFile?>
) {
val topLevelFiles = mutableListOf<CXFile>()
var mainFile: CXFile? = null
@@ -524,7 +567,7 @@ internal fun getHeaders(
// If it is included with `#include "$name"`, then `name` can also be the path relative to the includer.
val includerFile = includeLocation.getContainingFile()!!
val includerName = headerToName[includerFile] ?: ""
val includerPath = clang_getFileName(includerFile).convertAndDispose()
val includerPath = includerFile.path
if (clang_getFile(translationUnit, Paths.get(includerPath).resolveSibling(name).toString()) == file) {
// included file is accessible from the includer by `name` used as relative path, so
@@ -536,14 +579,14 @@ internal fun getHeaders(
}
headerToName[file] = headerName
if (library.headerInclusionPolicy.includeAll(headerName, getHeaderId(library, file))) {
if (!filter.policy.excludeUnused(headerName)) {
ownHeaders.add(file)
}
}
})
if (library.excludeDepdendentModules) {
ModulesMap(library, translationUnit).use { modulesMap ->
if (filter.excludeDepdendentModules) {
ModulesMap(compilation, translationUnit).use { modulesMap ->
val topLevelModules = topLevelFiles.map { modulesMap.getModule(it) }.toSet()
ownHeaders.removeAll {
val module = modulesMap.getModule(it!!)
@@ -553,25 +596,41 @@ internal fun getHeaders(
// then all non-modular headers are included.
}
} else {
if (library.headerInclusionPolicy.includeAll(headerName = null, headerId = getHeaderId(library, null))) {
if (!filter.policy.excludeUnused(headerName = null)) {
// Builtins.
ownHeaders.add(null)
}
}
ownHeaders.add(mainFile!!)
}
return NativeLibraryHeaders(ownHeaders, allHeaders - ownHeaders)
private fun filterHeadersByPredefined(
filter: NativeLibraryHeaderFilter.Predefined,
index: CXIndex,
translationUnit: CXTranslationUnit,
ownHeaders: MutableSet<CXFile?>,
allHeaders: MutableSet<CXFile?>
) {
// Note: suboptimal but simple.
indexTranslationUnit(index, translationUnit, 0, object : Indexer {
override fun ppIncludedFile(info: CXIdxIncludedFileInfo) {
val file = info.file
allHeaders += file
if (file?.canonicalPath in filter.headers) {
ownHeaders += file
}
}
})
}
fun NativeLibrary.getHeaderPaths(): NativeLibraryHeaders<String> {
val index = clang_createIndex(0, 0)!!
try {
withIndex { index ->
val translationUnit =
this.parse(index, options = CXTranslationUnit_DetailedPreprocessingRecord).ensureNoCompileErrors()
try {
fun getPath(file: CXFile?) = if (file == null) "<builtins>" else clang_getFileName(file).convertAndDispose()
fun getPath(file: CXFile?) = if (file == null) "<builtins>" else file.canonicalPath
val headers = getHeaders(this, index, translationUnit)
return NativeLibraryHeaders(
@@ -581,8 +640,6 @@ fun NativeLibrary.getHeaderPaths(): NativeLibraryHeaders<String> {
} finally {
clang_disposeTranslationUnit(translationUnit)
}
} finally {
clang_disposeIndex(index)
}
}
@@ -618,6 +675,9 @@ internal fun getContainingFile(cursor: CValue<CXCursor>): CXFile? {
return clang_getCursorLocation(cursor).getContainingFile()
}
internal val CXFile.path: String get() = clang_getFileName(this).convertAndDispose()
internal val CXFile.canonicalPath: String get() = File(this.path).canonicalPath
private fun createVfsOverlayFileContents(virtualPathToReal: Map<Path, Path>): ByteArray {
val overlay = clang_VirtualFileOverlay_create(0)
Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/Imports.kt
+7 -7
View File
@@ -16,9 +16,7 @@
package org.jetbrains.kotlin.native.interop.gen
import org.jetbrains.kotlin.native.interop.indexer.HeaderId
import org.jetbrains.kotlin.native.interop.indexer.HeaderInclusionPolicy
import org.jetbrains.kotlin.native.interop.indexer.Location
import org.jetbrains.kotlin.native.interop.indexer.*
interface Imports {
fun getPackage(location: Location): String?
@@ -31,10 +29,7 @@ class ImportsImpl(internal val headerIdToPackage: Map<HeaderId, String>) : Impor
}
class HeadersInclusionPolicyImpl(
private val nameGlobs: List<String>,
private val importsImpl: ImportsImpl
) : HeaderInclusionPolicy {
class HeaderInclusionPolicyImpl(private val nameGlobs: List<String>) : HeaderInclusionPolicy {
override fun excludeUnused(headerName: String?): Boolean {
if (nameGlobs.isEmpty()) {
@@ -48,6 +43,11 @@ class HeadersInclusionPolicyImpl(
return nameGlobs.all { !headerName.matchesToGlob(it) }
}
}
class HeaderExclusionPolicyImpl(
private val importsImpl: ImportsImpl
) : HeaderExclusionPolicy {
override fun excludeAll(headerId: HeaderId): Boolean {
return headerId in importsImpl.headerIdToPackage
Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt
+39 -10
View File
@@ -337,20 +337,49 @@ internal fun buildNativeLibrary(
})
}
val excludeSystemLibs = def.config.excludeSystemLibs
val excludeDependentModules = def.config.excludeDependentModules
val compilation = object : Compilation {
override val includes = headerFiles
override val additionalPreambleLines = def.defHeaderLines
override val compilerArgs = compilerOpts + tool.platformCompilerOpts
override val language = language
}
val headerFilterGlobs = def.config.headerFilter
val headerInclusionPolicy = HeadersInclusionPolicyImpl(headerFilterGlobs, imports)
val headerFilter: NativeLibraryHeaderFilter
val includes: List<String>
val modules = def.config.modules
if (modules.isEmpty()) {
val excludeDependentModules = def.config.excludeDependentModules
val headerFilterGlobs = def.config.headerFilter
val headerInclusionPolicy = HeaderInclusionPolicyImpl(headerFilterGlobs)
headerFilter = NativeLibraryHeaderFilter.NameBased(headerInclusionPolicy, excludeDependentModules)
includes = headerFiles
} else {
require(language == Language.OBJECTIVE_C) { "cinterop supports 'modules' only when 'language = Objective-C'" }
require(headerFiles.isEmpty()) { "cinterop doesn't support having headers and modules specified at the same time" }
require(def.config.headerFilter.isEmpty()) { "cinterop doesn't support 'headerFilter' with 'modules'" }
val modulesInfo = getModulesInfo(compilation, modules)
headerFilter = NativeLibraryHeaderFilter.Predefined(modulesInfo.ownHeaders)
includes = modulesInfo.topLevelHeaders
}
val excludeSystemLibs = def.config.excludeSystemLibs
val headerExclusionPolicy = HeaderExclusionPolicyImpl(imports)
return NativeLibrary(
includes = headerFiles,
additionalPreambleLines = def.defHeaderLines,
compilerArgs = compilerOpts + tool.platformCompilerOpts,
includes = includes,
additionalPreambleLines = compilation.additionalPreambleLines,
compilerArgs = compilation.compilerArgs,
headerToIdMapper = HeaderToIdMapper(sysRoot = tool.sysRoot),
language = language,
language = compilation.language,
excludeSystemLibs = excludeSystemLibs,
excludeDepdendentModules = excludeDependentModules,
headerInclusionPolicy = headerInclusionPolicy
headerExclusionPolicy = headerExclusionPolicy,
headerFilter = headerFilter
)
}
shared/src/main/kotlin/org/jetbrains/kotlin/konan/util/DefFile.kt
+4
View File
@@ -32,6 +32,10 @@ class DefFile(val file:File?, val config:DefFileConfig, val manifestAddendProper
properties.getSpaceSeparated("headers")
}
val modules by lazy {
properties.getSpaceSeparated("modules")
}
val language by lazy {
properties.getProperty("language")
}