Implement internal defFileDependencies tool

2018-09-18 14:55:29 +03:00
parent 9dfd347ef2
commit 8ebcc61a70
5 changed files with 278 additions and 63 deletions
@@ -114,14 +114,7 @@ internal class NativeIndexImpl(val library: NativeLibrary) : NativeIndex() {

    }

-    internal fun getHeaderId(file: CXFile?): HeaderId {
-        if (file == null) {
-            return HeaderId("builtins")
-        }
-
-        val filePath = clang_getFileName(file).convertAndDispose()
-        return library.headerToIdMapper.getHeaderId(filePath)
-    }
+    internal fun getHeaderId(file: CXFile?): HeaderId = getHeaderId(this.library, file)

    private fun getLocation(cursor: CValue<CXCursor>): Location {
        val headerId = getHeaderId(getContainingFile(cursor))
@@ -451,13 +451,30 @@ internal class ModulesMap(
 fun HeaderInclusionPolicy.includeAll(headerName: String?, headerId: HeaderId): Boolean =
        !this.excludeUnused(headerName) && !this.excludeAll(headerId)

+internal fun getHeaderId(library: NativeLibrary, header: CXFile?): HeaderId {
+    if (header == null) {
+        return HeaderId("builtins")
+    }
+
+    val filePath = clang_getFileName(header).convertAndDispose()
+    return library.headerToIdMapper.getHeaderId(filePath)
+}
+
 internal fun getFilteredHeaders(
        nativeIndex: NativeIndexImpl,
        index: CXIndex,
        translationUnit: CXTranslationUnit
-): Set<CXFile?> {
-    val library = nativeIndex.library
-    val result = mutableSetOf<CXFile?>()
+): Set<CXFile?> = getHeaders(nativeIndex.library, index, translationUnit).ownHeaders
+
+class NativeLibraryHeaders<Header>(val ownHeaders: Set<Header>, val importedHeaders: Set<Header>)
+
+internal fun getHeaders(
+        library: NativeLibrary,
+        index: CXIndex,
+        translationUnit: CXTranslationUnit
+): NativeLibraryHeaders<CXFile?> {
+    val ownHeaders = mutableSetOf<CXFile?>()
+    val allHeaders = mutableSetOf<CXFile?>(null)
    val topLevelFiles = mutableListOf<CXFile>()
    var mainFile: CXFile? = null

@@ -467,12 +484,15 @@ internal fun getFilteredHeaders(

        override fun enteredMainFile(file: CXFile) {
            mainFile = file
+            allHeaders += file
        }

        override fun ppIncludedFile(info: CXIdxIncludedFileInfo) {
            val includeLocation = clang_indexLoc_getCXSourceLocation(info.hashLoc.readValue())
            val file = info.file!!

+            allHeaders += file
+
            if (clang_Location_isFromMainFile(includeLocation) != 0) {
                topLevelFiles.add(file)
            }
@@ -498,8 +518,8 @@ internal fun getFilteredHeaders(
            }

            headerToName[file] = headerName
-            if (library.headerInclusionPolicy.includeAll(headerName, nativeIndex.getHeaderId(file))) {
-                result.add(file)
+            if (library.headerInclusionPolicy.includeAll(headerName, getHeaderId(library, file))) {
+                ownHeaders.add(file)
            }
        }
    })
@@ -507,7 +527,7 @@ internal fun getFilteredHeaders(
    if (library.excludeDepdendentModules) {
        ModulesMap(library, translationUnit).use { modulesMap ->
            val topLevelModules = topLevelFiles.map { modulesMap.getModule(it) }.toSet()
-            result.removeAll {
+            ownHeaders.removeAll {
                val module = modulesMap.getModule(it!!)
                module !in topLevelModules
            }
@@ -515,15 +535,37 @@ internal fun getFilteredHeaders(
            // then all non-modular headers are included.
        }
    } else {
-        if (library.headerInclusionPolicy.includeAll(headerName = null, headerId = nativeIndex.getHeaderId(null))) {
+        if (library.headerInclusionPolicy.includeAll(headerName = null, headerId = getHeaderId(library, null))) {
            // Builtins.
-            result.add(null)
+            ownHeaders.add(null)
        }
    }

-    result.add(mainFile!!)
+    ownHeaders.add(mainFile!!)

-    return result
+    return NativeLibraryHeaders(ownHeaders, allHeaders - ownHeaders)
+}
+
+fun NativeLibrary.getHeaderPaths(): NativeLibraryHeaders<String> {
+    val index = clang_createIndex(0, 0)!!
+    try {
+        val translationUnit =
+                this.parse(index, options = CXTranslationUnit_DetailedPreprocessingRecord).ensureNoCompileErrors()
+        try {
+
+            fun getPath(file: CXFile?) = if (file == null) "<builtins>" else clang_getFileName(file).convertAndDispose()
+
+            val headers = getHeaders(this, index, translationUnit)
+            return NativeLibraryHeaders(
+                    headers.ownHeaders.map(::getPath).toSet(),
+                    headers.importedHeaders.map(::getPath).toSet()
+            )
+        } finally {
+            clang_disposeTranslationUnit(translationUnit)
+        }
+    } finally {
+        clang_disposeIndex(index)
+    }
 }

 fun ObjCMethod.replaces(other: ObjCMethod): Boolean =
@@ -0,0 +1,163 @@
+package org.jetbrains.kotlin.native.interop.gen
+
+import org.jetbrains.kotlin.konan.util.DefFile
+import org.jetbrains.kotlin.native.interop.gen.jvm.KotlinPlatform
+import org.jetbrains.kotlin.native.interop.gen.jvm.buildNativeLibrary
+import org.jetbrains.kotlin.native.interop.gen.jvm.prepareTool
+import org.jetbrains.kotlin.native.interop.indexer.NativeLibraryHeaders
+import org.jetbrains.kotlin.native.interop.indexer.getHeaderPaths
+import org.jetbrains.kotlin.native.interop.tool.CInteropArguments
+import java.io.File
+
+fun defFileDependencies(args: Array<String>) {
+    val defFiles = mutableListOf<File>()
+    val targets = mutableListOf<String>()
+
+    var index = 0
+    while (index < args.size) {
+        val arg = args[index]
+
+        ++index
+
+        when (arg) {
+            "-target" -> {
+                targets += args[index]
+                ++index
+            }
+            else -> {
+                defFiles.add(File(arg))
+            }
+        }
+    }
+
+    defFileDependencies(makeDependencyAssigner(targets, defFiles))
+}
+
+private fun makeDependencyAssigner(targets: List<String>, defFiles: List<File>) =
+        CompositeDependencyAssigner(targets.map { makeDependencyAssignerForTarget(it, defFiles) })
+
+private fun makeDependencyAssignerForTarget(target: String, defFiles: List<File>): SingleTargetDependencyAssigner {
+    val tool = prepareTool(target, KotlinPlatform.NATIVE)
+
+    val libraries = defFiles.associateWith {
+        buildNativeLibrary(
+                tool,
+                DefFile(it, tool.substitutions),
+                CInteropArguments(),
+                ImportsImpl(emptyMap())
+        ).getHeaderPaths()
+    }
+    return SingleTargetDependencyAssigner(libraries)
+}
+
+private fun defFileDependencies(dependencyAssigner: DependencyAssigner) {
+    while (!dependencyAssigner.isDone()) {
+        val (file, depends) = dependencyAssigner.getReady().entries.sortedBy { it.key }.first()
+        dependencyAssigner.markDone(file)
+        patchDepends(file, depends.sorted())
+        println("${file.name} done.")
+    }
+}
+
+private fun patchDepends(file: File, newDepends: List<String>) {
+    val defFileLines = file.readLines()
+    val dependsLine = buildString {
+        append("depends =")
+        newDepends.forEach {
+            append(" ")
+            append(it)
+        }
+    }
+    val newDefFileLines = listOf(dependsLine) + defFileLines.filter { !it.startsWith("depends =") }
+
+    file.bufferedWriter().use { writer ->
+        newDefFileLines.forEach { writer.appendln(it) }
+    }
+}
+
+private interface DependencyAssigner {
+    fun isDone(): Boolean
+    fun getReady(): Map<File, Set<String>>
+    fun markDone(file: File)
+}
+
+private class CompositeDependencyAssigner(val dependencyAssigners: List<DependencyAssigner>) : DependencyAssigner {
+    override fun isDone(): Boolean = dependencyAssigners.all { it.isDone() }
+
+    override fun getReady(): Map<File, Set<String>> {
+        return dependencyAssigners.map { it.getReady() }.reduce { left, right ->
+            (left.keys intersect right.keys)
+                    .associateWith { left[it]!! union right[it]!! }
+        }.also {
+            require(it.isNotEmpty()) { "incompatible dependencies" } // TODO: add more info.
+        }
+    }
+
+    override fun markDone(file: File) {
+        dependencyAssigners.forEach { it.markDone(file) }
+    }
+}
+
+private class SingleTargetDependencyAssigner(
+        defFilesToHeaders: Map<File, NativeLibraryHeaders<String>>
+) : DependencyAssigner {
+    private val pendingDefFilesToHeaders = defFilesToHeaders.toMutableMap()
+
+    private val processedHeadersToDefFiles = mutableMapOf<String, File>()
+
+    init {
+        val ownedHeaders = pendingDefFilesToHeaders.values.flatMap { it.ownHeaders }
+        val unownedHeadersToDefFiles = mutableMapOf<String, File>()
+
+        pendingDefFilesToHeaders.forEach { (def, lib) ->
+            (lib.importedHeaders - ownedHeaders).forEach {
+                unownedHeadersToDefFiles.putIfAbsent(it, def)
+            }
+        }
+
+        if (unownedHeadersToDefFiles.isNotEmpty()) {
+            error("Unowned headers:\n" +
+                    unownedHeadersToDefFiles.entries.joinToString("\n") { "${it.key}\n  imported by: ${it.value.name}" })
+        }
+    }
+
+    override fun isDone(): Boolean = pendingDefFilesToHeaders.isEmpty()
+
+    override fun getReady(): Map<File, Set<String>> {
+        val result = mutableMapOf<File, Set<String>>()
+
+        defFiles@for ((defFile, headers) in pendingDefFilesToHeaders) {
+            val depends = mutableSetOf<String>()
+
+            headers@for (header in (headers.ownHeaders + headers.importedHeaders)) {
+                val dependency = processedHeadersToDefFiles[header]
+                        ?: if (header in headers.ownHeaders) continue@headers else continue@defFiles
+
+                depends.add(dependency.nameWithoutExtension)
+            }
+            result[defFile] = depends
+        }
+
+        if (result.isEmpty()) {
+            pendingDefFilesToHeaders.entries.forEach { (def, headers) ->
+                println(def.name)
+                println("Own headers:")
+                headers.ownHeaders.forEach { println(it) }
+                println("Unowned imported headers:")
+                headers.importedHeaders.forEach { if (it !in processedHeadersToDefFiles) println(it) }
+                println()
+            }
+            error("Cyclic dependency? Remaining libs:\n" + pendingDefFilesToHeaders.keys.joinToString("\n") { it.name })
+        }
+
+        return result
+    }
+
+    override fun markDone(file: File) {
+        val headers = pendingDefFilesToHeaders.remove(file)!!
+
+        headers.ownHeaders.forEach {
+            processedHeadersToDefFiles.putIfAbsent(it, file)
+        }
+    }
+}
@@ -19,9 +19,7 @@ package org.jetbrains.kotlin.native.interop.gen.jvm
 import org.jetbrains.kotlin.konan.TempFiles
 import org.jetbrains.kotlin.konan.exec.Command
 import org.jetbrains.kotlin.konan.util.DefFile
-import org.jetbrains.kotlin.native.interop.gen.HeadersInclusionPolicyImpl
-import org.jetbrains.kotlin.native.interop.gen.ImportsImpl
-import org.jetbrains.kotlin.native.interop.gen.argsToCompiler
+import org.jetbrains.kotlin.native.interop.gen.*
 import org.jetbrains.kotlin.native.interop.gen.wasm.processIdlLib
 import org.jetbrains.kotlin.native.interop.indexer.*
 import org.jetbrains.kotlin.native.interop.tool.*
@@ -190,44 +188,15 @@ private fun processCLib(args: Array<String>): Array<String>? {
        return null
    }

-    val tool = ToolConfig(
-        arguments.target,
-        flavor
-    )
-    tool.downloadDependencies()
+    val tool = prepareTool(arguments.target, flavor)

    val def = DefFile(defFile, tool.substitutions)

-    val additionalHeaders = arguments.header
-    val additionalCompilerOpts = arguments.compilerOpts
    val additionalLinkerOpts = arguments.linkerOpts
    val generateShims = arguments.shims
    val verbose = arguments.verbose

-    System.load(tool.libclang)
-
-    val headerFiles = def.config.headers + additionalHeaders
    val language = selectNativeLanguage(def.config)
-    val compilerOpts: List<String> = mutableListOf<String>().apply {
-        addAll(def.config.compilerOpts)
-        addAll(tool.defaultCompilerOpts)
-        addAll(additionalCompilerOpts)
-        addAll(getCompilerFlagsForVfsOverlay(arguments.headerFilterPrefix, def))
-        addAll(when (language) {
-            Language.C -> emptyList()
-            Language.OBJECTIVE_C -> {
-                // "Objective-C" within interop means "Objective-C with ARC":
-                listOf("-fobjc-arc")
-                // Using this flag here has two effects:
-                // 1. The headers are parsed with ARC enabled, thus the API is visible correctly.
-                // 2. The generated Objective-C stubs are compiled with ARC enabled, so reference counting
-                // calls are inserted automatically.
-            }
-        })
-    }
-
-    val excludeSystemLibs = def.config.excludeSystemLibs
-    val excludeDependentModules = def.config.excludeDependentModules

    val entryPoint = def.config.entryPoints.atMostOne()
    val linkerOpts =
@@ -254,20 +223,9 @@ private fun processCLib(args: Array<String>): Array<String>? {

    val tempFiles = TempFiles(libName, arguments.temporaryFilesDir)

-    val headerFilterGlobs = def.config.headerFilter
    val imports = parseImports(arguments.import)
-    val headerInclusionPolicy = HeadersInclusionPolicyImpl(headerFilterGlobs, imports)

-    val library = NativeLibrary(
-            includes = headerFiles,
-            additionalPreambleLines = def.defHeaderLines,
-            compilerArgs = compilerOpts + tool.platformCompilerOpts,
-            headerToIdMapper = HeaderToIdMapper(sysRoot = tool.sysRoot),
-            language = language,
-            excludeSystemLibs = excludeSystemLibs,
-            excludeDepdendentModules = excludeDependentModules,
-            headerInclusionPolicy = headerInclusionPolicy
-    )
+    val library = buildNativeLibrary(tool, def, arguments, imports)

    val configuration = InteropConfiguration(
            library = library,
@@ -337,3 +295,59 @@ private fun processCLib(args: Array<String>): Array<String>? {
    }
    return argsToCompiler(staticLibraries, libraryPaths)
 }
+
+internal fun prepareTool(target: String?, flavor: KotlinPlatform): ToolConfig {
+    val tool = ToolConfig(target, flavor)
+    tool.downloadDependencies()
+
+    System.load(tool.libclang)
+
+    return tool
+}
+
+internal fun buildNativeLibrary(
+        tool: ToolConfig,
+        def: DefFile,
+        arguments: CInteropArguments,
+        imports: ImportsImpl
+): NativeLibrary {
+    val additionalHeaders = arguments.header
+    val additionalCompilerOpts = arguments.compilerOpts
+
+    val headerFiles = def.config.headers + additionalHeaders
+    val language = selectNativeLanguage(def.config)
+    val compilerOpts: List<String> = mutableListOf<String>().apply {
+        addAll(def.config.compilerOpts)
+        addAll(tool.defaultCompilerOpts)
+        addAll(additionalCompilerOpts)
+        addAll(getCompilerFlagsForVfsOverlay(arguments.headerFilterPrefix, def))
+        addAll(when (language) {
+            Language.C -> emptyList()
+            Language.OBJECTIVE_C -> {
+                // "Objective-C" within interop means "Objective-C with ARC":
+                listOf("-fobjc-arc")
+                // Using this flag here has two effects:
+                // 1. The headers are parsed with ARC enabled, thus the API is visible correctly.
+                // 2. The generated Objective-C stubs are compiled with ARC enabled, so reference counting
+                // calls are inserted automatically.
+            }
+        })
+    }
+
+    val excludeSystemLibs = def.config.excludeSystemLibs
+    val excludeDependentModules = def.config.excludeDependentModules
+
+    val headerFilterGlobs = def.config.headerFilter
+    val headerInclusionPolicy = HeadersInclusionPolicyImpl(headerFilterGlobs, imports)
+
+    return NativeLibrary(
+            includes = headerFiles,
+            additionalPreambleLines = def.defHeaderLines,
+            compilerArgs = compilerOpts + tool.platformCompilerOpts,
+            headerToIdMapper = HeaderToIdMapper(sysRoot = tool.sysRoot),
+            language = language,
+            excludeSystemLibs = excludeSystemLibs,
+            excludeDepdendentModules = excludeDependentModules,
+            headerInclusionPolicy = headerInclusionPolicy
+    )
+}
@@ -4,6 +4,7 @@
 */
 package org.jetbrains.kotlin.cli.utilities

+import org.jetbrains.kotlin.native.interop.gen.defFileDependencies
 import org.jetbrains.kotlin.cli.bc.main as konancMain
 import org.jetbrains.kotlin.cli.klib.main as klibMain

@@ -23,6 +24,8 @@ fun main(args: Array<String>) {
        }
        "klib" ->
            klibMain(utilityArgs)
+        "defFileDependencies" ->
+            defFileDependencies(utilityArgs)
        else ->
            error("Unexpected utility name")
    }