Implement internal defFileDependencies tool

Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt

@@ -114,14 +114,7 @@ internal class NativeIndexImpl(val library: NativeLibrary) : NativeIndex() {
 
     }
 
-    internal fun getHeaderId(file: CXFile?): HeaderId {
+    internal fun getHeaderId(file: CXFile?): HeaderId = getHeaderId(this.library, file)
-        if (file == null) {
-            return HeaderId("builtins")
-        }
-
-        val filePath = clang_getFileName(file).convertAndDispose()
-        return library.headerToIdMapper.getHeaderId(filePath)
-    }
 
     private fun getLocation(cursor: CValue<CXCursor>): Location {
         val headerId = getHeaderId(getContainingFile(cursor))

Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Utils.kt

@@ -451,13 +451,30 @@ internal class ModulesMap(
 fun HeaderInclusionPolicy.includeAll(headerName: String?, headerId: HeaderId): Boolean =
         !this.excludeUnused(headerName) && !this.excludeAll(headerId)
 
+internal fun getHeaderId(library: NativeLibrary, header: CXFile?): HeaderId {
+    if (header == null) {
+        return HeaderId("builtins")
+    }
+
+    val filePath = clang_getFileName(header).convertAndDispose()
+    return library.headerToIdMapper.getHeaderId(filePath)
+}
+
 internal fun getFilteredHeaders(
         nativeIndex: NativeIndexImpl,
         index: CXIndex,
         translationUnit: CXTranslationUnit
-): Set<CXFile?> {
+): Set<CXFile?> = getHeaders(nativeIndex.library, index, translationUnit).ownHeaders
-    val library = nativeIndex.library
+
-    val result = mutableSetOf<CXFile?>()
+class NativeLibraryHeaders<Header>(val ownHeaders: Set<Header>, val importedHeaders: Set<Header>)
+
+internal fun getHeaders(
+        library: NativeLibrary,
+        index: CXIndex,
+        translationUnit: CXTranslationUnit
+): NativeLibraryHeaders<CXFile?> {
+    val ownHeaders = mutableSetOf<CXFile?>()
+    val allHeaders = mutableSetOf<CXFile?>(null)
     val topLevelFiles = mutableListOf<CXFile>()
     var mainFile: CXFile? = null
 
@@ -467,12 +484,15 @@ internal fun getFilteredHeaders(
 
         override fun enteredMainFile(file: CXFile) {
             mainFile = file
+            allHeaders += file
         }
 
         override fun ppIncludedFile(info: CXIdxIncludedFileInfo) {
             val includeLocation = clang_indexLoc_getCXSourceLocation(info.hashLoc.readValue())
             val file = info.file!!
 
+            allHeaders += file
+
             if (clang_Location_isFromMainFile(includeLocation) != 0) {
                 topLevelFiles.add(file)
             }
@@ -498,8 +518,8 @@ internal fun getFilteredHeaders(
             }
 
             headerToName[file] = headerName
-            if (library.headerInclusionPolicy.includeAll(headerName, nativeIndex.getHeaderId(file))) {
+            if (library.headerInclusionPolicy.includeAll(headerName, getHeaderId(library, file))) {
-                result.add(file)
+                ownHeaders.add(file)
             }
         }
     })
@@ -507,7 +527,7 @@ internal fun getFilteredHeaders(
     if (library.excludeDepdendentModules) {
         ModulesMap(library, translationUnit).use { modulesMap ->
             val topLevelModules = topLevelFiles.map { modulesMap.getModule(it) }.toSet()
-            result.removeAll {
+            ownHeaders.removeAll {
                 val module = modulesMap.getModule(it!!)
                 module !in topLevelModules
             }
@@ -515,15 +535,37 @@ internal fun getFilteredHeaders(
             // then all non-modular headers are included.
         }
     } else {
-        if (library.headerInclusionPolicy.includeAll(headerName = null, headerId = nativeIndex.getHeaderId(null))) {
+        if (library.headerInclusionPolicy.includeAll(headerName = null, headerId = getHeaderId(library, null))) {
             // Builtins.
-            result.add(null)
+            ownHeaders.add(null)
         }
     }
 
-    result.add(mainFile!!)
+    ownHeaders.add(mainFile!!)
 
-    return result
+    return NativeLibraryHeaders(ownHeaders, allHeaders - ownHeaders)
+}
+
+fun NativeLibrary.getHeaderPaths(): NativeLibraryHeaders<String> {
+    val index = clang_createIndex(0, 0)!!
+    try {
+        val translationUnit =
+                this.parse(index, options = CXTranslationUnit_DetailedPreprocessingRecord).ensureNoCompileErrors()
+        try {
+
+            fun getPath(file: CXFile?) = if (file == null) "<builtins>" else clang_getFileName(file).convertAndDispose()
+
+            val headers = getHeaders(this, index, translationUnit)
+            return NativeLibraryHeaders(
+                    headers.ownHeaders.map(::getPath).toSet(),
+                    headers.importedHeaders.map(::getPath).toSet()
+            )
+        } finally {
+            clang_disposeTranslationUnit(translationUnit)
+        }
+    } finally {
+        clang_disposeIndex(index)
+    }
 }
 
 fun ObjCMethod.replaces(other: ObjCMethod): Boolean =

Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/defFileDependencies.kt

@@ -0,0 +1,163 @@
+package org.jetbrains.kotlin.native.interop.gen
+
+import org.jetbrains.kotlin.konan.util.DefFile
+import org.jetbrains.kotlin.native.interop.gen.jvm.KotlinPlatform
+import org.jetbrains.kotlin.native.interop.gen.jvm.buildNativeLibrary
+import org.jetbrains.kotlin.native.interop.gen.jvm.prepareTool
+import org.jetbrains.kotlin.native.interop.indexer.NativeLibraryHeaders
+import org.jetbrains.kotlin.native.interop.indexer.getHeaderPaths
+import org.jetbrains.kotlin.native.interop.tool.CInteropArguments
+import java.io.File
+
+fun defFileDependencies(args: Array<String>) {
+    val defFiles = mutableListOf<File>()
+    val targets = mutableListOf<String>()
+
+    var index = 0
+    while (index < args.size) {
+        val arg = args[index]
+
+        ++index
+
+        when (arg) {
+            "-target" -> {
+                targets += args[index]
+                ++index
+            }
+            else -> {
+                defFiles.add(File(arg))
+            }
+        }
+    }
+
+    defFileDependencies(makeDependencyAssigner(targets, defFiles))
+}
+
+private fun makeDependencyAssigner(targets: List<String>, defFiles: List<File>) =
+        CompositeDependencyAssigner(targets.map { makeDependencyAssignerForTarget(it, defFiles) })
+
+private fun makeDependencyAssignerForTarget(target: String, defFiles: List<File>): SingleTargetDependencyAssigner {
+    val tool = prepareTool(target, KotlinPlatform.NATIVE)
+
+    val libraries = defFiles.associateWith {
+        buildNativeLibrary(
+                tool,
+                DefFile(it, tool.substitutions),
+                CInteropArguments(),
+                ImportsImpl(emptyMap())
+        ).getHeaderPaths()
+    }
+    return SingleTargetDependencyAssigner(libraries)
+}
+
+private fun defFileDependencies(dependencyAssigner: DependencyAssigner) {
+    while (!dependencyAssigner.isDone()) {
+        val (file, depends) = dependencyAssigner.getReady().entries.sortedBy { it.key }.first()
+        dependencyAssigner.markDone(file)
+        patchDepends(file, depends.sorted())
+        println("${file.name} done.")
+    }
+}
+
+private fun patchDepends(file: File, newDepends: List<String>) {
+    val defFileLines = file.readLines()
+    val dependsLine = buildString {
+        append("depends =")
+        newDepends.forEach {
+            append(" ")
+            append(it)
+        }
+    }
+    val newDefFileLines = listOf(dependsLine) + defFileLines.filter { !it.startsWith("depends =") }
+
+    file.bufferedWriter().use { writer ->
+        newDefFileLines.forEach { writer.appendln(it) }
+    }
+}
+
+private interface DependencyAssigner {
+    fun isDone(): Boolean
+    fun getReady(): Map<File, Set<String>>
+    fun markDone(file: File)
+}
+
+private class CompositeDependencyAssigner(val dependencyAssigners: List<DependencyAssigner>) : DependencyAssigner {
+    override fun isDone(): Boolean = dependencyAssigners.all { it.isDone() }
+
+    override fun getReady(): Map<File, Set<String>> {
+        return dependencyAssigners.map { it.getReady() }.reduce { left, right ->
+            (left.keys intersect right.keys)
+                    .associateWith { left[it]!! union right[it]!! }
+        }.also {
+            require(it.isNotEmpty()) { "incompatible dependencies" } // TODO: add more info.
+        }
+    }
+
+    override fun markDone(file: File) {
+        dependencyAssigners.forEach { it.markDone(file) }
+    }
+}
+
+private class SingleTargetDependencyAssigner(
+        defFilesToHeaders: Map<File, NativeLibraryHeaders<String>>
+) : DependencyAssigner {
+    private val pendingDefFilesToHeaders = defFilesToHeaders.toMutableMap()
+
+    private val processedHeadersToDefFiles = mutableMapOf<String, File>()
+
+    init {
+        val ownedHeaders = pendingDefFilesToHeaders.values.flatMap { it.ownHeaders }
+        val unownedHeadersToDefFiles = mutableMapOf<String, File>()
+
+        pendingDefFilesToHeaders.forEach { (def, lib) ->
+            (lib.importedHeaders - ownedHeaders).forEach {
+                unownedHeadersToDefFiles.putIfAbsent(it, def)
+            }
+        }
+
+        if (unownedHeadersToDefFiles.isNotEmpty()) {
+            error("Unowned headers:\n" +
+                    unownedHeadersToDefFiles.entries.joinToString("\n") { "${it.key}\n  imported by: ${it.value.name}" })
+        }
+    }
+
+    override fun isDone(): Boolean = pendingDefFilesToHeaders.isEmpty()
+
+    override fun getReady(): Map<File, Set<String>> {
+        val result = mutableMapOf<File, Set<String>>()
+
+        defFiles@for ((defFile, headers) in pendingDefFilesToHeaders) {
+            val depends = mutableSetOf<String>()
+
+            headers@for (header in (headers.ownHeaders + headers.importedHeaders)) {
+                val dependency = processedHeadersToDefFiles[header]
+                        ?: if (header in headers.ownHeaders) continue@headers else continue@defFiles
+
+                depends.add(dependency.nameWithoutExtension)
+            }
+            result[defFile] = depends
+        }
+
+        if (result.isEmpty()) {
+            pendingDefFilesToHeaders.entries.forEach { (def, headers) ->
+                println(def.name)
+                println("Own headers:")
+                headers.ownHeaders.forEach { println(it) }
+                println("Unowned imported headers:")
+                headers.importedHeaders.forEach { if (it !in processedHeadersToDefFiles) println(it) }
+                println()
+            }
+            error("Cyclic dependency? Remaining libs:\n" + pendingDefFilesToHeaders.keys.joinToString("\n") { it.name })
+        }
+
+        return result
+    }
+
+    override fun markDone(file: File) {
+        val headers = pendingDefFilesToHeaders.remove(file)!!
+
+        headers.ownHeaders.forEach {
+            processedHeadersToDefFiles.putIfAbsent(it, file)
+        }
+    }
+}

Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt

@@ -19,9 +19,7 @@ package org.jetbrains.kotlin.native.interop.gen.jvm
 import org.jetbrains.kotlin.konan.TempFiles
 import org.jetbrains.kotlin.konan.exec.Command
 import org.jetbrains.kotlin.konan.util.DefFile
-import org.jetbrains.kotlin.native.interop.gen.HeadersInclusionPolicyImpl
+import org.jetbrains.kotlin.native.interop.gen.*
-import org.jetbrains.kotlin.native.interop.gen.ImportsImpl
-import org.jetbrains.kotlin.native.interop.gen.argsToCompiler
 import org.jetbrains.kotlin.native.interop.gen.wasm.processIdlLib
 import org.jetbrains.kotlin.native.interop.indexer.*
 import org.jetbrains.kotlin.native.interop.tool.*
@@ -190,44 +188,15 @@ private fun processCLib(args: Array<String>): Array<String>? {
         return null
     }
 
-    val tool = ToolConfig(
+    val tool = prepareTool(arguments.target, flavor)
-        arguments.target,
-        flavor
-    )
-    tool.downloadDependencies()
 
     val def = DefFile(defFile, tool.substitutions)
 
-    val additionalHeaders = arguments.header
-    val additionalCompilerOpts = arguments.compilerOpts
     val additionalLinkerOpts = arguments.linkerOpts
     val generateShims = arguments.shims
     val verbose = arguments.verbose
 
-    System.load(tool.libclang)
-
-    val headerFiles = def.config.headers + additionalHeaders
     val language = selectNativeLanguage(def.config)
-    val compilerOpts: List<String> = mutableListOf<String>().apply {
-        addAll(def.config.compilerOpts)
-        addAll(tool.defaultCompilerOpts)
-        addAll(additionalCompilerOpts)
-        addAll(getCompilerFlagsForVfsOverlay(arguments.headerFilterPrefix, def))
-        addAll(when (language) {
-            Language.C -> emptyList()
-            Language.OBJECTIVE_C -> {
-                // "Objective-C" within interop means "Objective-C with ARC":
-                listOf("-fobjc-arc")
-                // Using this flag here has two effects:
-                // 1. The headers are parsed with ARC enabled, thus the API is visible correctly.
-                // 2. The generated Objective-C stubs are compiled with ARC enabled, so reference counting
-                // calls are inserted automatically.
-            }
-        })
-    }
-
-    val excludeSystemLibs = def.config.excludeSystemLibs
-    val excludeDependentModules = def.config.excludeDependentModules
 
     val entryPoint = def.config.entryPoints.atMostOne()
     val linkerOpts =
@@ -254,20 +223,9 @@ private fun processCLib(args: Array<String>): Array<String>? {
 
     val tempFiles = TempFiles(libName, arguments.temporaryFilesDir)
 
-    val headerFilterGlobs = def.config.headerFilter
     val imports = parseImports(arguments.import)
-    val headerInclusionPolicy = HeadersInclusionPolicyImpl(headerFilterGlobs, imports)
 
-    val library = NativeLibrary(
+    val library = buildNativeLibrary(tool, def, arguments, imports)
-            includes = headerFiles,
-            additionalPreambleLines = def.defHeaderLines,
-            compilerArgs = compilerOpts + tool.platformCompilerOpts,
-            headerToIdMapper = HeaderToIdMapper(sysRoot = tool.sysRoot),
-            language = language,
-            excludeSystemLibs = excludeSystemLibs,
-            excludeDepdendentModules = excludeDependentModules,
-            headerInclusionPolicy = headerInclusionPolicy
-    )
 
     val configuration = InteropConfiguration(
             library = library,
@@ -337,3 +295,59 @@ private fun processCLib(args: Array<String>): Array<String>? {
     }
     return argsToCompiler(staticLibraries, libraryPaths)
 }
+
+internal fun prepareTool(target: String?, flavor: KotlinPlatform): ToolConfig {
+    val tool = ToolConfig(target, flavor)
+    tool.downloadDependencies()
+
+    System.load(tool.libclang)
+
+    return tool
+}
+
+internal fun buildNativeLibrary(
+        tool: ToolConfig,
+        def: DefFile,
+        arguments: CInteropArguments,
+        imports: ImportsImpl
+): NativeLibrary {
+    val additionalHeaders = arguments.header
+    val additionalCompilerOpts = arguments.compilerOpts
+
+    val headerFiles = def.config.headers + additionalHeaders
+    val language = selectNativeLanguage(def.config)
+    val compilerOpts: List<String> = mutableListOf<String>().apply {
+        addAll(def.config.compilerOpts)
+        addAll(tool.defaultCompilerOpts)
+        addAll(additionalCompilerOpts)
+        addAll(getCompilerFlagsForVfsOverlay(arguments.headerFilterPrefix, def))
+        addAll(when (language) {
+            Language.C -> emptyList()
+            Language.OBJECTIVE_C -> {
+                // "Objective-C" within interop means "Objective-C with ARC":
+                listOf("-fobjc-arc")
+                // Using this flag here has two effects:
+                // 1. The headers are parsed with ARC enabled, thus the API is visible correctly.
+                // 2. The generated Objective-C stubs are compiled with ARC enabled, so reference counting
+                // calls are inserted automatically.
+            }
+        })
+    }
+
+    val excludeSystemLibs = def.config.excludeSystemLibs
+    val excludeDependentModules = def.config.excludeDependentModules
+
+    val headerFilterGlobs = def.config.headerFilter
+    val headerInclusionPolicy = HeadersInclusionPolicyImpl(headerFilterGlobs, imports)
+
+    return NativeLibrary(
+            includes = headerFiles,
+            additionalPreambleLines = def.defHeaderLines,
+            compilerArgs = compilerOpts + tool.platformCompilerOpts,
+            headerToIdMapper = HeaderToIdMapper(sysRoot = tool.sysRoot),
+            language = language,
+            excludeSystemLibs = excludeSystemLibs,
+            excludeDepdendentModules = excludeDependentModules,
+            headerInclusionPolicy = headerInclusionPolicy
+    )
+}

utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt

@@ -4,6 +4,7 @@
  */
 package org.jetbrains.kotlin.cli.utilities
 
+import org.jetbrains.kotlin.native.interop.gen.defFileDependencies
 import org.jetbrains.kotlin.cli.bc.main as konancMain
 import org.jetbrains.kotlin.cli.klib.main as klibMain
 
@@ -23,6 +24,8 @@ fun main(args: Array<String>) {
         }
         "klib" ->
             klibMain(utilityArgs)
+        "defFileDependencies" ->
+            defFileDependencies(utilityArgs)
         else ->
             error("Unexpected utility name")
     }