From 8ebcc61a70b3edab2a9a23166cd26c3736d789dd Mon Sep 17 00:00:00 2001 From: Svyatoslav Scherbina Date: Tue, 18 Sep 2018 14:55:29 +0300 Subject: [PATCH] Implement internal defFileDependencies tool --- .../kotlin/native/interop/indexer/Indexer.kt | 9 +- .../kotlin/native/interop/indexer/Utils.kt | 62 +++++-- .../native/interop/gen/defFileDependencies.kt | 163 ++++++++++++++++++ .../kotlin/native/interop/gen/jvm/main.kt | 104 ++++++----- .../jetbrains/kotlin/cli/utilities/main.kt | 3 + 5 files changed, 278 insertions(+), 63 deletions(-) create mode 100644 Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/defFileDependencies.kt diff --git a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt index d4f4299dd4c..431c8808777 100644 --- a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt +++ b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Indexer.kt @@ -114,14 +114,7 @@ internal class NativeIndexImpl(val library: NativeLibrary) : NativeIndex() { } - internal fun getHeaderId(file: CXFile?): HeaderId { - if (file == null) { - return HeaderId("builtins") - } - - val filePath = clang_getFileName(file).convertAndDispose() - return library.headerToIdMapper.getHeaderId(filePath) - } + internal fun getHeaderId(file: CXFile?): HeaderId = getHeaderId(this.library, file) private fun getLocation(cursor: CValue): Location { val headerId = getHeaderId(getContainingFile(cursor)) diff --git a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Utils.kt b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Utils.kt index 591ebe2653d..ca1a0d4b2d8 100644 --- a/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Utils.kt +++ b/Interop/Indexer/src/main/kotlin/org/jetbrains/kotlin/native/interop/indexer/Utils.kt @@ -451,13 +451,30 @@ internal class ModulesMap( fun HeaderInclusionPolicy.includeAll(headerName: String?, headerId: HeaderId): Boolean = !this.excludeUnused(headerName) && !this.excludeAll(headerId) +internal fun getHeaderId(library: NativeLibrary, header: CXFile?): HeaderId { + if (header == null) { + return HeaderId("builtins") + } + + val filePath = clang_getFileName(header).convertAndDispose() + return library.headerToIdMapper.getHeaderId(filePath) +} + internal fun getFilteredHeaders( nativeIndex: NativeIndexImpl, index: CXIndex, translationUnit: CXTranslationUnit -): Set { - val library = nativeIndex.library - val result = mutableSetOf() +): Set = getHeaders(nativeIndex.library, index, translationUnit).ownHeaders + +class NativeLibraryHeaders
(val ownHeaders: Set
, val importedHeaders: Set
) + +internal fun getHeaders( + library: NativeLibrary, + index: CXIndex, + translationUnit: CXTranslationUnit +): NativeLibraryHeaders { + val ownHeaders = mutableSetOf() + val allHeaders = mutableSetOf(null) val topLevelFiles = mutableListOf() var mainFile: CXFile? = null @@ -467,12 +484,15 @@ internal fun getFilteredHeaders( override fun enteredMainFile(file: CXFile) { mainFile = file + allHeaders += file } override fun ppIncludedFile(info: CXIdxIncludedFileInfo) { val includeLocation = clang_indexLoc_getCXSourceLocation(info.hashLoc.readValue()) val file = info.file!! + allHeaders += file + if (clang_Location_isFromMainFile(includeLocation) != 0) { topLevelFiles.add(file) } @@ -498,8 +518,8 @@ internal fun getFilteredHeaders( } headerToName[file] = headerName - if (library.headerInclusionPolicy.includeAll(headerName, nativeIndex.getHeaderId(file))) { - result.add(file) + if (library.headerInclusionPolicy.includeAll(headerName, getHeaderId(library, file))) { + ownHeaders.add(file) } } }) @@ -507,7 +527,7 @@ internal fun getFilteredHeaders( if (library.excludeDepdendentModules) { ModulesMap(library, translationUnit).use { modulesMap -> val topLevelModules = topLevelFiles.map { modulesMap.getModule(it) }.toSet() - result.removeAll { + ownHeaders.removeAll { val module = modulesMap.getModule(it!!) module !in topLevelModules } @@ -515,15 +535,37 @@ internal fun getFilteredHeaders( // then all non-modular headers are included. } } else { - if (library.headerInclusionPolicy.includeAll(headerName = null, headerId = nativeIndex.getHeaderId(null))) { + if (library.headerInclusionPolicy.includeAll(headerName = null, headerId = getHeaderId(library, null))) { // Builtins. - result.add(null) + ownHeaders.add(null) } } - result.add(mainFile!!) + ownHeaders.add(mainFile!!) - return result + return NativeLibraryHeaders(ownHeaders, allHeaders - ownHeaders) +} + +fun NativeLibrary.getHeaderPaths(): NativeLibraryHeaders { + val index = clang_createIndex(0, 0)!! + try { + val translationUnit = + this.parse(index, options = CXTranslationUnit_DetailedPreprocessingRecord).ensureNoCompileErrors() + try { + + fun getPath(file: CXFile?) = if (file == null) "" else clang_getFileName(file).convertAndDispose() + + val headers = getHeaders(this, index, translationUnit) + return NativeLibraryHeaders( + headers.ownHeaders.map(::getPath).toSet(), + headers.importedHeaders.map(::getPath).toSet() + ) + } finally { + clang_disposeTranslationUnit(translationUnit) + } + } finally { + clang_disposeIndex(index) + } } fun ObjCMethod.replaces(other: ObjCMethod): Boolean = diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/defFileDependencies.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/defFileDependencies.kt new file mode 100644 index 00000000000..4c8b1f8b166 --- /dev/null +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/defFileDependencies.kt @@ -0,0 +1,163 @@ +package org.jetbrains.kotlin.native.interop.gen + +import org.jetbrains.kotlin.konan.util.DefFile +import org.jetbrains.kotlin.native.interop.gen.jvm.KotlinPlatform +import org.jetbrains.kotlin.native.interop.gen.jvm.buildNativeLibrary +import org.jetbrains.kotlin.native.interop.gen.jvm.prepareTool +import org.jetbrains.kotlin.native.interop.indexer.NativeLibraryHeaders +import org.jetbrains.kotlin.native.interop.indexer.getHeaderPaths +import org.jetbrains.kotlin.native.interop.tool.CInteropArguments +import java.io.File + +fun defFileDependencies(args: Array) { + val defFiles = mutableListOf() + val targets = mutableListOf() + + var index = 0 + while (index < args.size) { + val arg = args[index] + + ++index + + when (arg) { + "-target" -> { + targets += args[index] + ++index + } + else -> { + defFiles.add(File(arg)) + } + } + } + + defFileDependencies(makeDependencyAssigner(targets, defFiles)) +} + +private fun makeDependencyAssigner(targets: List, defFiles: List) = + CompositeDependencyAssigner(targets.map { makeDependencyAssignerForTarget(it, defFiles) }) + +private fun makeDependencyAssignerForTarget(target: String, defFiles: List): SingleTargetDependencyAssigner { + val tool = prepareTool(target, KotlinPlatform.NATIVE) + + val libraries = defFiles.associateWith { + buildNativeLibrary( + tool, + DefFile(it, tool.substitutions), + CInteropArguments(), + ImportsImpl(emptyMap()) + ).getHeaderPaths() + } + return SingleTargetDependencyAssigner(libraries) +} + +private fun defFileDependencies(dependencyAssigner: DependencyAssigner) { + while (!dependencyAssigner.isDone()) { + val (file, depends) = dependencyAssigner.getReady().entries.sortedBy { it.key }.first() + dependencyAssigner.markDone(file) + patchDepends(file, depends.sorted()) + println("${file.name} done.") + } +} + +private fun patchDepends(file: File, newDepends: List) { + val defFileLines = file.readLines() + val dependsLine = buildString { + append("depends =") + newDepends.forEach { + append(" ") + append(it) + } + } + val newDefFileLines = listOf(dependsLine) + defFileLines.filter { !it.startsWith("depends =") } + + file.bufferedWriter().use { writer -> + newDefFileLines.forEach { writer.appendln(it) } + } +} + +private interface DependencyAssigner { + fun isDone(): Boolean + fun getReady(): Map> + fun markDone(file: File) +} + +private class CompositeDependencyAssigner(val dependencyAssigners: List) : DependencyAssigner { + override fun isDone(): Boolean = dependencyAssigners.all { it.isDone() } + + override fun getReady(): Map> { + return dependencyAssigners.map { it.getReady() }.reduce { left, right -> + (left.keys intersect right.keys) + .associateWith { left[it]!! union right[it]!! } + }.also { + require(it.isNotEmpty()) { "incompatible dependencies" } // TODO: add more info. + } + } + + override fun markDone(file: File) { + dependencyAssigners.forEach { it.markDone(file) } + } +} + +private class SingleTargetDependencyAssigner( + defFilesToHeaders: Map> +) : DependencyAssigner { + private val pendingDefFilesToHeaders = defFilesToHeaders.toMutableMap() + + private val processedHeadersToDefFiles = mutableMapOf() + + init { + val ownedHeaders = pendingDefFilesToHeaders.values.flatMap { it.ownHeaders } + val unownedHeadersToDefFiles = mutableMapOf() + + pendingDefFilesToHeaders.forEach { (def, lib) -> + (lib.importedHeaders - ownedHeaders).forEach { + unownedHeadersToDefFiles.putIfAbsent(it, def) + } + } + + if (unownedHeadersToDefFiles.isNotEmpty()) { + error("Unowned headers:\n" + + unownedHeadersToDefFiles.entries.joinToString("\n") { "${it.key}\n imported by: ${it.value.name}" }) + } + } + + override fun isDone(): Boolean = pendingDefFilesToHeaders.isEmpty() + + override fun getReady(): Map> { + val result = mutableMapOf>() + + defFiles@for ((defFile, headers) in pendingDefFilesToHeaders) { + val depends = mutableSetOf() + + headers@for (header in (headers.ownHeaders + headers.importedHeaders)) { + val dependency = processedHeadersToDefFiles[header] + ?: if (header in headers.ownHeaders) continue@headers else continue@defFiles + + depends.add(dependency.nameWithoutExtension) + } + result[defFile] = depends + } + + if (result.isEmpty()) { + pendingDefFilesToHeaders.entries.forEach { (def, headers) -> + println(def.name) + println("Own headers:") + headers.ownHeaders.forEach { println(it) } + println("Unowned imported headers:") + headers.importedHeaders.forEach { if (it !in processedHeadersToDefFiles) println(it) } + println() + } + error("Cyclic dependency? Remaining libs:\n" + pendingDefFilesToHeaders.keys.joinToString("\n") { it.name }) + } + + return result + } + + override fun markDone(file: File) { + val headers = pendingDefFilesToHeaders.remove(file)!! + + headers.ownHeaders.forEach { + processedHeadersToDefFiles.putIfAbsent(it, file) + } + } +} \ No newline at end of file diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt index 4d6ffdf9f02..e338432082e 100644 --- a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt @@ -19,9 +19,7 @@ package org.jetbrains.kotlin.native.interop.gen.jvm import org.jetbrains.kotlin.konan.TempFiles import org.jetbrains.kotlin.konan.exec.Command import org.jetbrains.kotlin.konan.util.DefFile -import org.jetbrains.kotlin.native.interop.gen.HeadersInclusionPolicyImpl -import org.jetbrains.kotlin.native.interop.gen.ImportsImpl -import org.jetbrains.kotlin.native.interop.gen.argsToCompiler +import org.jetbrains.kotlin.native.interop.gen.* import org.jetbrains.kotlin.native.interop.gen.wasm.processIdlLib import org.jetbrains.kotlin.native.interop.indexer.* import org.jetbrains.kotlin.native.interop.tool.* @@ -190,44 +188,15 @@ private fun processCLib(args: Array): Array? { return null } - val tool = ToolConfig( - arguments.target, - flavor - ) - tool.downloadDependencies() + val tool = prepareTool(arguments.target, flavor) val def = DefFile(defFile, tool.substitutions) - val additionalHeaders = arguments.header - val additionalCompilerOpts = arguments.compilerOpts val additionalLinkerOpts = arguments.linkerOpts val generateShims = arguments.shims val verbose = arguments.verbose - System.load(tool.libclang) - - val headerFiles = def.config.headers + additionalHeaders val language = selectNativeLanguage(def.config) - val compilerOpts: List = mutableListOf().apply { - addAll(def.config.compilerOpts) - addAll(tool.defaultCompilerOpts) - addAll(additionalCompilerOpts) - addAll(getCompilerFlagsForVfsOverlay(arguments.headerFilterPrefix, def)) - addAll(when (language) { - Language.C -> emptyList() - Language.OBJECTIVE_C -> { - // "Objective-C" within interop means "Objective-C with ARC": - listOf("-fobjc-arc") - // Using this flag here has two effects: - // 1. The headers are parsed with ARC enabled, thus the API is visible correctly. - // 2. The generated Objective-C stubs are compiled with ARC enabled, so reference counting - // calls are inserted automatically. - } - }) - } - - val excludeSystemLibs = def.config.excludeSystemLibs - val excludeDependentModules = def.config.excludeDependentModules val entryPoint = def.config.entryPoints.atMostOne() val linkerOpts = @@ -254,20 +223,9 @@ private fun processCLib(args: Array): Array? { val tempFiles = TempFiles(libName, arguments.temporaryFilesDir) - val headerFilterGlobs = def.config.headerFilter val imports = parseImports(arguments.import) - val headerInclusionPolicy = HeadersInclusionPolicyImpl(headerFilterGlobs, imports) - val library = NativeLibrary( - includes = headerFiles, - additionalPreambleLines = def.defHeaderLines, - compilerArgs = compilerOpts + tool.platformCompilerOpts, - headerToIdMapper = HeaderToIdMapper(sysRoot = tool.sysRoot), - language = language, - excludeSystemLibs = excludeSystemLibs, - excludeDepdendentModules = excludeDependentModules, - headerInclusionPolicy = headerInclusionPolicy - ) + val library = buildNativeLibrary(tool, def, arguments, imports) val configuration = InteropConfiguration( library = library, @@ -337,3 +295,59 @@ private fun processCLib(args: Array): Array? { } return argsToCompiler(staticLibraries, libraryPaths) } + +internal fun prepareTool(target: String?, flavor: KotlinPlatform): ToolConfig { + val tool = ToolConfig(target, flavor) + tool.downloadDependencies() + + System.load(tool.libclang) + + return tool +} + +internal fun buildNativeLibrary( + tool: ToolConfig, + def: DefFile, + arguments: CInteropArguments, + imports: ImportsImpl +): NativeLibrary { + val additionalHeaders = arguments.header + val additionalCompilerOpts = arguments.compilerOpts + + val headerFiles = def.config.headers + additionalHeaders + val language = selectNativeLanguage(def.config) + val compilerOpts: List = mutableListOf().apply { + addAll(def.config.compilerOpts) + addAll(tool.defaultCompilerOpts) + addAll(additionalCompilerOpts) + addAll(getCompilerFlagsForVfsOverlay(arguments.headerFilterPrefix, def)) + addAll(when (language) { + Language.C -> emptyList() + Language.OBJECTIVE_C -> { + // "Objective-C" within interop means "Objective-C with ARC": + listOf("-fobjc-arc") + // Using this flag here has two effects: + // 1. The headers are parsed with ARC enabled, thus the API is visible correctly. + // 2. The generated Objective-C stubs are compiled with ARC enabled, so reference counting + // calls are inserted automatically. + } + }) + } + + val excludeSystemLibs = def.config.excludeSystemLibs + val excludeDependentModules = def.config.excludeDependentModules + + val headerFilterGlobs = def.config.headerFilter + val headerInclusionPolicy = HeadersInclusionPolicyImpl(headerFilterGlobs, imports) + + return NativeLibrary( + includes = headerFiles, + additionalPreambleLines = def.defHeaderLines, + compilerArgs = compilerOpts + tool.platformCompilerOpts, + headerToIdMapper = HeaderToIdMapper(sysRoot = tool.sysRoot), + language = language, + excludeSystemLibs = excludeSystemLibs, + excludeDepdendentModules = excludeDependentModules, + headerInclusionPolicy = headerInclusionPolicy + ) +} diff --git a/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt b/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt index 593564152de..aa6a4cdd3ef 100644 --- a/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt +++ b/utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt @@ -4,6 +4,7 @@ */ package org.jetbrains.kotlin.cli.utilities +import org.jetbrains.kotlin.native.interop.gen.defFileDependencies import org.jetbrains.kotlin.cli.bc.main as konancMain import org.jetbrains.kotlin.cli.klib.main as klibMain @@ -23,6 +24,8 @@ fun main(args: Array) { } "klib" -> klibMain(utilityArgs) + "defFileDependencies" -> + defFileDependencies(utilityArgs) else -> error("Unexpected utility name") }