[K/N] Merged CachedLibraries.Dependency into DependenciesTracker

2022-12-13 10:13:35 +02:00
parent c1bc6682fa
commit 04fcf25ae0
6 changed files with 149 additions and 107 deletions
@@ -25,12 +25,6 @@ class CachedLibraries(
 ) {
    enum class Kind { DYNAMIC, STATIC }

-    sealed class BitcodeDependency(val libName: String) {
-        class WholeModule(libName: String) : BitcodeDependency(libName)
-
-        class CertainFiles(libName: String, val files: List<String>) : BitcodeDependency(libName)
-    }
-
    sealed class Cache(protected val target: KonanTarget, val kind: Kind, val path: String) {
        val bitcodeDependencies by lazy { computeBitcodeDependencies() }
        val binariesPaths by lazy { computeBinariesPaths() }
@@ -38,7 +32,7 @@ class CachedLibraries(
        val serializedClassFields by lazy { computeSerializedClassFields() }
        val serializedEagerInitializedFiles by lazy { computeSerializedEagerInitializedFiles() }

-        protected abstract fun computeBitcodeDependencies(): List<BitcodeDependency>
+        protected abstract fun computeBitcodeDependencies(): List<DependenciesTracker.UnresolvedDependency>
        protected abstract fun computeBinariesPaths(): List<String>
        protected abstract fun computeSerializedInlineFunctionBodies(): List<SerializedInlineFunctionReference>
        protected abstract fun computeSerializedClassFields(): List<SerializedClassFields>
@@ -49,26 +43,12 @@ class CachedLibraries(
            Kind.STATIC -> CompilerOutputKind.STATIC_CACHE
        }

-        protected fun parseDependencies(dependencies: List<String>): List<BitcodeDependency> {
-            val wholeModuleDependencies = mutableListOf<String>()
-            val fileDependencies = mutableMapOf<String, MutableList<String>>()
-            for (dependency in dependencies) {
-                val delimiterIndex = dependency.lastIndexOf(DEPENDENCIES_DELIMITER)
-                require(delimiterIndex >= 0) { "Invalid dependency $dependency of library $path" }
-                val libName = dependency.substring(0, delimiterIndex)
-                val file = dependency.substring(delimiterIndex + 1, dependency.length)
-                if (file.isEmpty())
-                    wholeModuleDependencies.add(libName)
-                else
-                    fileDependencies.getOrPut(libName) { mutableListOf() }.add(file)
-            }
-            return wholeModuleDependencies.map { BitcodeDependency.WholeModule(it) } +
-                    fileDependencies.map { (libName, files) -> BitcodeDependency.CertainFiles(libName, files) }
-        }
-
        class Monolithic(target: KonanTarget, kind: Kind, path: String) : Cache(target, kind, path) {
-            override fun computeBitcodeDependencies() =
-                    parseDependencies(File(path).parentFile.child(BITCODE_DEPENDENCIES_FILE_NAME).readStrings())
+            override fun computeBitcodeDependencies(): List<DependenciesTracker.UnresolvedDependency> {
+                val directory = File(path).absoluteFile.parentFile
+                val data = directory.child(BITCODE_DEPENDENCIES_FILE_NAME).readStrings()
+                return DependenciesSerializer.deserialize(path, data)
+            }

            override fun computeBinariesPaths() = listOf(path)

@@ -96,7 +76,8 @@ class CachedLibraries(

            private val perFileBitcodeDependencies by lazy {
                fileDirs.associate {
-                    it.name to parseDependencies(it.child(PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME).child(BITCODE_DEPENDENCIES_FILE_NAME).readStrings())
+                    val data = it.child(PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME).child(BITCODE_DEPENDENCIES_FILE_NAME).readStrings()
+                    it.name to DependenciesSerializer.deserialize(it.absolutePath, data)
                }
            }

@@ -212,7 +193,5 @@ class CachedLibraries(
        const val INLINE_FUNCTION_BODIES_FILE_NAME = "inline_bodies"
        const val CLASS_FIELDS_FILE_NAME = "class_fields"
        const val EAGER_INITIALIZED_PROPERTIES_FILE_NAME = "eager_init"
-
-        const val DEPENDENCIES_DELIMITER = '|'
    }
 }
@@ -21,6 +21,39 @@ import org.jetbrains.kotlin.library.metadata.DeserializedKlibModuleOrigin
 import org.jetbrains.kotlin.library.metadata.resolver.TopologicalLibraryOrder
 import org.jetbrains.kotlin.name.FqName

+interface DependenciesTracker {
+    sealed class DependencyKind {
+        object WholeModule : DependencyKind()
+        class CertainFiles(val files: List<String>) : DependencyKind()
+    }
+
+    data class UnresolvedDependency(val libName: String, val kind: DependencyKind) {
+        companion object {
+            fun wholeModule(libName: String) = UnresolvedDependency(libName, DependencyKind.WholeModule)
+            fun certainFiles(libName: String, files: List<String>) = UnresolvedDependency(libName, DependencyKind.CertainFiles(files))
+        }
+    }
+
+    data class ResolvedDependency(val library: KonanLibrary, val kind: DependencyKind) {
+        companion object {
+            fun wholeModule(library: KonanLibrary) = ResolvedDependency(library, DependencyKind.WholeModule)
+            fun certainFiles(library: KonanLibrary, files: List<String>) = ResolvedDependency(library, DependencyKind.CertainFiles(files))
+        }
+    }
+
+    fun add(irFile: IrFile, onlyBitcode: Boolean = false)
+    fun add(declaration: IrDeclaration, onlyBitcode: Boolean = false)
+    fun addNativeRuntime(onlyBitcode: Boolean = false)
+    fun add(functionOrigin: FunctionOrigin, onlyBitcode: Boolean = false)
+
+    val immediateBitcodeDependencies: List<ResolvedDependency>
+    val allCachedBitcodeDependencies: List<ResolvedDependency>
+    val allBitcodeDependencies: List<ResolvedDependency>
+    val nativeDependenciesToLink: List<KonanLibrary>
+    val allNativeDependencies: List<KonanLibrary>
+    val bitcodeToLink: List<KonanLibrary>
+}
+
 private sealed class FileOrigin {
    object CurrentFile : FileOrigin() // No dependency should be added.

@@ -33,14 +66,8 @@ private sealed class FileOrigin {
    class CertainFile(val library: KotlinLibrary, val fqName: String, val filePath: String) : FileOrigin()
 }

-internal class DependenciesTracker(private val generationState: NativeGenerationState) {
-    data class LibraryFile(val library: KotlinLibrary, val fqName: String, val filePath: String)
-
-    sealed class CachedBitcodeDependency(val library: KonanLibrary) {
-        class WholeModule(library: KonanLibrary) : CachedBitcodeDependency(library)
-
-        class CertainFiles(library: KonanLibrary, val files: List<String>) : CachedBitcodeDependency(library)
-    }
+internal class DependenciesTrackerImpl(private val generationState: NativeGenerationState) : DependenciesTracker {
+    private data class LibraryFile(val library: KotlinLibrary, val fqName: String, val filePath: String)

    private val config = generationState.config
    private val context = generationState.context
@@ -66,20 +93,20 @@ internal class DependenciesTracker(private val generationState: NativeGeneration

    private var sealed = false

-    fun add(functionOrigin: FunctionOrigin, onlyBitcode: Boolean = false) = when (functionOrigin) {
+    override fun add(functionOrigin: FunctionOrigin, onlyBitcode: Boolean) = when (functionOrigin) {
        FunctionOrigin.FromNativeRuntime -> addNativeRuntime(onlyBitcode)
        is FunctionOrigin.OwnedBy -> add(functionOrigin.declaration, onlyBitcode)
    }

-    fun add(irFile: IrFile, onlyBitcode: Boolean = false): Unit =
+    override fun add(irFile: IrFile, onlyBitcode: Boolean): Unit =
            add(computeFileOrigin(irFile) { irFile.path }, onlyBitcode)

-    fun add(declaration: IrDeclaration, onlyBitcode: Boolean = false): Unit =
+    override fun add(declaration: IrDeclaration, onlyBitcode: Boolean): Unit =
            add(computeFileOrigin(declaration.getPackageFragment()) {
                context.irLinker.getExternalDeclarationFileName(declaration)
            }, onlyBitcode)

-    fun addNativeRuntime(onlyBitcode: Boolean = false) =
+    override fun addNativeRuntime(onlyBitcode: Boolean) =
            add(FileOrigin.StdlibRuntime, onlyBitcode)

    private fun computeFileOrigin(packageFragment: IrPackageFragment, filePathGetter: () -> String): FileOrigin {
@@ -123,9 +150,9 @@ internal class DependenciesTracker(private val generationState: NativeGeneration
        require(!(sealed && isNewDependency)) { "The dependencies have been sealed off" }
    }

-    fun bitcodeIsUsed(library: KonanLibrary) = library in usedBitcode
+    private fun bitcodeIsUsed(library: KonanLibrary) = library in usedBitcode

-    fun usedBitcode(): List<LibraryFile> = usedBitcodeOfFile.toList()
+    private fun usedBitcode(): List<LibraryFile> = usedBitcodeOfFile.toList()

    private val topSortedLibraries by lazy {
        context.config.resolvedLibraries.getFullList(TopologicalLibraryOrder).map { it as KonanLibrary }
@@ -138,7 +165,7 @@ internal class DependenciesTracker(private val generationState: NativeGeneration
        private val moduleDependencies = mutableSetOf<KonanLibrary>()
        private val fileDependencies = mutableMapOf<KonanLibrary, MutableSet<String>>()

-        val allDependencies: List<CachedBitcodeDependency>
+        val allDependencies: List<DependenciesTracker.ResolvedDependency>

        init {
            val immediateBitcodeDependencies = topSortedLibraries
@@ -174,13 +201,20 @@ internal class DependenciesTracker(private val generationState: NativeGeneration
                }
            }

-            allDependencies = moduleDependencies.map { CachedBitcodeDependency.WholeModule(it) } +
+            allDependencies = moduleDependencies.map { DependenciesTracker.ResolvedDependency.wholeModule(it) } +
                    fileDependencies.filterNot { it.key in moduleDependencies }
-                            .map { (library, files) -> CachedBitcodeDependency.CertainFiles(library, files.toList()) }
+                            .map { (library, files) -> DependenciesTracker.ResolvedDependency.certainFiles(library, files.toList()) }
        }

+        private fun resolveDependency(dependency: DependenciesTracker.UnresolvedDependency) =
+                DependenciesTracker.ResolvedDependency(
+                        allLibraries[dependency.libName] ?: error("Unknown library: ${dependency.libName}"),
+                        dependency.kind)
+
        private fun addAllDependencies(cachedLibrary: CachedLibraries.Cache) {
-            cachedLibrary.bitcodeDependencies.forEach { addDependency(it) }
+            cachedLibrary.bitcodeDependencies
+                    .map { resolveDependency(it) }
+                    .forEach { addDependency(it) }
        }

        private fun addDependencies(cachedLibrary: CachedLibraries.Cache, files: List<String>) = when (cachedLibrary) {
@@ -188,25 +222,26 @@ internal class DependenciesTracker(private val generationState: NativeGeneration

            is CachedLibraries.Cache.PerFile ->
                files.forEach { file ->
-                    cachedLibrary.getFileDependencies(file).forEach { addDependency(it) }
+                    cachedLibrary.getFileDependencies(file)
+                            .map { resolveDependency(it) }
+                            .forEach { addDependency(it) }
                }
        }

-        private fun addDependency(dependency: CachedLibraries.BitcodeDependency) {
-            val dependencyLib = allLibraries[dependency.libName]
-                    ?: error("Bitcode dependency to an unknown library: ${dependency.libName}")
-            if (dependencyLib in moduleDependencies) return
-            val cachedDependency = context.config.cachedLibraries.getLibraryCache(dependencyLib)
-                    ?: error("Library ${dependency.libName} is expected to be cached")
+        private fun addDependency(dependency: DependenciesTracker.ResolvedDependency) {
+            val (library, kind) = dependency
+            if (library in moduleDependencies) return
+            val cachedDependency = context.config.cachedLibraries.getLibraryCache(library)
+                    ?: error("Library ${library.libraryName} is expected to be cached")

-            when (dependency) {
-                is CachedLibraries.BitcodeDependency.WholeModule -> {
-                    moduleDependencies.add(dependencyLib)
+            when (kind) {
+                is DependenciesTracker.DependencyKind.WholeModule -> {
+                    moduleDependencies.add(library)
                    addAllDependencies(cachedDependency)
                }
-                is CachedLibraries.BitcodeDependency.CertainFiles -> {
-                    val handledFiles = fileDependencies.getOrPut(dependencyLib) { mutableSetOf() }
-                    val notHandledFiles = dependency.files.toMutableSet()
+                is DependenciesTracker.DependencyKind.CertainFiles -> {
+                    val handledFiles = fileDependencies.getOrPut(library) { mutableSetOf() }
+                    val notHandledFiles = kind.files.toMutableSet()
                    notHandledFiles.removeAll(handledFiles)
                    handledFiles.addAll(notHandledFiles)
                    if (notHandledFiles.isNotEmpty())
@@ -217,19 +252,33 @@ internal class DependenciesTracker(private val generationState: NativeGeneration
    }

    private inner class Dependencies {
+        val immediateBitcodeDependencies = run {
+            val usedBitcode = usedBitcode().groupBy { it.library }
+            val bitcodeModuleDependencies = mutableListOf<DependenciesTracker.ResolvedDependency>()
+            val bitcodeFileDependencies = mutableListOf<DependenciesTracker.ResolvedDependency>()
+            val libraryToCache = config.cacheSupport.libraryToCache
+            val strategy = libraryToCache?.strategy as? CacheDeserializationStrategy.SingleFile
+            topSortedLibraries.forEach { library ->
+                val filesUsed = usedBitcode[library]
+                if (filesUsed == null && bitcodeIsUsed(library) && library != libraryToCache?.klib /* Skip loops */) {
+                    // Dependency on the entire library.
+                    bitcodeModuleDependencies.add(DependenciesTracker.ResolvedDependency.wholeModule(library))
+                }
+                filesUsed?.filter { library != libraryToCache?.klib || strategy?.filePath != it.filePath /* Skip loops */ }
+                        ?.map { CacheSupport.cacheFileId(it.fqName, it.filePath) }
+                        ?.takeIf { it.isNotEmpty() }
+                        ?.let { bitcodeFileDependencies.add(DependenciesTracker.ResolvedDependency.certainFiles(library, it)) }
+            }
+            bitcodeModuleDependencies + bitcodeFileDependencies
+        }
+
        val allCachedBitcodeDependencies = CachedBitcodeDependenciesComputer().allDependencies

-        val nativeDependenciesToLink = topSortedLibraries.filter { (!it.isDefault && !context.config.purgeUserLibs) || it in usedNativeDependencies }
-
-        val allNativeDependencies = (nativeDependenciesToLink +
-                allCachedBitcodeDependencies.map { it.library } // Native dependencies are per library
-                ).distinct()
-
-        val allBitcodeDependencies: List<CachedBitcodeDependency> = run {
-            val allBitcodeDependencies = mutableMapOf<KonanLibrary, CachedBitcodeDependency>()
+        val allBitcodeDependencies: List<DependenciesTracker.ResolvedDependency> = run {
+            val allBitcodeDependencies = mutableMapOf<KonanLibrary, DependenciesTracker.ResolvedDependency>()
            for (library in context.librariesWithDependencies) {
                if (context.config.cachedLibraries.getLibraryCache(library) == null || library == context.config.libraryToCache?.klib)
-                    allBitcodeDependencies[library] = CachedBitcodeDependency.WholeModule(library)
+                    allBitcodeDependencies[library] = DependenciesTracker.ResolvedDependency.wholeModule(library)
            }
            for (dependency in allCachedBitcodeDependencies)
                allBitcodeDependencies[dependency.library] = dependency
@@ -239,6 +288,12 @@ internal class DependenciesTracker(private val generationState: NativeGeneration
            topSortedLibraries.mapNotNull { allBitcodeDependencies[it] }
        }

+        val nativeDependenciesToLink = topSortedLibraries.filter { (!it.isDefault && !context.config.purgeUserLibs) || it in usedNativeDependencies }
+
+        val allNativeDependencies = (nativeDependenciesToLink +
+                allCachedBitcodeDependencies.map { it.library } // Native dependencies are per library
+                ).distinct()
+
        val bitcodeToLink = topSortedLibraries.filter { shouldContainBitcode(it) }

        private fun shouldContainBitcode(library: KonanLibrary): Boolean {
@@ -261,9 +316,40 @@ internal class DependenciesTracker(private val generationState: NativeGeneration
        Dependencies()
    }

-    val allCachedBitcodeDependencies get() = dependencies.allCachedBitcodeDependencies
-    val nativeDependenciesToLink get() = dependencies.nativeDependenciesToLink
-    val allNativeDependencies get() = dependencies.allNativeDependencies
-    val allBitcodeDependencies get() = dependencies.allBitcodeDependencies
-    val bitcodeToLink get() = dependencies.bitcodeToLink
+    override val immediateBitcodeDependencies get() = dependencies.immediateBitcodeDependencies
+    override val allCachedBitcodeDependencies get() = dependencies.allCachedBitcodeDependencies
+    override val allBitcodeDependencies get() = dependencies.allBitcodeDependencies
+    override val nativeDependenciesToLink get() = dependencies.nativeDependenciesToLink
+    override val allNativeDependencies get() = dependencies.allNativeDependencies
+    override val bitcodeToLink get() = dependencies.bitcodeToLink
+}
+
+internal object DependenciesSerializer {
+    fun serialize(dependencies: List<DependenciesTracker.ResolvedDependency>) =
+            dependencies.flatMap { (library, kind) ->
+                val libName = library.uniqueName
+                when (kind) {
+                    DependenciesTracker.DependencyKind.WholeModule -> listOf("$libName$DEPENDENCIES_DELIMITER")
+                    is DependenciesTracker.DependencyKind.CertainFiles -> kind.files.map { "$libName$DEPENDENCIES_DELIMITER$it" }
+                }
+            }
+
+    fun deserialize(path: String, dependencies: List<String>): List<DependenciesTracker.UnresolvedDependency> {
+        val wholeModuleDependencies = mutableListOf<String>()
+        val fileDependencies = mutableMapOf<String, MutableList<String>>()
+        for (dependency in dependencies) {
+            val delimiterIndex = dependency.lastIndexOf(DEPENDENCIES_DELIMITER)
+            require(delimiterIndex >= 0) { "Invalid dependency $dependency at $path" }
+            val libName = dependency.substring(0, delimiterIndex)
+            val file = dependency.substring(delimiterIndex + 1, dependency.length)
+            if (file.isEmpty())
+                wholeModuleDependencies.add(libName)
+            else
+                fileDependencies.getOrPut(libName) { mutableListOf() }.add(file)
+        }
+        return wholeModuleDependencies.map { DependenciesTracker.UnresolvedDependency.wholeModule(it) } +
+                fileDependencies.map { (libName, files) -> DependenciesTracker.UnresolvedDependency.certainFiles(libName, files) }
+    }
+
+    private const val DEPENDENCIES_DELIMITER = '|'
 }
@@ -12,8 +12,6 @@ import org.jetbrains.kotlin.konan.target.CompilerOutputKind
 import org.jetbrains.kotlin.konan.target.Family
 import org.jetbrains.kotlin.konan.target.LinkerOutputKind
 import org.jetbrains.kotlin.konan.target.presetName
-import org.jetbrains.kotlin.library.metadata.resolver.TopologicalLibraryOrder
-import org.jetbrains.kotlin.library.uniqueName

 internal fun determineLinkerOutput(context: PhaseContext): LinkerOutputKind =
        when (context.config.produce) {
@@ -60,29 +58,8 @@ internal class CacheStorage(val generationState: NativeGenerationState) {
    }

    private fun saveCacheBitcodeDependencies() {
-        val libraryToCache = config.cacheSupport.libraryToCache!!
-        val usedBitcode = generationState.dependenciesTracker.usedBitcode().groupBy { it.library }
-        val bitcodeModuleDependencies = mutableListOf<String>()
-        val bitcodeFileDependencies = mutableListOf<String>()
-        val strategy = libraryToCache.strategy as? CacheDeserializationStrategy.SingleFile
-        config.resolvedLibraries
-                .getFullList(TopologicalLibraryOrder)
-                .forEach { library ->
-                    require(library is KonanLibrary)
-                    val filesUsed = usedBitcode[library]
-                    if (filesUsed == null && generationState.dependenciesTracker.bitcodeIsUsed(library)
-                            && library != libraryToCache.klib /* Skip loops */) {
-                        // Dependency on the entire library.
-                        bitcodeModuleDependencies.add("${library.uniqueName}${CachedLibraries.DEPENDENCIES_DELIMITER}")
-                    }
-                    filesUsed?.forEach { libraryFile ->
-                        if (library != libraryToCache.klib || strategy?.filePath != libraryFile.filePath /* Skip loops */) {
-                            val fileId = CacheSupport.cacheFileId(libraryFile.fqName, libraryFile.filePath)
-                            bitcodeFileDependencies.add("${library.uniqueName}${CachedLibraries.DEPENDENCIES_DELIMITER}$fileId")
-                        }
-                    }
-                }
-        outputFiles.bitcodeDependenciesFile!!.writeLines(bitcodeModuleDependencies + bitcodeFileDependencies)
+        outputFiles.bitcodeDependenciesFile!!.writeLines(
+                DependenciesSerializer.serialize(generationState.dependenciesTracker.immediateBitcodeDependencies))
    }

    private fun saveInlineFunctionBodies() {
@@ -276,8 +253,8 @@ private fun determineCachesToLink(generationState: NativeGenerationState): Cache
            CachedLibraries.Kind.STATIC -> staticCaches
        }

-        list += if (dependency is DependenciesTracker.CachedBitcodeDependency.CertainFiles && cache is CachedLibraries.Cache.PerFile)
-            dependency.files.map { cache.getFileBinaryPath(it) }
+        list += if (dependency.kind is DependenciesTracker.DependencyKind.CertainFiles && cache is CachedLibraries.Cache.PerFile)
+            dependency.kind.files.map { cache.getFileBinaryPath(it) }
        else cache.binariesPaths
    }
    return CachesToLink(static = staticCaches, dynamic = dynamicCaches)
@@ -84,7 +84,7 @@ internal class NativeGenerationState(

    val producedLlvmModuleContainsStdlib get() = llvmModuleSpecification.containsModule(context.stdlibModule)

-    val dependenciesTracker = DependenciesTracker(this)
+    val dependenciesTracker: DependenciesTracker = DependenciesTrackerImpl(this)

    private val runtimeDelegate = lazy { Runtime(llvmContext, config.distribution.compilerInterface(config.target)) }
    private val llvmDelegate = lazy { Llvm(this, LLVMModuleCreateWithNameInContext("out", llvmContext)!!) }
@@ -2787,11 +2787,11 @@ internal class CodeGeneratorVisitor(val generationState: NativeGenerationState,
                when (cache) {
                    is CachedLibraries.Cache.Monolithic -> listOf(addCtorFunction(ctorName))
                    is CachedLibraries.Cache.PerFile -> {
-                        val files = when (dependency) {
-                            is DependenciesTracker.CachedBitcodeDependency.WholeModule ->
+                        val files = when (dependency.kind) {
+                            is DependenciesTracker.DependencyKind.WholeModule ->
                                context.irLinker.klibToModuleDeserializerMap[library]!!.sortedFileIds
-                            is DependenciesTracker.CachedBitcodeDependency.CertainFiles ->
-                                dependency.files
+                            is DependenciesTracker.DependencyKind.CertainFiles ->
+                                dependency.kind.files
                        }
                        files.map { addCtorFunction(fileCtorName(library.uniqueName, it)) }
                    }
@@ -147,7 +147,7 @@ internal open class LlvmFunctionSignature(
    }
 }

-internal sealed class FunctionOrigin {
+sealed class FunctionOrigin {
    object FromNativeRuntime : FunctionOrigin()

    class OwnedBy(val declaration: IrDeclaration) : FunctionOrigin()