From bddada22e507179fbd489214b0300ffc6632983e Mon Sep 17 00:00:00 2001 From: Alexander Gorshenev Date: Tue, 17 Oct 2017 17:12:56 +0300 Subject: [PATCH] Rewritten unneeded library purge machinery. --- .../kotlin/backend/konan/CompilerOutput.kt | 8 +-- .../kotlin/backend/konan/EscapeAnalysis.kt | 4 +- .../kotlin/backend/konan/KonanConfig.kt | 10 ++- .../kotlin/backend/konan/LinkStage.kt | 2 +- .../konan/library/KonanLibraryReader.kt | 3 +- .../konan/library/SearchPathResolver.kt | 68 ++++++++++--------- .../library/impl/KonanLibraryReaderImpl.kt | 7 +- 7 files changed, 56 insertions(+), 46 deletions(-) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt index ef307fd51f3..4e96a92bffb 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CompilerOutput.kt @@ -49,7 +49,7 @@ internal fun produceOutput(context: Context) { val output = context.config.outputName val libraryName = context.config.moduleId val neededLibraries - = context.config.immediateLibraries.purgeUnneeded() + = context.config.librariesWithDependencies val abiVersion = context.config.currentAbiVersion val target = context.config.targetManager.target val nopack = config.getBoolean(KonanConfigKeys.NOPACK) @@ -80,8 +80,4 @@ internal fun produceOutput(context: Context) { } } -internal fun List.purgeUnneeded(): List { - return this.map { - if (!it.isNeededForLink && it.isDefaultLink) null else it - }.filterNotNull() -} + diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/EscapeAnalysis.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/EscapeAnalysis.kt index 948f90a53c0..478b7586366 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/EscapeAnalysis.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/EscapeAnalysis.kt @@ -1297,7 +1297,7 @@ internal object EscapeAnalysis { val isStdlib = context.config.configuration[KonanConfigKeys.NOSTDLIB] == true val externalFunctionEAResults = mutableMapOf() - context.config.libraries.forEach { library -> + context.config.librariesWithDependencies.forEach { library -> val libraryEscapeAnalysis = library.escapeAnalysis if (libraryEscapeAnalysis != null) { DEBUG_OUTPUT(0) { @@ -1338,4 +1338,4 @@ internal object EscapeAnalysis { } } } -} \ No newline at end of file +} diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index 1475ef498be..d7374beb18d 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -19,7 +19,7 @@ package org.jetbrains.kotlin.backend.konan import com.intellij.openapi.project.Project import org.jetbrains.kotlin.backend.konan.descriptors.createForwardDeclarationsModule import org.jetbrains.kotlin.backend.konan.library.* -import org.jetbrains.kotlin.backend.konan.library.impl.LibraryReaderImpl +import org.jetbrains.kotlin.backend.konan.library.impl.* import org.jetbrains.kotlin.backend.konan.util.profile import org.jetbrains.kotlin.backend.konan.util.removeSuffixIfPresent import org.jetbrains.kotlin.backend.konan.util.suffixIfNot @@ -92,8 +92,13 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration } } - val libraries: List by lazy { + // Accessing this field before resolve is a bad idea. + // purgeUnneeded() only works correctly after we have + // completed resolve and successfully marked package files + // as needed or not needed. + val librariesWithDependencies: List by lazy { resolver.resolveLibrariesRecursive(immediateLibraries, targetManager.target, currentAbiVersion) + immediateLibraries.purgeUnneeded().withResolvedDependencies() } private val loadedDescriptors = loadLibMetadata() @@ -109,6 +114,7 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration val allMetadata = mutableListOf() val specifics = configuration.get(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS)!! + val libraries = immediateLibraries.withResolvedDependencies() for (klib in libraries) { profile("Loading ${klib.libraryName}") { // MutableModuleContext needs ModuleDescriptorImpl, rather than ModuleDescriptor. diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LinkStage.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LinkStage.kt index cb59f0400e5..8fb4561f326 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LinkStage.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LinkStage.kt @@ -235,7 +235,7 @@ internal class LinkStage(val context: Context) { private val debug = config.get(KonanConfigKeys.DEBUG) ?: false private val nomain = config.get(KonanConfigKeys.NOMAIN) ?: false private val emitted = context.bitcodeFileName - private val libraries = context.config.libraries.purgeUnneeded() + private val libraries = context.config.librariesWithDependencies private fun MutableList.addNonEmpty(elements: List) { addAll(elements.filter { !it.isEmpty() }) } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryReader.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryReader.kt index 2ce7b79bfc0..e7fab414309 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryReader.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/KonanLibraryReader.kt @@ -26,10 +26,9 @@ interface KonanLibraryReader { val bitcodePaths: List val includedPaths: List val linkerOpts: List - val dependencies: List + val unresolvedDependencies: List val escapeAnalysis: ByteArray? val isNeededForLink: Boolean get() = true - val isDefaultLink: Boolean get() = false val manifestProperties: Properties val moduleHeaderData: ByteArray fun packageMetadata(fqName: String): ByteArray diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/SearchPathResolver.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/SearchPathResolver.kt index 1efd032e192..74c2f063515 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/SearchPathResolver.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/SearchPathResolver.kt @@ -49,7 +49,7 @@ fun SearchPathResolver.resolveImmediateLibraries(libraryNames: List, removeDuplicates: Boolean = true): List { val defaultLibraries = defaultLinks(nostdlib = noStdLib, noDefaultLibs = noDefaultLibs).map { - LibraryReaderImpl(it, abiVersion, target, isDefaultLink = true) + LibraryReaderImpl(it, abiVersion, target) } val userProvidedLibraries = libraryNames @@ -65,33 +65,40 @@ fun SearchPathResolver.resolveImmediateLibraries(libraryNames: List, fun SearchPathResolver.resolveLibrariesRecursive(immediateLibraries: List, target: KonanTarget, - abiVersion: Int): List { - val result = mutableMapOf() - result.putAll(immediateLibraries.map { it.libraryFile.absoluteFile to it }) - var newDependencies: Map = result + abiVersion: Int) { + val cache = mutableMapOf() + cache.putAll(immediateLibraries.map { it.libraryFile.absoluteFile to it }) + var newDependencies = cache.values.toList() do { - val defaultOverrides = mutableSetOf() - newDependencies = newDependencies.values.asSequence() - .flatMap { it.dependencies.asSequence() } - .map { resolve(it).absoluteFile } - .onEach { - if (result[it]?.isDefaultLink == true) { - defaultOverrides.add(it) - } - } - .filter { it !in result } - .map { it to LibraryReaderImpl(it, abiVersion, target) }.toMap() - - result.putAll(newDependencies) - - // If there is a default link in immediateLibraries, - // and we get the same library as a dependency, - // the resultant surviving library should not be a defaultLink anymore. - defaultOverrides.forEach { - result[it] = LibraryReaderImpl(it, abiVersion, target) - } + newDependencies = newDependencies.map { library: LibraryReaderImpl -> + library.unresolvedDependencies + .map { resolve(it).absoluteFile } + .map { + if (it in cache) { + library.resolvedDependencies.add(cache[it]!!) + null + } else { + val reader = LibraryReaderImpl(it, abiVersion, target) + cache.put(it,reader) + library.resolvedDependencies.add(reader) + reader + } + }.filterNotNull() + } .flatten() } while (newDependencies.isNotEmpty()) - return result.values.toList() +} + +fun List.withResolvedDependencies(): List { + val result = mutableSetOf() + result.addAll(this) + var newDependencies = result.toList() + do { + newDependencies = newDependencies + .map { it -> it.resolvedDependencies } .flatten() + .filter { it !in result } + result.addAll(newDependencies) + } while (newDependencies.isNotEmpty()) + return result.toList() } fun SearchPathResolver.resolveLibrariesRecursive(libraryNames: List, @@ -99,17 +106,16 @@ fun SearchPathResolver.resolveLibrariesRecursive(libraryNames: List, abiVersion: Int = 1, noStdLib: Boolean = false, noDefaultLibs: Boolean = false): List { - return resolveLibrariesRecursive( - resolveImmediateLibraries( + val immediateLibraries = resolveImmediateLibraries( libraryNames = libraryNames, target = target, abiVersion = abiVersion, noStdLib = noStdLib, noDefaultLibs = noDefaultLibs, removeDuplicates = true - ), - target, abiVersion - ) + ) + resolveLibrariesRecursive(immediateLibraries, target, abiVersion) + return immediateLibraries.withResolvedDependencies() } class KonanLibrarySearchPathResolver( diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibraryReaderImpl.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibraryReaderImpl.kt index 0e28b32428e..7998c61bab4 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibraryReaderImpl.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibraryReaderImpl.kt @@ -26,7 +26,7 @@ import org.jetbrains.kotlin.config.LanguageVersionSettings import org.jetbrains.kotlin.konan.target.KonanTarget class LibraryReaderImpl(var libraryFile: File, val currentAbiVersion: Int, - val target: KonanTarget? = null, override val isDefaultLink: Boolean = false) + val target: KonanTarget? = null) : KonanLibraryReader { // For the zipped libraries inPlace gives files from zip file system @@ -68,9 +68,11 @@ class LibraryReaderImpl(var libraryFile: File, val currentAbiVersion: Int, override val linkerOpts: List get() = manifestProperties.propertyList("linkerOpts", target!!.detailedName) - override val dependencies: List + override val unresolvedDependencies: List get() = manifestProperties.propertyList("dependencies") + val resolvedDependencies = mutableListOf() + override val moduleHeaderData: ByteArray by lazy { reader.loadSerializedModule() } @@ -96,3 +98,4 @@ class LibraryReaderImpl(var libraryFile: File, val currentAbiVersion: Int, } +internal fun List.purgeUnneeded(): List = this.filter{ it.isNeededForLink }