From 68e1acd2cb1a37c3287982ce4da356fff6aca7f3 Mon Sep 17 00:00:00 2001 From: Dmitriy Dolovov Date: Thu, 18 Jun 2020 18:08:00 +0700 Subject: [PATCH] [Commonizer] More detailed progress logging --- .../descriptors/commonizer/Parameters.kt | 3 +- .../descriptors/commonizer/builder/context.kt | 5 +- .../descriptors/commonizer/cli/nativeTasks.kt | 3 +- .../kotlin/descriptors/commonizer/facade.kt | 7 +- .../konan/NativeDistributionCommonizer.kt | 78 ++++++++++++++----- .../commonizer/mergedtree/CirTreeMerger.kt | 8 +- .../commonizer/utils/ResettableClockMark.kt | 9 ++- 7 files changed, 81 insertions(+), 32 deletions(-) diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/Parameters.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/Parameters.kt index b33666ee1de..b4b42d9d53f 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/Parameters.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/Parameters.kt @@ -8,7 +8,8 @@ package org.jetbrains.kotlin.descriptors.commonizer import org.jetbrains.kotlin.descriptors.commonizer.stats.StatsCollector class Parameters( - val statsCollector: StatsCollector? = null + val statsCollector: StatsCollector? = null, + val progressLogger: ((String) -> Unit)? = null ) { // use linked hash map to preserve order private val _targetProviders = LinkedHashMap() diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt index 153f3eb26ab..f596ba470fd 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt @@ -7,6 +7,7 @@ package org.jetbrains.kotlin.descriptors.commonizer.builder import org.jetbrains.kotlin.builtins.KotlinBuiltIns import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.descriptors.commonizer.Parameters import org.jetbrains.kotlin.descriptors.commonizer.Target import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirNode.Companion.dimension import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirNode.Companion.indexOfCommon @@ -166,7 +167,7 @@ class TargetDeclarationsBuilderComponents( fun CirRootNode.createGlobalBuilderComponents( storageManager: StorageManager, - statsCollector: StatsCollector? + parameters: Parameters ): GlobalDeclarationsBuilderComponents { val cache = DeclarationsBuilderCache(dimension) @@ -189,7 +190,7 @@ fun CirRootNode.createGlobalBuilderComponents( ) } - return GlobalDeclarationsBuilderComponents(storageManager, targetContexts, cache, statsCollector) + return GlobalDeclarationsBuilderComponents(storageManager, targetContexts, cache, parameters.statsCollector) } interface TypeParameterResolver { diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeTasks.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeTasks.kt index 34036641320..e0ff95e4642 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeTasks.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeTasks.kt @@ -47,8 +47,9 @@ internal class NativeDistributionCommonize(options: Collection>) : Tas val copyEndorsedLibs = getOptional { it == "copy-endorsed-libs" } ?: false val statsType = getOptional { it == "log-stats" } ?: StatsType.NONE + val targetNames = targets.joinToString { "[${it.name}]" } val descriptionSuffix = estimateLibrariesCount(distribution, targets)?.let { " ($it items)" } ?: "" - val description = "${logPrefix}Preparing commonized Kotlin/Native libraries for targets $targets$descriptionSuffix" + val description = "${logPrefix}Preparing commonized Kotlin/Native libraries for targets $targetNames$descriptionSuffix" println(description) diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt index a3659bc575c..54dd7c4e7e7 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt @@ -20,13 +20,14 @@ fun runCommonization(parameters: Parameters): Result { val storageManager = LockBasedStorageManager("Declaration descriptors commonization") // build merged tree: - val mergedTree = CirTreeMerger(storageManager, parameters.targetProviders).merge() + val mergedTree = CirTreeMerger(storageManager, parameters).merge() // commonize: mergedTree.accept(CommonizationVisitor(mergedTree), Unit) + parameters.progressLogger?.invoke("Commonized declarations") // build resulting descriptors: - val components = mergedTree.createGlobalBuilderComponents(storageManager, parameters.statsCollector) + val components = mergedTree.createGlobalBuilderComponents(storageManager, parameters) mergedTree.accept(DeclarationsBuilderVisitor1(components), emptyList()) mergedTree.accept(DeclarationsBuilderVisitor2(components), emptyList()) @@ -38,5 +39,7 @@ fun runCommonization(parameters: Parameters): Result { modulesByTargets[target] = components.cache.getAllModules(it.index) } + parameters.progressLogger?.invoke("Prepared new descriptors") + return CommonizationPerformed(modulesByTargets) } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt index 4fa0213ecc9..ec30237d079 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt @@ -48,24 +48,22 @@ class NativeDistributionCommonizer( RAW, AGGREGATED, NONE } + private val clockMark = ResettableClockMark() + fun run() { checkPreconditions() + clockMark.reset() - with(ResettableClockMark()) { - // 1. load libraries - val librariesByTargets = loadLibraries() - logger.log("* Loaded lazy (uninitialized) libraries in ${elapsedSinceLast()}") + // 1. load libraries + val librariesByTargets = loadLibraries() - // 2. run commonization - val result = commonize(librariesByTargets) - logger.log("* Commonization performed in ${elapsedSinceLast()}") + // 2. run commonization + val result = commonize(librariesByTargets) - // 3. write new libraries - saveModules(librariesByTargets, result) - logger.log("* Written libraries in ${elapsedSinceLast()}") + // 3. write new libraries + saveModules(librariesByTargets, result) - logger.log("TOTAL: ${elapsedSinceStart()}") - } + logTotal() } private fun checkPreconditions() { @@ -84,11 +82,15 @@ class NativeDistributionCommonizer( } } + private fun logProgress(message: String) = logger.log("* $message in ${clockMark.elapsedSinceLast()}") + + private fun logTotal() = logger.log("TOTAL: ${clockMark.elapsedSinceStart()}") + private fun loadLibraries(): Map { val stdlibPath = repository.resolve(konanCommonLibraryPath(KONAN_STDLIB_NAME)) val stdlib = loadLibrary(stdlibPath) - return targets.associate { target -> + val result = targets.associate { target -> val inputTarget = InputTarget(target.name, target) val platformLibs = inputTarget.platformLibrariesSource @@ -103,6 +105,10 @@ class NativeDistributionCommonizer( inputTarget to NativeDistributionLibraries(stdlib, platformLibs) } + + logProgress("Read lazy (uninitialized) libraries") + + return result } private fun loadLibrary(location: File): KotlinLibrary { @@ -137,7 +143,7 @@ class NativeDistributionCommonizer( NONE -> null } statsCollector.use { - val parameters = Parameters(statsCollector).apply { + val parameters = Parameters(statsCollector, ::logProgress).apply { librariesByTargets.forEach { (target, libraries) -> if (libraries.platformLibs.isEmpty()) return@forEach @@ -174,7 +180,9 @@ class NativeDistributionCommonizer( // It may happen that all targets to be commonized (or at least all but one target) miss platform libraries. // In such case commonizer will do nothing and return a special result value 'NothingToCommonize'. // So, let's just copy platform libraries from those target where they are to the new destination. - originalLibrariesByTargets.keys.forEach(::copyTargetAsIs) + originalLibrariesByTargets.forEach { (target, libraries) -> + copyTargetAsIs(target, libraries.platformLibs.size) + } } is CommonizationPerformed -> { @@ -186,18 +194,34 @@ class NativeDistributionCommonizer( // 'targetsToCopy' are some targets with empty set of platform libraries val targetsToCopy = originalLibrariesByTargets.keys - result.concreteTargets - targetsToCopy.forEach(::copyTargetAsIs) + if (targetsToCopy.isNotEmpty()) { + targetsToCopy.forEach { target -> + val libraries = originalLibrariesByTargets.getValue(target) + copyTargetAsIs(target, libraries.platformLibs.size) + } + } + + val concreteTargetNames = result.concreteTargets.map { it.name } val targetsToSerialize = result.concreteTargets + result.commonTarget targetsToSerialize.forEach { target -> val newModules = result.modulesByTargets.getValue(target) - val manifestProvider = when (target) { - is InputTarget -> originalLibrariesByTargets.getValue(target) - is OutputTarget -> CommonNativeManifestDataProvider(originalLibrariesByTargets.values) + val manifestProvider: NativeManifestDataProvider + val starredTarget: String? + when (target) { + is InputTarget -> { + manifestProvider = originalLibrariesByTargets.getValue(target) + starredTarget = target.name + } + is OutputTarget -> { + manifestProvider = CommonNativeManifestDataProvider(originalLibrariesByTargets.values) + starredTarget = null + } } - serializeTarget(target, newModules, manifestProvider, serializer) + val targetName = concreteTargetNames.joinToString { if (it == starredTarget) "$it(*)" else it } + serializeTarget(target, targetName, newModules, manifestProvider, serializer) } } } @@ -218,19 +242,29 @@ class NativeDistributionCommonizer( val libraryDestination = destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR).resolve(libraryOrigin.name) libraryOrigin.copyRecursively(libraryDestination) } + + val what = listOfNotNull( + "standard library".takeIf { copyStdlib }, + "endorsed libraries".takeIf { copyEndorsedLibs } + ).joinToString(separator = " and ") + + logProgress("Copied $what") } } - private fun copyTargetAsIs(target: InputTarget) { + private fun copyTargetAsIs(target: InputTarget, platformLibrariesCount: Int) { val librariesDestination = target.librariesDestination librariesDestination.mkdirs() // always create an empty directory even if there is nothing to copy val librariesSource = target.platformLibrariesSource if (librariesSource.isDirectory) librariesSource.copyRecursively(librariesDestination) + + logProgress("Copied of $platformLibrariesCount libraries for [${target.name}]") } private fun serializeTarget( target: Target, + targetName: String, newModules: Collection, manifestProvider: NativeManifestDataProvider, serializer: KlibMetadataMonolithicSerializer @@ -251,6 +285,8 @@ class NativeDistributionCommonizer( writeLibrary(metadata, manifestData, libraryDestination) } + + logProgress("Written libraries for [$targetName]") } private fun writeLibrary( diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt index 8860e80a312..952151b426b 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.descriptors.commonizer.mergedtree import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.descriptors.commonizer.Parameters import org.jetbrains.kotlin.descriptors.commonizer.TargetProvider import org.jetbrains.kotlin.descriptors.commonizer.cir.CirClass import org.jetbrains.kotlin.descriptors.commonizer.cir.factory.* @@ -20,17 +21,18 @@ import org.jetbrains.kotlin.storage.StorageManager class CirTreeMerger( private val storageManager: StorageManager, - private val targetProviders: List + private val parameters: Parameters ) { - private val size = targetProviders.size + private val size = parameters.targetProviders.size private lateinit var cacheRW: CirClassifiersCacheImpl fun merge(): CirRootNode { val rootNode: CirRootNode = buildRootNode(storageManager, size) cacheRW = rootNode.cache - targetProviders.forEachIndexed { targetIndex, targetProvider -> + parameters.targetProviders.forEachIndexed { targetIndex, targetProvider -> processTarget(rootNode, targetIndex, targetProvider) + parameters.progressLogger?.invoke("Loaded declarations for [${targetProvider.target.name}]") System.gc() } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/ResettableClockMark.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/ResettableClockMark.kt index c8c3c85d7e2..6ea204003a5 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/ResettableClockMark.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/ResettableClockMark.kt @@ -32,11 +32,16 @@ internal class ResettableClockMark { } } - private val startMark = currentTimeMillis() - private var lastMark = startMark + private var startMark: Long = 0 + private var lastMark: Long = 0 fun elapsedSinceLast(): Period = Period(lastMark, currentTimeMillis()).also { lastMark = it.end } fun elapsedSinceStart(): Period = Period(startMark, currentTimeMillis()) + + fun reset() { + startMark = currentTimeMillis() + lastMark = startMark + } } // TODO: this is how it should be when Kotlin Time will become non-experimental