[Commonizer] More detailed progress logging

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/Parameters.kt

@@ -8,7 +8,8 @@ package org.jetbrains.kotlin.descriptors.commonizer
 import org.jetbrains.kotlin.descriptors.commonizer.stats.StatsCollector
 
 class Parameters(
-    val statsCollector: StatsCollector? = null
+    val statsCollector: StatsCollector? = null,
+    val progressLogger: ((String) -> Unit)? = null
 ) {
     // use linked hash map to preserve order
     private val _targetProviders = LinkedHashMap<InputTarget, TargetProvider>()

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt

@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.descriptors.commonizer.builder
 
 import org.jetbrains.kotlin.builtins.KotlinBuiltIns
 import org.jetbrains.kotlin.descriptors.*
+import org.jetbrains.kotlin.descriptors.commonizer.Parameters
 import org.jetbrains.kotlin.descriptors.commonizer.Target
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirNode.Companion.dimension
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirNode.Companion.indexOfCommon
@@ -166,7 +167,7 @@ class TargetDeclarationsBuilderComponents(
 
 fun CirRootNode.createGlobalBuilderComponents(
     storageManager: StorageManager,
-    statsCollector: StatsCollector?
+    parameters: Parameters
 ): GlobalDeclarationsBuilderComponents {
     val cache = DeclarationsBuilderCache(dimension)
 
@@ -189,7 +190,7 @@ fun CirRootNode.createGlobalBuilderComponents(
         )
     }
 
-    return GlobalDeclarationsBuilderComponents(storageManager, targetContexts, cache, statsCollector)
+    return GlobalDeclarationsBuilderComponents(storageManager, targetContexts, cache, parameters.statsCollector)
 }
 
 interface TypeParameterResolver {

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeTasks.kt

@@ -47,8 +47,9 @@ internal class NativeDistributionCommonize(options: Collection<Option<*>>) : Tas
         val copyEndorsedLibs = getOptional<Boolean, BooleanOptionType> { it == "copy-endorsed-libs" } ?: false
         val statsType = getOptional<StatsType, StatsTypeOptionType> { it == "log-stats" } ?: StatsType.NONE
 
+        val targetNames = targets.joinToString { "[${it.name}]" }
         val descriptionSuffix = estimateLibrariesCount(distribution, targets)?.let { " ($it items)" } ?: ""
-        val description = "${logPrefix}Preparing commonized Kotlin/Native libraries for targets $targets$descriptionSuffix"
+        val description = "${logPrefix}Preparing commonized Kotlin/Native libraries for targets $targetNames$descriptionSuffix"
 
         println(description)
 

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt

@@ -20,13 +20,14 @@ fun runCommonization(parameters: Parameters): Result {
     val storageManager = LockBasedStorageManager("Declaration descriptors commonization")
 
     // build merged tree:
-    val mergedTree = CirTreeMerger(storageManager, parameters.targetProviders).merge()
+    val mergedTree = CirTreeMerger(storageManager, parameters).merge()
 
     // commonize:
     mergedTree.accept(CommonizationVisitor(mergedTree), Unit)
+    parameters.progressLogger?.invoke("Commonized declarations")
 
     // build resulting descriptors:
-    val components = mergedTree.createGlobalBuilderComponents(storageManager, parameters.statsCollector)
+    val components = mergedTree.createGlobalBuilderComponents(storageManager, parameters)
     mergedTree.accept(DeclarationsBuilderVisitor1(components), emptyList())
     mergedTree.accept(DeclarationsBuilderVisitor2(components), emptyList())
 
@@ -38,5 +39,7 @@ fun runCommonization(parameters: Parameters): Result {
         modulesByTargets[target] = components.cache.getAllModules(it.index)
     }
 
+    parameters.progressLogger?.invoke("Prepared new descriptors")
+
     return CommonizationPerformed(modulesByTargets)
 }

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt

@@ -48,24 +48,22 @@ class NativeDistributionCommonizer(
         RAW, AGGREGATED, NONE
     }
 
+    private val clockMark = ResettableClockMark()
+
     fun run() {
         checkPreconditions()
+        clockMark.reset()
 
-        with(ResettableClockMark()) {
+        // 1. load libraries
-            // 1. load libraries
+        val librariesByTargets = loadLibraries()
-            val librariesByTargets = loadLibraries()
-            logger.log("* Loaded lazy (uninitialized) libraries in ${elapsedSinceLast()}")
 
-            // 2. run commonization
+        // 2. run commonization
-            val result = commonize(librariesByTargets)
+        val result = commonize(librariesByTargets)
-            logger.log("* Commonization performed in ${elapsedSinceLast()}")
 
-            // 3. write new libraries
+        // 3. write new libraries
-            saveModules(librariesByTargets, result)
+        saveModules(librariesByTargets, result)
-            logger.log("* Written libraries in ${elapsedSinceLast()}")
 
-            logger.log("TOTAL: ${elapsedSinceStart()}")
+        logTotal()
-        }
     }
 
     private fun checkPreconditions() {
@@ -84,11 +82,15 @@ class NativeDistributionCommonizer(
         }
     }
 
+    private fun logProgress(message: String) = logger.log("* $message in ${clockMark.elapsedSinceLast()}")
+
+    private fun logTotal() = logger.log("TOTAL: ${clockMark.elapsedSinceStart()}")
+
     private fun loadLibraries(): Map<InputTarget, NativeDistributionLibraries> {
         val stdlibPath = repository.resolve(konanCommonLibraryPath(KONAN_STDLIB_NAME))
         val stdlib = loadLibrary(stdlibPath)
 
-        return targets.associate { target ->
+        val result = targets.associate { target ->
             val inputTarget = InputTarget(target.name, target)
 
             val platformLibs = inputTarget.platformLibrariesSource
@@ -103,6 +105,10 @@ class NativeDistributionCommonizer(
 
             inputTarget to NativeDistributionLibraries(stdlib, platformLibs)
         }
+
+        logProgress("Read lazy (uninitialized) libraries")
+
+        return result
     }
 
     private fun loadLibrary(location: File): KotlinLibrary {
@@ -137,7 +143,7 @@ class NativeDistributionCommonizer(
             NONE -> null
         }
         statsCollector.use {
-            val parameters = Parameters(statsCollector).apply {
+            val parameters = Parameters(statsCollector, ::logProgress).apply {
                 librariesByTargets.forEach { (target, libraries) ->
                     if (libraries.platformLibs.isEmpty()) return@forEach
 
@@ -174,7 +180,9 @@ class NativeDistributionCommonizer(
                 // It may happen that all targets to be commonized (or at least all but one target) miss platform libraries.
                 // In such case commonizer will do nothing and return a special result value 'NothingToCommonize'.
                 // So, let's just copy platform libraries from those target where they are to the new destination.
-                originalLibrariesByTargets.keys.forEach(::copyTargetAsIs)
+                originalLibrariesByTargets.forEach { (target, libraries) ->
+                    copyTargetAsIs(target, libraries.platformLibs.size)
+                }
             }
 
             is CommonizationPerformed -> {
@@ -186,18 +194,34 @@ class NativeDistributionCommonizer(
 
                 // 'targetsToCopy' are some targets with empty set of platform libraries
                 val targetsToCopy = originalLibrariesByTargets.keys - result.concreteTargets
-                targetsToCopy.forEach(::copyTargetAsIs)
+                if (targetsToCopy.isNotEmpty()) {
+                    targetsToCopy.forEach { target ->
+                        val libraries = originalLibrariesByTargets.getValue(target)
+                        copyTargetAsIs(target, libraries.platformLibs.size)
+                    }
+                }
 
+
+                val concreteTargetNames = result.concreteTargets.map { it.name }
                 val targetsToSerialize = result.concreteTargets + result.commonTarget
                 targetsToSerialize.forEach { target ->
                     val newModules = result.modulesByTargets.getValue(target)
 
-                    val manifestProvider = when (target) {
+                    val manifestProvider: NativeManifestDataProvider
-                        is InputTarget -> originalLibrariesByTargets.getValue(target)
+                    val starredTarget: String?
-                        is OutputTarget -> CommonNativeManifestDataProvider(originalLibrariesByTargets.values)
+                    when (target) {
+                        is InputTarget -> {
+                            manifestProvider = originalLibrariesByTargets.getValue(target)
+                            starredTarget = target.name
+                        }
+                        is OutputTarget -> {
+                            manifestProvider = CommonNativeManifestDataProvider(originalLibrariesByTargets.values)
+                            starredTarget = null
+                        }
                     }
 
-                    serializeTarget(target, newModules, manifestProvider, serializer)
+                    val targetName = concreteTargetNames.joinToString { if (it == starredTarget) "$it(*)" else it }
+                    serializeTarget(target, targetName, newModules, manifestProvider, serializer)
                 }
             }
         }
@@ -218,19 +242,29 @@ class NativeDistributionCommonizer(
                     val libraryDestination = destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR).resolve(libraryOrigin.name)
                     libraryOrigin.copyRecursively(libraryDestination)
                 }
+
+            val what = listOfNotNull(
+                "standard library".takeIf { copyStdlib },
+                "endorsed libraries".takeIf { copyEndorsedLibs }
+            ).joinToString(separator = " and ")
+
+            logProgress("Copied $what")
         }
     }
 
-    private fun copyTargetAsIs(target: InputTarget) {
+    private fun copyTargetAsIs(target: InputTarget, platformLibrariesCount: Int) {
         val librariesDestination = target.librariesDestination
         librariesDestination.mkdirs() // always create an empty directory even if there is nothing to copy
 
         val librariesSource = target.platformLibrariesSource
         if (librariesSource.isDirectory) librariesSource.copyRecursively(librariesDestination)
+
+        logProgress("Copied of $platformLibrariesCount libraries for [${target.name}]")
     }
 
     private fun serializeTarget(
         target: Target,
+        targetName: String,
         newModules: Collection<ModuleDescriptor>,
         manifestProvider: NativeManifestDataProvider,
         serializer: KlibMetadataMonolithicSerializer
@@ -251,6 +285,8 @@ class NativeDistributionCommonizer(
 
             writeLibrary(metadata, manifestData, libraryDestination)
         }
+
+        logProgress("Written libraries for [$targetName]")
     }
 
     private fun writeLibrary(

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.descriptors.commonizer.mergedtree
 
 import org.jetbrains.kotlin.descriptors.*
+import org.jetbrains.kotlin.descriptors.commonizer.Parameters
 import org.jetbrains.kotlin.descriptors.commonizer.TargetProvider
 import org.jetbrains.kotlin.descriptors.commonizer.cir.CirClass
 import org.jetbrains.kotlin.descriptors.commonizer.cir.factory.*
@@ -20,17 +21,18 @@ import org.jetbrains.kotlin.storage.StorageManager
 
 class CirTreeMerger(
     private val storageManager: StorageManager,
-    private val targetProviders: List<TargetProvider>
+    private val parameters: Parameters
 ) {
-    private val size = targetProviders.size
+    private val size = parameters.targetProviders.size
     private lateinit var cacheRW: CirClassifiersCacheImpl
 
     fun merge(): CirRootNode {
         val rootNode: CirRootNode = buildRootNode(storageManager, size)
         cacheRW = rootNode.cache
 
-        targetProviders.forEachIndexed { targetIndex, targetProvider ->
+        parameters.targetProviders.forEachIndexed { targetIndex, targetProvider ->
             processTarget(rootNode, targetIndex, targetProvider)
+            parameters.progressLogger?.invoke("Loaded declarations for [${targetProvider.target.name}]")
             System.gc()
         }
 

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/ResettableClockMark.kt

@@ -32,11 +32,16 @@ internal class ResettableClockMark {
         }
     }
 
-    private val startMark = currentTimeMillis()
+    private var startMark: Long = 0
-    private var lastMark = startMark
+    private var lastMark: Long = 0
 
     fun elapsedSinceLast(): Period = Period(lastMark, currentTimeMillis()).also { lastMark = it.end }
     fun elapsedSinceStart(): Period = Period(startMark, currentTimeMillis())
+
+    fun reset() {
+        startMark = currentTimeMillis()
+        lastMark = startMark
+    }
 }
 
 // TODO: this is how it should be when Kotlin Time will become non-experimental