[Commonizer] Logging: Implement ProgressLogger.fork

^KT-36679

native/commonizer/src/org/jetbrains/kotlin/commonizer/CommonizerParameters.kt

@@ -9,15 +9,16 @@ import org.jetbrains.kotlin.commonizer.konan.NativeManifestDataProvider
 import org.jetbrains.kotlin.commonizer.mergedtree.CirFictitiousFunctionClassifiers
 import org.jetbrains.kotlin.commonizer.mergedtree.CirProvidedClassifiers
 import org.jetbrains.kotlin.commonizer.stats.StatsCollector
+import org.jetbrains.kotlin.commonizer.utils.ProgressLogger
 
-class CommonizerParameters(
+data class CommonizerParameters(
     val outputTarget: SharedCommonizerTarget,
     val manifestProvider: TargetDependent<NativeManifestDataProvider>,
     val dependenciesProvider: TargetDependent<ModulesProvider?>,
     val targetProviders: TargetDependent<TargetProvider?>,
     val resultsConsumer: ResultsConsumer,
     val statsCollector: StatsCollector? = null,
-    val progressLogger: ((String) -> Unit)? = null,
+    val logger: ProgressLogger? = null,
 )
 
 internal fun CommonizerParameters.dependencyClassifiers(target: CommonizerTarget): CirProvidedClassifiers {
@@ -30,3 +31,5 @@ internal fun CommonizerParameters.dependencyClassifiers(target: CommonizerTarget
         CirProvidedClassifiers.of(classifiers, CirProvidedClassifiers.by(module))
     }
 }
+
+internal fun CommonizerParameters.with(logger: ProgressLogger?) = copy(logger = logger)

native/commonizer/src/org/jetbrains/kotlin/commonizer/ResultsConsumer.kt

@@ -31,19 +31,19 @@ interface ResultsConsumer {
     /**
      * Consume a single [ModuleResult] for the specified [CommonizerTarget].
      */
-    fun consume(target: CommonizerTarget, moduleResult: ModuleResult) = Unit
+    fun consume(parameters: CommonizerParameters, target: CommonizerTarget, moduleResult: ModuleResult) = Unit
 
     /**
      * Mark the specified [CommonizerTarget] as fully consumed.
      * It's forbidden to make subsequent [consume] calls for fully consumed targets.
      */
-    fun targetConsumed(target: CommonizerTarget) = Unit
+    fun targetConsumed(parameters: CommonizerParameters, target: CommonizerTarget) = Unit
 
     /**
      * Notify that all results have been consumed.
      * It's forbidden to make any subsequent [consume] and [targetConsumed] calls after this call.
      */
-    fun allConsumed(status: Status) = Unit
+    fun allConsumed(parameters: CommonizerParameters, status: Status) = Unit
 }
 
 internal class ResultsConsumerBuilder {
@@ -72,21 +72,21 @@ operator fun ResultsConsumer.plus(other: ResultsConsumer?): ResultsConsumer {
 }
 
 private class CompositeResultsConsumer(val consumers: List<ResultsConsumer>) : ResultsConsumer {
-    override fun consume(target: CommonizerTarget, moduleResult: ResultsConsumer.ModuleResult) {
+    override fun consume(parameters: CommonizerParameters, target: CommonizerTarget, moduleResult: ResultsConsumer.ModuleResult) {
         consumers.forEach { consumer ->
-            consumer.consume(target, moduleResult)
+            consumer.consume(parameters, target, moduleResult)
         }
     }
 
-    override fun targetConsumed(target: CommonizerTarget) {
+    override fun targetConsumed(parameters: CommonizerParameters, target: CommonizerTarget) {
         consumers.forEach { consumer ->
-            consumer.targetConsumed(target)
+            consumer.targetConsumed(parameters, target)
         }
     }
 
-    override fun allConsumed(status: ResultsConsumer.Status) {
+    override fun allConsumed(parameters: CommonizerParameters, status: ResultsConsumer.Status) {
         consumers.forEach { consumer ->
-            consumer.allConsumed(status)
+            consumer.allConsumed(parameters, status)
         }
     }
 }

native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/CliLoggerAdapter.kt

@@ -8,13 +8,15 @@ package org.jetbrains.kotlin.commonizer.cli
 import org.jetbrains.kotlin.util.Logger
 import kotlin.system.exitProcess
 
-internal class CliLoggerAdapter(indentSize: Int) : Logger {
+internal class CliLoggerAdapter(indentSize: Int = 0) : Logger {
     private val indent = " ".repeat(indentSize)
 
     override fun log(message: String) = printlnIndented(message)
+
     override fun warning(message: String) = printlnIndented("Warning: $message")
 
     override fun error(message: String) = fatal(message)
+
     override fun fatal(message: String): Nothing {
         printlnIndented("Error: $message\n")
         exitProcess(1)

native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/nativeTasks.kt

@@ -54,7 +54,7 @@ internal class NativeKlibCommonize(options: Collection<Option<*>>) : Task(option
         val konanTargets = outputCommonizerTarget.konanTargets
         val commonizerTargets = konanTargets.map(::CommonizerTarget)
 
-        val progressLogger = ProgressLogger(CliLoggerAdapter(2), startImmediately = true)
+        val progressLogger = ProgressLogger(CliLoggerAdapter(2))
         val libraryLoader = DefaultNativeLibraryLoader(progressLogger)
         val statsCollector = StatsCollector(statsType, commonizerTargets)
         val repository = FilesRepository(targetLibraries.toSet(), libraryLoader)
@@ -62,9 +62,9 @@ internal class NativeKlibCommonize(options: Collection<Option<*>>) : Task(option
         val resultsConsumer = buildResultsConsumer {
             this add ModuleSerializer(destination, HierarchicalCommonizerOutputLayout)
             this add CopyUnconsumedModulesAsIsConsumer(
-                repository, destination, commonizerTargets.toSet(), NativeDistributionCommonizerOutputLayout, progressLogger
+                repository, destination, commonizerTargets.toSet(), NativeDistributionCommonizerOutputLayout
             )
-            this add LoggingResultsConsumer(outputCommonizerTarget, progressLogger)
+            this add LoggingResultsConsumer(outputCommonizerTarget)
         }
 
         LibraryCommonizer(
@@ -98,17 +98,17 @@ internal class NativeDistributionCommonize(options: Collection<Option<*>>) : Tas
         val copyEndorsedLibs = getOptional<Boolean, BooleanOptionType> { it == "copy-endorsed-libs" } ?: false
         val statsType = getOptional<StatsType, StatsTypeOptionType> { it == "log-stats" } ?: StatsType.NONE
 
-        val progressLogger = ProgressLogger(CliLoggerAdapter(2), startImmediately = true)
+        val progressLogger = ProgressLogger(CliLoggerAdapter(2))
         val libraryLoader = DefaultNativeLibraryLoader(progressLogger)
         val repository = KonanDistributionRepository(distribution, outputTarget.konanTargets, libraryLoader)
         val statsCollector = StatsCollector(statsType, outputTarget.withAllAncestors().toList())
 
         val resultsConsumer = buildResultsConsumer {
             this add ModuleSerializer(destination, outputLayout)
-            this add CopyUnconsumedModulesAsIsConsumer(repository, destination, outputTarget.allLeaves(), outputLayout, progressLogger)
+            this add CopyUnconsumedModulesAsIsConsumer(repository, destination, outputTarget.allLeaves(), outputLayout)
-            if (copyStdlib) this add CopyStdlibResultsConsumer(distribution, destination, progressLogger)
+            if (copyStdlib) this add CopyStdlibResultsConsumer(distribution, destination)
-            if (copyEndorsedLibs) this add CopyEndorsedLibrairesResultsConsumer(distribution, destination, progressLogger)
+            if (copyEndorsedLibs) this add CopyEndorsedLibrairesResultsConsumer(distribution, destination)
-            this add LoggingResultsConsumer(outputTarget, progressLogger)
+            this add LoggingResultsConsumer(outputTarget)
         }
 
         val descriptionSuffix = estimateLibrariesCount(repository, outputTarget.allLeaves()).let { " ($it items)" }

native/commonizer/src/org/jetbrains/kotlin/commonizer/facade.kt

@@ -26,13 +26,13 @@ import org.jetbrains.kotlin.storage.StorageManager
 
 fun runCommonization(parameters: CommonizerParameters) {
     if (!parameters.containsCommonModuleNames()) {
-        parameters.resultsConsumer.allConsumed(Status.NOTHING_TO_DO)
+        parameters.resultsConsumer.allConsumed(parameters, Status.NOTHING_TO_DO)
         return
     }
 
     val storageManager = LockBasedStorageManager("Declarations commonization")
     commonize(parameters, storageManager, parameters.outputTarget)
-    parameters.resultsConsumer.allConsumed(Status.DONE)
+    parameters.resultsConsumer.allConsumed(parameters, Status.DONE)
 }
 
 private fun commonize(
@@ -41,7 +41,7 @@ private fun commonize(
     target: SharedCommonizerTarget,
 ): CirRootNode? {
     val cirTrees = getCirTree(parameters, storageManager, target)
-    parameters.progressLogger?.invoke("Build cir tree for $target")
+    parameters.logProgress("Built declaration tree for $target")
 
     // build merged tree:
     val classifiers = CirKnownClassifiers(
@@ -51,10 +51,9 @@ private fun commonize(
 
     val mergedTree = merge(storageManager, classifiers, cirTrees) ?: return null
     mergedTree.accept(CommonizationVisitor(classifiers, mergedTree), Unit)
-    parameters.progressLogger?.invoke("Commonized declarations for $target")
+    parameters.logProgress("Commonized declarations for $target")
 
     serialize(parameters, mergedTree, target)
-    parameters.progressLogger?.invoke("Commonized target $target")
 
     return mergedTree
 }
@@ -65,7 +64,9 @@ private fun getCirTree(
     return EagerTargetDependent(target.targets) { childTarget ->
         when (childTarget) {
             is LeafCommonizerTarget -> deserialize(parameters, childTarget)
-            is SharedCommonizerTarget -> commonize(parameters, storageManager, childTarget)?.assembleCirTree()
+            is SharedCommonizerTarget -> commonize(parameters.fork(), storageManager, childTarget)?.assembleCirTree().also {
+                parameters.logProgress("Commonized target $childTarget")
+            }
         }
     }
 }
@@ -103,9 +104,9 @@ private fun serialize(parameters: CommonizerParameters, mergedTree: CirRootNode,
             SerializedMetadata(header, fragments, fragmentNames)
         }
         val manifestData = parameters.manifestProvider[target].buildManifest(libraryName)
-        parameters.resultsConsumer.consume(target, ModuleResult.Commonized(libraryName, serializedMetadata, manifestData))
+        parameters.resultsConsumer.consume(parameters, target, ModuleResult.Commonized(libraryName, serializedMetadata, manifestData))
     }
-    parameters.resultsConsumer.targetConsumed(target)
+    parameters.resultsConsumer.targetConsumed(parameters, target)
 }
 
 private fun serializeMissingModules(parameters: CommonizerParameters, requestedTarget: CommonizerTarget) {
@@ -114,8 +115,14 @@ private fun serializeMissingModules(parameters: CommonizerParameters, requestedT
 
     targetProvider.modulesProvider.loadModuleInfos()
         .filter { it.name !in commonModuleNames }
-        .forEach { missingModule -> parameters.resultsConsumer.consume(requestedTarget, Missing(missingModule.originalLocation)) }
+        .forEach { missingModule ->
-
+            parameters.resultsConsumer.consume(parameters, requestedTarget, Missing(missingModule.originalLocation))
+        }
 }
 
+private fun CommonizerParameters.fork(): CommonizerParameters = with(logger?.fork())
+
+private fun CommonizerParameters.logProgress(message: String) = logger?.progress(message)
+
 private val KLIB_FRAGMENT_WRITE_STRATEGY = ChunkedKlibModuleFragmentWriteStrategy()
+

native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/CopyLibrariesFromKonanDistributionResultsConsumer.kt

@@ -7,23 +7,22 @@
 
 package org.jetbrains.kotlin.commonizer.konan
 
+import org.jetbrains.kotlin.commonizer.CommonizerParameters
 import org.jetbrains.kotlin.commonizer.KonanDistribution
 import org.jetbrains.kotlin.commonizer.ResultsConsumer
 import org.jetbrains.kotlin.commonizer.klibDir
 import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMON_LIBS_DIR
 import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME
-import org.jetbrains.kotlin.util.Logger
 import java.io.File
 
 internal fun CopyStdlibResultsConsumer(
     konanDistribution: KonanDistribution,
-    destination: File,
+    destination: File
-    logger: Logger
 ): ResultsConsumer {
     return CopyLibrariesFromKonanDistributionResultsConsumer(
         konanDistribution,
         destination,
-        invokeWhenCopied = { logger.log("Copied standard library") },
+        invokeWhenCopied = { logger?.progress("Copied standard library") },
         copyFileIf = { file -> file.endsWith(KONAN_STDLIB_NAME) }
     )
 }
@@ -31,12 +30,11 @@ internal fun CopyStdlibResultsConsumer(
 internal fun CopyEndorsedLibrairesResultsConsumer(
     konanDistribution: KonanDistribution,
     destination: File,
-    logger: Logger
 ): ResultsConsumer {
     return CopyLibrariesFromKonanDistributionResultsConsumer(
         konanDistribution,
         destination,
-        invokeWhenCopied = { logger.log("Copied endorsed libraries") },
+        invokeWhenCopied = { logger?.progress("Copied endorsed libraries") },
         copyFileIf = { file -> !file.endsWith(KONAN_STDLIB_NAME) }
     )
 }
@@ -44,10 +42,10 @@ internal fun CopyEndorsedLibrairesResultsConsumer(
 private class CopyLibrariesFromKonanDistributionResultsConsumer(
     private val konanDistribution: KonanDistribution,
     private val destination: File,
-    private val invokeWhenCopied: () -> Unit = {},
+    private val invokeWhenCopied: CommonizerParameters.() -> Unit = {},
     private val copyFileIf: (File) -> Boolean = { true }
 ) : ResultsConsumer {
-    override fun allConsumed(status: ResultsConsumer.Status) {
+    override fun allConsumed(parameters: CommonizerParameters, status: ResultsConsumer.Status) {
         konanDistribution.klibDir
             .resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR)
             .listFiles().orEmpty()
@@ -57,6 +55,6 @@ private class CopyLibrariesFromKonanDistributionResultsConsumer(
                 val libraryDestination = destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR).resolve(libraryOrigin.name)
                 libraryOrigin.copyRecursively(libraryDestination)
             }
-        invokeWhenCopied()
+        parameters.invokeWhenCopied()
     }
 }

native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/CopyUnconsumedModulesAsIsConsumer.kt

@@ -7,7 +7,6 @@ package org.jetbrains.kotlin.commonizer.konan
 
 import org.jetbrains.kotlin.commonizer.*
 import org.jetbrains.kotlin.commonizer.repository.Repository
-import org.jetbrains.kotlin.util.Logger
 import java.io.File
 
 internal class CopyUnconsumedModulesAsIsConsumer(
@@ -15,25 +14,24 @@ internal class CopyUnconsumedModulesAsIsConsumer(
     private val destination: File,
     private val targets: Set<LeafCommonizerTarget>,
     private val outputLayout: CommonizerOutputLayout,
-    private val logger: Logger? = null
 ) : ResultsConsumer {
 
     private val consumedTargets = mutableSetOf<LeafCommonizerTarget>()
 
-    override fun targetConsumed(target: CommonizerTarget) {
+    override fun targetConsumed(parameters: CommonizerParameters, target: CommonizerTarget) {
         if (target is LeafCommonizerTarget) {
             consumedTargets += target
         }
     }
 
-    override fun allConsumed(status: ResultsConsumer.Status) {
+    override fun allConsumed(parameters: CommonizerParameters, status: ResultsConsumer.Status) {
         when (status) {
-            ResultsConsumer.Status.NOTHING_TO_DO -> targets.forEach(::copyTargetAsIs)
+            ResultsConsumer.Status.NOTHING_TO_DO -> targets.forEach { target -> copyTargetAsIs(parameters, target) }
-            ResultsConsumer.Status.DONE -> targets.minus(consumedTargets).forEach(::copyTargetAsIs)
+            ResultsConsumer.Status.DONE -> targets.minus(consumedTargets).forEach { target -> copyTargetAsIs(parameters, target) }
         }
     }
 
-    private fun copyTargetAsIs(target: LeafCommonizerTarget) {
+    private fun copyTargetAsIs(parameters: CommonizerParameters, target: LeafCommonizerTarget) {
         val libraries = repository.getLibraries(target)
         val librariesDestination = outputLayout.getTargetDirectory(destination, target)
         librariesDestination.mkdirs() // always create an empty directory even if there is nothing to copy
@@ -41,6 +39,6 @@ internal class CopyUnconsumedModulesAsIsConsumer(
             libraryFile.copyRecursively(destination.resolve(libraryFile.name))
         }
 
-        logger?.log("Copied ${libraries.size} libraries for ${target.prettyName}")
+        parameters.logger?.progress("Copied ${libraries.size} libraries for ${target.prettyName}")
     }
 }

native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/LibraryCommonizer.kt

@@ -39,7 +39,7 @@ internal class LibraryCommonizer internal constructor(
                 )
         }
 
-        progressLogger.log("Resolved libraries to be commonized")
+        progressLogger.progress("Resolved libraries to be commonized")
         return libraries
     }
 
@@ -51,7 +51,7 @@ internal class LibraryCommonizer internal constructor(
             dependenciesProvider = createDependenciesProvider(),
             resultsConsumer = resultsConsumer,
             statsCollector = statsCollector,
-            progressLogger = progressLogger::log
+            logger = progressLogger
         )
         runCommonization(parameters)
     }

native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/LoggingResultsConsumer.kt

@@ -5,16 +5,12 @@
 
 package org.jetbrains.kotlin.commonizer.konan
 
-import org.jetbrains.kotlin.commonizer.CommonizerTarget
+import org.jetbrains.kotlin.commonizer.*
-import org.jetbrains.kotlin.commonizer.ResultsConsumer
-import org.jetbrains.kotlin.commonizer.SharedCommonizerTarget
-import org.jetbrains.kotlin.commonizer.prettyName
-import org.jetbrains.kotlin.util.Logger
 
 internal class LoggingResultsConsumer(
-    private val outputCommonizerTarget: SharedCommonizerTarget, private val logger: Logger
+    private val outputCommonizerTarget: SharedCommonizerTarget
 ) : ResultsConsumer {
-    override fun targetConsumed(target: CommonizerTarget) {
+    override fun targetConsumed(parameters: CommonizerParameters, target: CommonizerTarget) {
-        logger.log("Written libraries for ${outputCommonizerTarget.prettyName(target)}")
+        parameters.logger?.progress("Written libraries for ${outputCommonizerTarget.prettyName(target)}")
     }
 }

native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/ModuleSerializer.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.commonizer.konan
 
 import org.jetbrains.kotlin.commonizer.CommonizerOutputLayout
+import org.jetbrains.kotlin.commonizer.CommonizerParameters
 import org.jetbrains.kotlin.commonizer.CommonizerTarget
 import org.jetbrains.kotlin.commonizer.ResultsConsumer
 import org.jetbrains.kotlin.library.SerializedMetadata
@@ -19,7 +20,7 @@ internal class ModuleSerializer(
     private val destination: File,
     private val outputLayout: CommonizerOutputLayout,
 ) : ResultsConsumer {
-    override fun consume(target: CommonizerTarget, moduleResult: ResultsConsumer.ModuleResult) {
+    override fun consume(parameters: CommonizerParameters, target: CommonizerTarget, moduleResult: ResultsConsumer.ModuleResult) {
         val librariesDestination = outputLayout.getTargetDirectory(destination, target)
         when (moduleResult) {
             is ResultsConsumer.ModuleResult.Commonized -> {

native/commonizer/src/org/jetbrains/kotlin/commonizer/utils/ProgressLogger.kt

@@ -5,28 +5,26 @@
 
 package org.jetbrains.kotlin.commonizer.utils
 
+import org.jetbrains.kotlin.commonizer.cli.CliLoggerAdapter
 import org.jetbrains.kotlin.util.Logger
 
 class ProgressLogger(
-    private val wrapped: Logger,
+    private val wrapped: Logger = CliLoggerAdapter(0),
-    startImmediately: Boolean = false
+    private val indent: Int = 0,
 ) : Logger by wrapped {
     private val clockMark = ResettableClockMark()
-    private var finished = true
+    private var finished = false
+
+    private val prefix = "  ".repeat(indent) + " * "
 
     init {
-        if (startImmediately)
-            reset()
-    }
-
-    fun reset() {
         clockMark.reset()
-        finished = false
+        require(indent >= 0) { "Required indent >= 1" }
     }
 
-    override fun log(message: String) {
+    fun progress(message: String) {
         check(!finished)
-        wrapped.log("* $message in ${clockMark.elapsedSinceLast()}")
+        wrapped.log("$prefix$message in ${clockMark.elapsedSinceLast()}")
     }
 
     fun logTotal() {
@@ -34,4 +32,8 @@ class ProgressLogger(
         wrapped.log("TOTAL: ${clockMark.elapsedSinceStart()}")
         finished = true
     }
+
+    fun fork(): ProgressLogger {
+        return ProgressLogger(this, indent + 1)
+    }
 }

native/commonizer/tests/org/jetbrains/kotlin/commonizer/utils/mocks.kt

@@ -146,7 +146,7 @@ internal class MockResultsConsumer : ResultsConsumer {
 
     lateinit var status: ResultsConsumer.Status
 
-    override fun consume(target: CommonizerTarget, moduleResult: ModuleResult) {
+    override fun consume(parameters: CommonizerParameters, target: CommonizerTarget, moduleResult: ModuleResult) {
         check(!this::status.isInitialized)
         check(target !in finishedTargets) { "$target already finished" }
         val moduleResults: ModuleResults = _modulesByTargets.getOrPut(target) { ModuleResults() }
@@ -154,13 +154,13 @@ internal class MockResultsConsumer : ResultsConsumer {
         check(oldResult == null) // to avoid accidental overwriting
     }
 
-    override fun targetConsumed(target: CommonizerTarget) {
+    override fun targetConsumed(parameters: CommonizerParameters, target: CommonizerTarget) {
         check(!this::status.isInitialized)
         check(target !in finishedTargets)
         finishedTargets += target
     }
 
-    override fun allConsumed(status: ResultsConsumer.Status) {
+    override fun allConsumed(parameters: CommonizerParameters, status: ResultsConsumer.Status) {
         check(!this::status.isInitialized)
         check(finishedTargets.containsAll(_modulesByTargets.keys))
         this.status = status