azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Commonizer] More detailed progress logging

Browse Source
This commit is contained in:
Dmitriy Dolovov
2020-06-18 18:08:00 +07:00
parent 6410aed1b4
commit 68e1acd2cb
7 changed files with 81 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/Parameters.kt
+2 -1
View File
@@ -8,7 +8,8 @@ package org.jetbrains.kotlin.descriptors.commonizer
import org.jetbrains.kotlin.descriptors.commonizer.stats.StatsCollector
class Parameters(
val statsCollector: StatsCollector? = null
val statsCollector: StatsCollector? = null,
val progressLogger: ((String) -> Unit)? = null
) {
// use linked hash map to preserve order
private val _targetProviders = LinkedHashMap<InputTarget, TargetProvider>()
native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt
+3 -2
View File
@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.descriptors.commonizer.builder
import org.jetbrains.kotlin.builtins.KotlinBuiltIns
import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.descriptors.commonizer.Parameters
import org.jetbrains.kotlin.descriptors.commonizer.Target
import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirNode.Companion.dimension
import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirNode.Companion.indexOfCommon
@@ -166,7 +167,7 @@ class TargetDeclarationsBuilderComponents(
fun CirRootNode.createGlobalBuilderComponents(
storageManager: StorageManager,
statsCollector: StatsCollector?
parameters: Parameters
): GlobalDeclarationsBuilderComponents {
val cache = DeclarationsBuilderCache(dimension)
@@ -189,7 +190,7 @@ fun CirRootNode.createGlobalBuilderComponents(
)
}
return GlobalDeclarationsBuilderComponents(storageManager, targetContexts, cache, statsCollector)
return GlobalDeclarationsBuilderComponents(storageManager, targetContexts, cache, parameters.statsCollector)
}
interface TypeParameterResolver {
native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeTasks.kt
+2 -1
View File
@@ -47,8 +47,9 @@ internal class NativeDistributionCommonize(options: Collection<Option<*>>) : Tas
val copyEndorsedLibs = getOptional<Boolean, BooleanOptionType> { it == "copy-endorsed-libs" } ?: false
val statsType = getOptional<StatsType, StatsTypeOptionType> { it == "log-stats" } ?: StatsType.NONE
val targetNames = targets.joinToString { "[${it.name}]" }
val descriptionSuffix = estimateLibrariesCount(distribution, targets)?.let { " ($it items)" } ?: ""
val description = "${logPrefix}Preparing commonized Kotlin/Native libraries for targets $targets$descriptionSuffix"
val description = "${logPrefix}Preparing commonized Kotlin/Native libraries for targets $targetNames$descriptionSuffix"
println(description)
native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt
+5 -2
View File
@@ -20,13 +20,14 @@ fun runCommonization(parameters: Parameters): Result {
val storageManager = LockBasedStorageManager("Declaration descriptors commonization")
// build merged tree:
val mergedTree = CirTreeMerger(storageManager, parameters.targetProviders).merge()
val mergedTree = CirTreeMerger(storageManager, parameters).merge()
// commonize:
mergedTree.accept(CommonizationVisitor(mergedTree), Unit)
parameters.progressLogger?.invoke("Commonized declarations")
// build resulting descriptors:
val components = mergedTree.createGlobalBuilderComponents(storageManager, parameters.statsCollector)
val components = mergedTree.createGlobalBuilderComponents(storageManager, parameters)
mergedTree.accept(DeclarationsBuilderVisitor1(components), emptyList())
mergedTree.accept(DeclarationsBuilderVisitor2(components), emptyList())
@@ -38,5 +39,7 @@ fun runCommonization(parameters: Parameters): Result {
modulesByTargets[target] = components.cache.getAllModules(it.index)
}
parameters.progressLogger?.invoke("Prepared new descriptors")
return CommonizationPerformed(modulesByTargets)
}
native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt
+57 -21
View File
@@ -48,24 +48,22 @@ class NativeDistributionCommonizer(
RAW, AGGREGATED, NONE
}
private val clockMark = ResettableClockMark()
fun run() {
checkPreconditions()
clockMark.reset()
with(ResettableClockMark()) {
// 1. load libraries
val librariesByTargets = loadLibraries()
logger.log("* Loaded lazy (uninitialized) libraries in ${elapsedSinceLast()}")
// 1. load libraries
val librariesByTargets = loadLibraries()
// 2. run commonization
val result = commonize(librariesByTargets)
logger.log("* Commonization performed in ${elapsedSinceLast()}")
// 2. run commonization
val result = commonize(librariesByTargets)
// 3. write new libraries
saveModules(librariesByTargets, result)
logger.log("* Written libraries in ${elapsedSinceLast()}")
// 3. write new libraries
saveModules(librariesByTargets, result)
logger.log("TOTAL: ${elapsedSinceStart()}")
}
logTotal()
}
private fun checkPreconditions() {
@@ -84,11 +82,15 @@ class NativeDistributionCommonizer(
}
}
private fun logProgress(message: String) = logger.log("* $message in ${clockMark.elapsedSinceLast()}")
private fun logTotal() = logger.log("TOTAL: ${clockMark.elapsedSinceStart()}")
private fun loadLibraries(): Map<InputTarget, NativeDistributionLibraries> {
val stdlibPath = repository.resolve(konanCommonLibraryPath(KONAN_STDLIB_NAME))
val stdlib = loadLibrary(stdlibPath)
return targets.associate { target ->
val result = targets.associate { target ->
val inputTarget = InputTarget(target.name, target)
val platformLibs = inputTarget.platformLibrariesSource
@@ -103,6 +105,10 @@ class NativeDistributionCommonizer(
inputTarget to NativeDistributionLibraries(stdlib, platformLibs)
}
logProgress("Read lazy (uninitialized) libraries")
return result
}
private fun loadLibrary(location: File): KotlinLibrary {
@@ -137,7 +143,7 @@ class NativeDistributionCommonizer(
NONE -> null
}
statsCollector.use {
val parameters = Parameters(statsCollector).apply {
val parameters = Parameters(statsCollector, ::logProgress).apply {
librariesByTargets.forEach { (target, libraries) ->
if (libraries.platformLibs.isEmpty()) return@forEach
@@ -174,7 +180,9 @@ class NativeDistributionCommonizer(
// It may happen that all targets to be commonized (or at least all but one target) miss platform libraries.
// In such case commonizer will do nothing and return a special result value 'NothingToCommonize'.
// So, let's just copy platform libraries from those target where they are to the new destination.
originalLibrariesByTargets.keys.forEach(::copyTargetAsIs)
originalLibrariesByTargets.forEach { (target, libraries) ->
copyTargetAsIs(target, libraries.platformLibs.size)
}
}
is CommonizationPerformed -> {
@@ -186,18 +194,34 @@ class NativeDistributionCommonizer(
// 'targetsToCopy' are some targets with empty set of platform libraries
val targetsToCopy = originalLibrariesByTargets.keys - result.concreteTargets
targetsToCopy.forEach(::copyTargetAsIs)
if (targetsToCopy.isNotEmpty()) {
targetsToCopy.forEach { target ->
val libraries = originalLibrariesByTargets.getValue(target)
copyTargetAsIs(target, libraries.platformLibs.size)
}
}
val concreteTargetNames = result.concreteTargets.map { it.name }
val targetsToSerialize = result.concreteTargets + result.commonTarget
targetsToSerialize.forEach { target ->
val newModules = result.modulesByTargets.getValue(target)
val manifestProvider = when (target) {
is InputTarget -> originalLibrariesByTargets.getValue(target)
is OutputTarget -> CommonNativeManifestDataProvider(originalLibrariesByTargets.values)
val manifestProvider: NativeManifestDataProvider
val starredTarget: String?
when (target) {
is InputTarget -> {
manifestProvider = originalLibrariesByTargets.getValue(target)
starredTarget = target.name
}
is OutputTarget -> {
manifestProvider = CommonNativeManifestDataProvider(originalLibrariesByTargets.values)
starredTarget = null
}
}
serializeTarget(target, newModules, manifestProvider, serializer)
val targetName = concreteTargetNames.joinToString { if (it == starredTarget) "$it(*)" else it }
serializeTarget(target, targetName, newModules, manifestProvider, serializer)
}
}
}
@@ -218,19 +242,29 @@ class NativeDistributionCommonizer(
val libraryDestination = destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR).resolve(libraryOrigin.name)
libraryOrigin.copyRecursively(libraryDestination)
}
val what = listOfNotNull(
"standard library".takeIf { copyStdlib },
"endorsed libraries".takeIf { copyEndorsedLibs }
).joinToString(separator = " and ")
logProgress("Copied $what")
}
}
private fun copyTargetAsIs(target: InputTarget) {
private fun copyTargetAsIs(target: InputTarget, platformLibrariesCount: Int) {
val librariesDestination = target.librariesDestination
librariesDestination.mkdirs() // always create an empty directory even if there is nothing to copy
val librariesSource = target.platformLibrariesSource
if (librariesSource.isDirectory) librariesSource.copyRecursively(librariesDestination)
logProgress("Copied of $platformLibrariesCount libraries for [${target.name}]")
}
private fun serializeTarget(
target: Target,
targetName: String,
newModules: Collection<ModuleDescriptor>,
manifestProvider: NativeManifestDataProvider,
serializer: KlibMetadataMonolithicSerializer
@@ -251,6 +285,8 @@ class NativeDistributionCommonizer(
writeLibrary(metadata, manifestData, libraryDestination)
}
logProgress("Written libraries for [$targetName]")
}
private fun writeLibrary(
native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt
+5 -3
View File
@@ -6,6 +6,7 @@
package org.jetbrains.kotlin.descriptors.commonizer.mergedtree
import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.descriptors.commonizer.Parameters
import org.jetbrains.kotlin.descriptors.commonizer.TargetProvider
import org.jetbrains.kotlin.descriptors.commonizer.cir.CirClass
import org.jetbrains.kotlin.descriptors.commonizer.cir.factory.*
@@ -20,17 +21,18 @@ import org.jetbrains.kotlin.storage.StorageManager
class CirTreeMerger(
private val storageManager: StorageManager,
private val targetProviders: List<TargetProvider>
private val parameters: Parameters
) {
private val size = targetProviders.size
private val size = parameters.targetProviders.size
private lateinit var cacheRW: CirClassifiersCacheImpl
fun merge(): CirRootNode {
val rootNode: CirRootNode = buildRootNode(storageManager, size)
cacheRW = rootNode.cache
targetProviders.forEachIndexed { targetIndex, targetProvider ->
parameters.targetProviders.forEachIndexed { targetIndex, targetProvider ->
processTarget(rootNode, targetIndex, targetProvider)
parameters.progressLogger?.invoke("Loaded declarations for [${targetProvider.target.name}]")
System.gc()
}
native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/ResettableClockMark.kt
+7 -2
View File
@@ -32,11 +32,16 @@ internal class ResettableClockMark {
}
}
private val startMark = currentTimeMillis()
private var lastMark = startMark
private var startMark: Long = 0
private var lastMark: Long = 0
fun elapsedSinceLast(): Period = Period(lastMark, currentTimeMillis()).also { lastMark = it.end }
fun elapsedSinceStart(): Period = Period(startMark, currentTimeMillis())
fun reset() {
startMark = currentTimeMillis()
lastMark = startMark
}
}
// TODO: this is how it should be when Kotlin Time will become non-experimental