From e0d90ccf4b93539f79450dddc5aa982b11283694 Mon Sep 17 00:00:00 2001 From: Dmitriy Dolovov Date: Wed, 25 Dec 2019 17:45:04 +0700 Subject: [PATCH] [Commonizer] Log commonization stats into a single file --- .../commonizer/CommonizationParameters.kt | 42 ++++++ .../commonizer/CommonizationResult.kt | 24 ++++ .../descriptors/commonizer/StatsCollector.kt | 13 ++ .../builder/DeclarationsBuilderVisitor1.kt | 16 +-- .../builder/DeclarationsBuilderVisitor2.kt | 9 +- .../commonizer/builder/classDescriptors.kt | 6 + .../descriptors/commonizer/builder/context.kt | 11 +- .../commonizer/builder/functionDescriptors.kt | 3 + .../commonizer/builder/moduleDescriptors.kt | 3 + .../commonizer/builder/propertyDescriptors.kt | 3 + .../builder/typeAliasDescriptors.kt | 3 + .../cli/nativeDistributionCommonizer.kt | 3 + .../kotlin/descriptors/commonizer/facade.kt | 52 +------ .../konan/NativeDistributionCommonizer.kt | 64 +++++---- .../commonizer/konan/NativeStatsCollector.kt | 132 ++++++++++++++++++ .../commonizer/utils/ResettableClockMark.kt | 19 +++ 16 files changed, 304 insertions(+), 99 deletions(-) create mode 100644 konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizationParameters.kt create mode 100644 konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizationResult.kt create mode 100644 konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/StatsCollector.kt create mode 100644 konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeStatsCollector.kt create mode 100644 konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/ResettableClockMark.kt diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizationParameters.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizationParameters.kt new file mode 100644 index 00000000000..c225ed6d5f0 --- /dev/null +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizationParameters.kt @@ -0,0 +1,42 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer + +import org.jetbrains.kotlin.descriptors.ModuleDescriptor + +class CommonizationParameters( + val statsCollector: StatsCollector? = null +) { + // use linked hash map to preserve order + private val modulesByTargets = LinkedHashMap>() + + fun addTarget(target: InputTarget, modules: Collection): CommonizationParameters { + require(target !in modulesByTargets) { "Target $target is already added" } + + val modulesWithUniqueNames = modules.groupingBy { it.name }.eachCount() + require(modulesWithUniqueNames.size == modules.size) { + "Modules with duplicated names found: ${modulesWithUniqueNames.filter { it.value > 1 }}" + } + + modulesByTargets[target] = modules + + return this + } + + // get them as ordered immutable collection (List) for further processing + fun getModulesByTargets(): List>> = + modulesByTargets.map { it.key to it.value } + + fun hasIntersection(): Boolean { + if (modulesByTargets.size < 2) + return false + + return modulesByTargets.flatMap { it.value } + .groupingBy { it.name } + .eachCount() + .any { it.value == modulesByTargets.size } + } +} diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizationResult.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizationResult.kt new file mode 100644 index 00000000000..93ded52af6b --- /dev/null +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizationResult.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer + +import org.jetbrains.kotlin.descriptors.ModuleDescriptor + +sealed class CommonizationResult + +object NothingToCommonize : CommonizationResult() + +class CommonizationPerformed( + val modulesByTargets: Map> +) : CommonizationResult() { + val commonTarget: OutputTarget by lazy { + modulesByTargets.keys.filterIsInstance().single() + } + + val concreteTargets: Set by lazy { + modulesByTargets.keys.filterIsInstance().toSet() + } +} diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/StatsCollector.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/StatsCollector.kt new file mode 100644 index 00000000000..0a78eee8c8d --- /dev/null +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/StatsCollector.kt @@ -0,0 +1,13 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer + +import org.jetbrains.kotlin.descriptors.DeclarationDescriptor +import java.io.Closeable + +interface StatsCollector : Closeable { + fun logStats(output: List) +} diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/DeclarationsBuilderVisitor1.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/DeclarationsBuilderVisitor1.kt index d0885def5ac..e54caacfba3 100644 --- a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/DeclarationsBuilderVisitor1.kt +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/DeclarationsBuilderVisitor1.kt @@ -52,8 +52,7 @@ internal class DeclarationsBuilderVisitor1( override fun visitModuleNode(node: CirModuleNode, data: List): List { // build module descriptors: - val moduleDescriptorsGroup = - CommonizedGroup(node.dimension) + val moduleDescriptorsGroup = CommonizedGroup(node.dimension) node.buildDescriptors(components, moduleDescriptorsGroup) val moduleDescriptors = moduleDescriptorsGroup.toList() @@ -76,8 +75,7 @@ internal class DeclarationsBuilderVisitor1( val containingDeclarations = data.asListContaining() // build package fragments: - val packageFragmentsGroup = - CommonizedGroup(node.dimension) + val packageFragmentsGroup = CommonizedGroup(node.dimension) node.buildDescriptors(components, packageFragmentsGroup, containingDeclarations) val packageFragments = packageFragmentsGroup.toList() @@ -105,10 +103,7 @@ internal class DeclarationsBuilderVisitor1( error("This method should not be called in ${this::class.java}") override fun visitClassNode(node: CirClassNode, data: List): List { - val classesGroup = - CommonizedGroup( - node.dimension - ) + val classesGroup = CommonizedGroup(node.dimension) node.buildDescriptors(components, classesGroup, data) val classes = classesGroup.toList().asListContaining() @@ -130,10 +125,7 @@ internal class DeclarationsBuilderVisitor1( error("This method should not be called in ${this::class.java}") override fun visitTypeAliasNode(node: CirTypeAliasNode, data: List): List { - val typeAliasesGroup = - CommonizedGroup( - node.dimension - ) + val typeAliasesGroup = CommonizedGroup(node.dimension) node.buildDescriptors(components, typeAliasesGroup, data) val typeAliases = typeAliasesGroup.toList() diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/DeclarationsBuilderVisitor2.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/DeclarationsBuilderVisitor2.kt index 0a94c5d5243..b2eefda0012 100644 --- a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/DeclarationsBuilderVisitor2.kt +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/DeclarationsBuilderVisitor2.kt @@ -65,16 +65,14 @@ internal class DeclarationsBuilderVisitor2( } override fun visitPropertyNode(node: CirPropertyNode, data: List): List { - val propertyDescriptorsGroup = - CommonizedGroup(node.dimension) + val propertyDescriptorsGroup = CommonizedGroup(node.dimension) node.buildDescriptors(components, propertyDescriptorsGroup, data) return propertyDescriptorsGroup.toList() } override fun visitFunctionNode(node: CirFunctionNode, data: List): List { - val functionDescriptorsGroup = - CommonizedGroup(node.dimension) + val functionDescriptorsGroup = CommonizedGroup(node.dimension) node.buildDescriptors(components, functionDescriptorsGroup, data) return functionDescriptorsGroup.toList() @@ -116,8 +114,7 @@ internal class DeclarationsBuilderVisitor2( override fun visitClassConstructorNode(node: CirClassConstructorNode, data: List): List { val containingDeclarations = data.asListContaining() - val constructorsGroup = - CommonizedGroup(node.dimension) + val constructorsGroup = CommonizedGroup(node.dimension) node.buildDescriptors(components, constructorsGroup, containingDeclarations) return constructorsGroup.toList() diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/classDescriptors.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/classDescriptors.kt index 9d457dd0f9a..d8ed51d75f1 100644 --- a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/classDescriptors.kt +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/classDescriptors.kt @@ -26,6 +26,9 @@ internal fun CirClassNode.buildDescriptors( } commonClass?.buildDescriptor(components, output, indexOfCommon, containingDeclarations, fqName, isExpect = true) + + // log stats + components.statsCollector?.logStats(output.toList()) } internal fun CirClass.buildDescriptor( @@ -79,6 +82,9 @@ internal fun CirClassConstructorNode.buildDescriptors( } commonConstructor?.buildDescriptor(components, output, indexOfCommon, containingDeclarations, isExpect = true) + + // log stats + components.statsCollector?.logStats(output.toList()) } private fun CirClassConstructor.buildDescriptor( diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt index 3ce2933ce1c..7102afe84ee 100644 --- a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt @@ -7,6 +7,7 @@ package org.jetbrains.kotlin.descriptors.commonizer.builder import org.jetbrains.kotlin.builtins.KotlinBuiltIns import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.descriptors.commonizer.StatsCollector import org.jetbrains.kotlin.descriptors.commonizer.Target import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ir.CirRootNode import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ir.dimension @@ -108,7 +109,8 @@ class DeclarationsBuilderCache(dimension: Int) { class GlobalDeclarationsBuilderComponents( val storageManager: StorageManager, val targetComponents: List, - val cache: DeclarationsBuilderCache + val cache: DeclarationsBuilderCache, + val statsCollector: StatsCollector? ) { init { check(targetComponents.size > 1) @@ -154,7 +156,10 @@ class TargetDeclarationsBuilderComponents( } } -fun CirRootNode.createGlobalBuilderComponents(storageManager: StorageManager): GlobalDeclarationsBuilderComponents { +fun CirRootNode.createGlobalBuilderComponents( + storageManager: StorageManager, + statsCollector: StatsCollector? +): GlobalDeclarationsBuilderComponents { val cache = DeclarationsBuilderCache(dimension) val targetContexts = (0 until dimension).map { index -> @@ -176,7 +181,7 @@ fun CirRootNode.createGlobalBuilderComponents(storageManager: StorageManager): G ) } - return GlobalDeclarationsBuilderComponents(storageManager, targetContexts, cache) + return GlobalDeclarationsBuilderComponents(storageManager, targetContexts, cache, statsCollector) } interface TypeParameterResolver { diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/functionDescriptors.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/functionDescriptors.kt index 92ea699028b..64145610824 100644 --- a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/functionDescriptors.kt +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/functionDescriptors.kt @@ -28,6 +28,9 @@ internal fun CirFunctionNode.buildDescriptors( } commonFunction?.buildDescriptor(components, output, indexOfCommon, containingDeclarations, isExpect = markAsExpectAndActual) + + // log stats + components.statsCollector?.logStats(output.toList()) } private fun CirFunction.buildDescriptor( diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/moduleDescriptors.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/moduleDescriptors.kt index f2cada42d57..f4eedf81d7f 100644 --- a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/moduleDescriptors.kt +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/moduleDescriptors.kt @@ -20,6 +20,9 @@ internal fun CirModuleNode.buildDescriptors( } common()?.buildDescriptor(components, output, indexOfCommon) + + // log stats + components.statsCollector?.logStats(output.toList()) } private fun CirModule.buildDescriptor( diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/propertyDescriptors.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/propertyDescriptors.kt index 1777bdfcd48..3cbd6fab51e 100644 --- a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/propertyDescriptors.kt +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/propertyDescriptors.kt @@ -31,6 +31,9 @@ internal fun CirPropertyNode.buildDescriptors( } commonProperty?.buildDescriptor(components, output, indexOfCommon, containingDeclarations, isExpect = markAsExpectAndActual) + + // log stats + components.statsCollector?.logStats(output.toList()) } private fun CirProperty.buildDescriptor( diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/typeAliasDescriptors.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/typeAliasDescriptors.kt index acdb349a06d..cf6e8db27ef 100644 --- a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/typeAliasDescriptors.kt +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/typeAliasDescriptors.kt @@ -27,6 +27,9 @@ internal fun CirTypeAliasNode.buildDescriptors( } commonClass?.buildDescriptor(components, output, indexOfCommon, containingDeclarations, fqName, isExpect = true) + + // log stats + components.statsCollector?.logStats(output.toList()) } private fun CirTypeAlias.buildDescriptor( diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeDistributionCommonizer.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeDistributionCommonizer.kt index 52f889d05df..966e328ac4a 100644 --- a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeDistributionCommonizer.kt +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeDistributionCommonizer.kt @@ -28,10 +28,13 @@ fun main(args: Array) { val destination = parsedArgs["-output"]?.firstOrNull()?.let(::File) ?: printUsageAndExit("output not specified") + val withStats = parsedArgs["-stats"]?.firstOrNull()?.toLowerCase() in setOf("1", "on", "yes", "true") + NativeDistributionCommonizer( repository = repository, targets = targets, destination = destination, + withStats = withStats, handleError = ::printErrorAndExit, log = ::println ).run() diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt index 71a4495573d..966795a5d45 100644 --- a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt @@ -6,61 +6,13 @@ package org.jetbrains.kotlin.descriptors.commonizer import org.jetbrains.kotlin.descriptors.ModuleDescriptor -import org.jetbrains.kotlin.descriptors.commonizer.builder.* import org.jetbrains.kotlin.descriptors.commonizer.builder.DeclarationsBuilderVisitor1 import org.jetbrains.kotlin.descriptors.commonizer.builder.DeclarationsBuilderVisitor2 +import org.jetbrains.kotlin.descriptors.commonizer.builder.createGlobalBuilderComponents import org.jetbrains.kotlin.descriptors.commonizer.core.CommonizationVisitor import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.mergeRoots import org.jetbrains.kotlin.storage.LockBasedStorageManager -class CommonizationParameters { - // use linked hash map to preserve order - private val modulesByTargets = LinkedHashMap>() - - fun addTarget(target: InputTarget, modules: Collection): CommonizationParameters { - require(target !in modulesByTargets) { "Target $target is already added" } - - val modulesWithUniqueNames = modules.groupingBy { it.name }.eachCount() - require(modulesWithUniqueNames.size == modules.size) { - "Modules with duplicated names found: ${modulesWithUniqueNames.filter { it.value > 1 }}" - } - - modulesByTargets[target] = modules - - return this - } - - // get them as ordered immutable collection (List) for further processing - fun getModulesByTargets(): List>> = - modulesByTargets.map { it.key to it.value } - - fun hasIntersection(): Boolean { - if (modulesByTargets.size < 2) - return false - - return modulesByTargets.flatMap { it.value } - .groupingBy { it.name } - .eachCount() - .any { it.value == modulesByTargets.size } - } -} - -sealed class CommonizationResult - -object NothingToCommonize : CommonizationResult() - -class CommonizationPerformed( - val modulesByTargets: Map> -) : CommonizationResult() { - val commonTarget: OutputTarget by lazy { - modulesByTargets.keys.filterIsInstance().single() - } - - val concreteTargets: Set by lazy { - modulesByTargets.keys.filterIsInstance().toSet() - } -} - fun runCommonization(parameters: CommonizationParameters): CommonizationResult { if (!parameters.hasIntersection()) return NothingToCommonize @@ -74,7 +26,7 @@ fun runCommonization(parameters: CommonizationParameters): CommonizationResult { mergedTree.accept(CommonizationVisitor(mergedTree), Unit) // build resulting descriptors: - val components = mergedTree.createGlobalBuilderComponents(storageManager) + val components = mergedTree.createGlobalBuilderComponents(storageManager, parameters.statsCollector) mergedTree.accept(DeclarationsBuilderVisitor1(components), emptyList()) mergedTree.accept(DeclarationsBuilderVisitor2(components), emptyList()) diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt index c3511ff3602..5e1ea015e30 100644 --- a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt @@ -14,6 +14,7 @@ import org.jetbrains.kotlin.descriptors.commonizer.Target import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeModuleForCommonization.DeserializedModule import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeModuleForCommonization.SyntheticModule import org.jetbrains.kotlin.descriptors.commonizer.utils.EmptyDescriptorTable +import org.jetbrains.kotlin.descriptors.commonizer.utils.ResettableClockMark import org.jetbrains.kotlin.descriptors.commonizer.utils.createKotlinNativeForwardDeclarationsModule import org.jetbrains.kotlin.konan.library.* import org.jetbrains.kotlin.konan.target.KonanTarget @@ -27,13 +28,13 @@ import org.jetbrains.kotlin.serialization.konan.impl.KlibResolvedModuleDescripto import org.jetbrains.kotlin.storage.LockBasedStorageManager import java.io.File import kotlin.time.ExperimentalTime -import kotlin.time.MonoClock import org.jetbrains.kotlin.konan.file.File as KFile class NativeDistributionCommonizer( private val repository: File, private val targets: List, private val destination: File, + private val withStats: Boolean, private val handleError: (String) -> Nothing, private val log: (String) -> Unit ) { @@ -41,23 +42,21 @@ class NativeDistributionCommonizer( fun run() { checkPreconditions() - val startMark = MonoClock.markNow() - val modulesByTargets = loadModules(repository, targets) - val loadDuration = startMark.elapsedNow() + with(ResettableClockMark()) { + // 1. load modules + val modulesByTargets = loadModules() + log("Loaded lazy (uninitialized) libraries in ${elapsedSinceLast()}") - log("Loaded lazy (uninitialized) libraries in $loadDuration") + // 2. run commonization + val result = commonize(modulesByTargets) + log("Commonization performed in ${elapsedSinceLast()}") - val result = commonize(modulesByTargets) - val commonizationDuration = startMark.elapsedNow() - loadDuration + // 3. write new libraries + saveModules(modulesByTargets, result) + log("Written libraries in ${elapsedSinceLast()}") - log("Commonization performed in $commonizationDuration") - - saveModules(modulesByTargets, destination, result) - val totalDuration = startMark.elapsedNow() - val writeDuration = totalDuration - commonizationDuration - loadDuration - - log("Written libraries in $writeDuration") - log("TOTAL: $totalDuration") + log("TOTAL: ${elapsedSinceStart()}") + } log("Done.") log("") @@ -79,7 +78,7 @@ class NativeDistributionCommonizer( } } - private fun loadModules(repository: File, targets: List): Map> { + private fun loadModules(): Map> { val stdlibPath = repository.resolve(konanCommonLibraryPath(KONAN_STDLIB_NAME)) val stdlib = loadLibrary(stdlibPath) @@ -162,22 +161,24 @@ class NativeDistributionCommonizer( } private fun commonize(modulesByTargets: Map>): CommonizationPerformed { - val parameters = CommonizationParameters().apply { - modulesByTargets.forEach { (target, modules) -> - addTarget(target, modules.map { it.module }) + val statsCollector = if (withStats) NativeStatsCollector(targets, destination) else null + statsCollector.use { + val parameters = CommonizationParameters(statsCollector).apply { + modulesByTargets.forEach { (target, modules) -> + addTarget(target, modules.map { it.module }) + } } - } - val result = runCommonization(parameters) - return when (result) { - is NothingToCommonize -> handleError("too few targets specified: ${modulesByTargets.keys}") - is CommonizationPerformed -> result + val result = runCommonization(parameters) + return when (result) { + is NothingToCommonize -> handleError("too few targets specified: ${modulesByTargets.keys}") + is CommonizationPerformed -> result + } } } private fun saveModules( originalModulesByTargets: Map>, - destination: File, result: CommonizationPerformed ) { // optimization: stdlib effectively remains the same across all Kotlin/Native targets, @@ -205,8 +206,6 @@ class NativeDistributionCommonizer( skipExpects = false ) - val stdlibName = Name.special("<$KONAN_STDLIB_NAME>") - fun serializeTarget(target: Target) { val libsDestination: File val newModulesManifestData: Map @@ -226,7 +225,9 @@ class NativeDistributionCommonizer( for (newModule in newModules) { val libraryName = newModule.name - if (libraryName == stdlibName || libraryName == KlibResolvedModuleDescriptorsFactoryImpl.FORWARD_DECLARATIONS_MODULE_NAME) continue + + if (!shouldBeSerialized(libraryName)) + continue val metadata = serializer.serializeModule(newModule) val manifestData = newModulesManifestData.getValue(newModule.name) @@ -250,4 +251,11 @@ class NativeDistributionCommonizer( manifestData.applyTo(library.base as BaseWriterImpl) library.commit() } + + private companion object { + val stdlibName = Name.special("<$KONAN_STDLIB_NAME>") + + fun shouldBeSerialized(libraryName: Name) = + libraryName != stdlibName && libraryName != KlibResolvedModuleDescriptorsFactoryImpl.FORWARD_DECLARATIONS_MODULE_NAME + } } diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeStatsCollector.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeStatsCollector.kt new file mode 100644 index 00000000000..02825e62798 --- /dev/null +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeStatsCollector.kt @@ -0,0 +1,132 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer.konan + +import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.descriptors.commonizer.StatsCollector +import org.jetbrains.kotlin.descriptors.commonizer.utils.firstNonNull +import org.jetbrains.kotlin.konan.target.KonanTarget +import org.jetbrains.kotlin.resolve.descriptorUtil.fqNameSafe +import java.io.File + +/** + * Allows printing commonization statistics to the file system. + * + * File format: text, "|"-separated columns. + * + * Header row: "FQ Name|Declaration Type|common||[|...]" + * + * Possible values for "Declaration Type": + * - MODULE + * - CLASS + * - INTERFACE + * - OBJECT + * - COMPANION_OBJECT + * - ENUM_CLASS + * - ENUM_ENTRY + * - TYPE_ALIAS + * - CLASS_CONSTRUCTOR + * - FUN + * - VAL + * + * Possible values for "common" column: + * - E = successfully commonized, expect declaration generated + * - "-" = no common declaration + * + * Possible values for each target platform column: + * - A = successfully commonized, actual declaration generated + * - O = not commonized, the declaration is as in the original library + * - "-" = no such declaration in the original library + * + * Example of output: + +FQ Name|Declaration Type|common|macos_x64|ios_x64 +|MODULE|E|A|A +platform.SystemConfiguration.SCPreferencesContext|CLASS|E|A|A +platform.SystemConfiguration.SCPreferencesContext.Companion|COMPANION_OBJECT|E|A|A +platform.SystemConfiguration.SCNetworkConnectionContext|CLASS|E|A|A +platform.SystemConfiguration.SCNetworkConnectionContext.Companion|COMPANION_OBJECT|E|A|A +platform.SystemConfiguration.SCDynamicStoreRefVar|TYPE_ALIAS|-|O|O +platform.SystemConfiguration.SCVLANInterfaceRef|TYPE_ALIAS|-|O|O + + */ +class NativeStatsCollector( + private val targets: List, + destination: File +) : StatsCollector { + + init { + destination.mkdirs() + } + + private val writer = destination.resolve("plain_stats.csv").printWriter() + private var headerWritten = false + + override fun logStats(output: List) { + if (!headerWritten) { + headerWritten = true + writeHeader() + } + + val firstNotNull = output.firstNonNull() + val lastIsNull = output.last() == null + + val row = buildString { + append((firstNotNull as? ModuleDescriptor)?.name ?: firstNotNull.fqNameSafe) // FQN (or name for module descriptor) + append(SEPARATOR) + append(firstNotNull.declarationType) // readable declaration type + append(SEPARATOR) + append(if (lastIsNull) '-' else 'E') // common + + for (index in 0 until output.size - 1) { + append(SEPARATOR) + append( + when { + output[index] == null -> '-' // absent + lastIsNull -> 'O' // original (not commonized) + else -> 'A' // actual (commonized) + } + ) + } + } + + writer.println(row) + } + + override fun close() = writer.close() + + private fun writeHeader() { + val row = buildString { + append("FQ Name") + append(SEPARATOR) + append("Declaration Type") + append(SEPARATOR) + append("common") + + targets.forEach { target -> + append(SEPARATOR) + append(target.name) + } + } + + writer.println(row) + } + + companion object { + private const val SEPARATOR = '|' + + private inline val DeclarationDescriptor.declarationType: String + get() = when (this) { + is ClassDescriptor -> if (isCompanionObject) "COMPANION_OBJECT" else kind.toString() + is TypeAliasDescriptor -> "TYPE_ALIAS" + is ClassConstructorDescriptor -> "CLASS_CONSTRUCTOR" + is FunctionDescriptor -> "FUN" + is PropertyDescriptor -> "VAL" + is ModuleDescriptor -> "MODULE" + else -> "UNKNOWN: ${this::class.java}" + } + } +} diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/ResettableClockMark.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/ResettableClockMark.kt new file mode 100644 index 00000000000..ae7ab65fe65 --- /dev/null +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/ResettableClockMark.kt @@ -0,0 +1,19 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer.utils + +import kotlin.time.Duration +import kotlin.time.ExperimentalTime +import kotlin.time.MonoClock + +@ExperimentalTime +internal class ResettableClockMark { + private val startMark = MonoClock.markNow() + private var lastMark = startMark + + fun elapsedSinceLast(): Duration = lastMark.elapsedNow().also { lastMark = lastMark.plus(it) } + fun elapsedSinceStart(): Duration = startMark.elapsedNow() +}