From b2022144e639c98cefbb39d85be2fe928f6d43cc Mon Sep 17 00:00:00 2001 From: Dmitriy Dolovov Date: Thu, 25 Jun 2020 13:45:38 +0700 Subject: [PATCH] [Commonizer] Fast-pass for library fragments absent for some targets ^KT-39897 --- .../kotlin/descriptors/commonizer/Result.kt | 25 +-- .../descriptors/commonizer/TargetProvider.kt | 17 +-- .../builder/DeclarationsBuilderVisitor1.kt | 12 +- .../kotlin/descriptors/commonizer/cli/cli.kt | 6 +- .../commonizer/core/RootCommonizer.kt | 2 +- .../kotlin/descriptors/commonizer/facade.kt | 22 ++- .../konan/NativeDistributionCommonizer.kt | 43 +++--- .../NativeDistributionModulesProvider.kt | 11 ++ .../commonizer/mergedtree/CirTreeMerger.kt | 31 +++- .../stats/AggregatedStatsCollector.kt | 4 +- .../AbstractCommonizationFromSourcesTest.kt | 46 +++--- .../commonizer/CommonizerFacadeTest.kt | 142 +++++++++++++----- .../commonizer/core/RootCommonizerTest.kt | 4 +- .../commonizer/utils/assertions.kt | 8 +- .../descriptors/commonizer/utils/mocks.kt | 30 ++++ 15 files changed, 271 insertions(+), 132 deletions(-) diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/Result.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/Result.kt index 97eb9ce600e..c5889410fc7 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/Result.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/Result.kt @@ -6,19 +6,20 @@ package org.jetbrains.kotlin.descriptors.commonizer import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import java.io.File -sealed class Result +sealed class Result { + object NothingToCommonize : Result() -object NothingToCommonize : Result() - -class CommonizationPerformed( - val modulesByTargets: Map> -) : Result() { - val commonTarget: OutputTarget by lazy { - modulesByTargets.keys.filterIsInstance().single() - } - - val concreteTargets: Set by lazy { - modulesByTargets.keys.filterIsInstance().toSet() + class Commonized( + val modulesByTargets: Map> + ) : Result() { + val sharedTarget: OutputTarget by lazy { modulesByTargets.keys.filterIsInstance().single() } + val leafTargets: Set by lazy { modulesByTargets.keys.filterIsInstance().toSet() } } } + +sealed class ModuleResult { + class Absent(val originalLocation: File) : ModuleResult() + class Commonized(val module: ModuleDescriptor) : ModuleResult() +} diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt index e7d8ea332ae..8efebf44abc 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt @@ -5,8 +5,10 @@ package org.jetbrains.kotlin.descriptors.commonizer +import org.jetbrains.kotlin.builtins.DefaultBuiltIns import org.jetbrains.kotlin.builtins.KotlinBuiltIns import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import java.io.File class TargetProvider( val target: InputTarget, @@ -19,20 +21,15 @@ interface BuiltInsProvider { fun loadBuiltIns(): KotlinBuiltIns companion object { - fun wrap(builtIns: KotlinBuiltIns) = object : BuiltInsProvider { - override fun loadBuiltIns() = builtIns + val defaultBuiltInsProvider = object : BuiltInsProvider { + override fun loadBuiltIns() = DefaultBuiltIns.Instance } } } interface ModulesProvider { + class ModuleInfo(val name: String, val originalLocation: File) + + fun loadModuleInfos(): Map fun loadModules(): Collection - - companion object { - fun wrap(modules: Collection) = object : ModulesProvider { - override fun loadModules() = modules - } - - fun wrap(vararg modules: ModuleDescriptor) = wrap(modules.toList()) - } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/DeclarationsBuilderVisitor1.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/DeclarationsBuilderVisitor1.kt index 8d122c94432..8fcdfcbdd1b 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/DeclarationsBuilderVisitor1.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/DeclarationsBuilderVisitor1.kt @@ -14,7 +14,6 @@ import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirNode.Companion. import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirNode.Companion.indexOfCommon import org.jetbrains.kotlin.descriptors.commonizer.utils.CommonizedGroup import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl -import org.jetbrains.kotlin.utils.addIfNotNull /** * Serves two goals: @@ -28,16 +27,17 @@ internal class DeclarationsBuilderVisitor1( check(data.isEmpty()) // root node may not have containing declarations check(components.targetComponents.size == node.dimension) - val allTargets = (node.targetDeclarations + node.commonDeclaration()).map { it!!.target } - - val modulesByTargets = HashMap>() + val allTargets: List = (node.targetDeclarations + node.commonDeclaration()).map { it!!.target } + val modulesByTargets: Map> = allTargets.associateWithTo(HashMap()) { mutableListOf() } // collect module descriptors: for (moduleNode in node.modules.values) { val modules = moduleNode.accept(this, noContainingDeclarations()).asListContaining() modules.forEachIndexed { index, module -> - val target = allTargets[index] - modulesByTargets.getOrPut(target) { mutableListOf() }.addIfNotNull(module) + if (module != null) { + val target = allTargets[index] + modulesByTargets.getValue(target) += module + } } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/cli.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/cli.kt index 6062752eff2..9a8af589811 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/cli.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/cli.kt @@ -50,9 +50,9 @@ private fun parseTask( fun buildOngoingTask() { // check options completeness - val missedMandatoryOptions = optionTypes.filterKeys { it !in options }.filterValues { it.mandatory }.keys - if (missedMandatoryOptions.isNotEmpty()) - printUsageAndExit("Mandatory options not specified in task $taskAlias: " + missedMandatoryOptions.joinToString { "-$it" }) + val missingMandatoryOptions = optionTypes.filterKeys { it !in options }.filterValues { it.mandatory }.keys + if (missingMandatoryOptions.isNotEmpty()) + printUsageAndExit("Mandatory options not specified in task $taskAlias: " + missingMandatoryOptions.joinToString { "-$it" }) tasks += taskType.taskConstructor(options.values) } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/core/RootCommonizer.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/core/RootCommonizer.kt index 1f5f2ab4e76..c1fd7285eac 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/core/RootCommonizer.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/core/RootCommonizer.kt @@ -20,7 +20,7 @@ class RootCommonizer : AbstractStandardCommonizer() { override fun commonizationResult() = CirRootFactory.create( target = OutputTarget(inputTargets), builtInsClass = if (konanBuiltInsProvider != null) KonanBuiltIns::class.java.name else DefaultBuiltIns::class.java.name, - builtInsProvider = konanBuiltInsProvider ?: BuiltInsProvider.wrap(DefaultBuiltIns.Instance) + builtInsProvider = konanBuiltInsProvider ?: BuiltInsProvider.defaultBuiltInsProvider ) override fun initialize(first: CirRoot) { diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt index 54dd7c4e7e7..56d50dc0347 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt @@ -5,7 +5,6 @@ package org.jetbrains.kotlin.descriptors.commonizer -import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.descriptors.commonizer.builder.DeclarationsBuilderVisitor1 import org.jetbrains.kotlin.descriptors.commonizer.builder.DeclarationsBuilderVisitor2 import org.jetbrains.kotlin.descriptors.commonizer.builder.createGlobalBuilderComponents @@ -15,14 +14,15 @@ import org.jetbrains.kotlin.storage.LockBasedStorageManager fun runCommonization(parameters: Parameters): Result { if (!parameters.hasAnythingToCommonize()) - return NothingToCommonize + return Result.NothingToCommonize val storageManager = LockBasedStorageManager("Declaration descriptors commonization") // build merged tree: - val mergedTree = CirTreeMerger(storageManager, parameters).merge() + val mergeResult = CirTreeMerger(storageManager, parameters).merge() // commonize: + val mergedTree = mergeResult.root mergedTree.accept(CommonizationVisitor(mergedTree), Unit) parameters.progressLogger?.invoke("Commonized declarations") @@ -31,15 +31,21 @@ fun runCommonization(parameters: Parameters): Result { mergedTree.accept(DeclarationsBuilderVisitor1(components), emptyList()) mergedTree.accept(DeclarationsBuilderVisitor2(components), emptyList()) - val modulesByTargets = LinkedHashMap>() // use linked hash map to preserve order - components.targetComponents.forEach { - val target = it.target + val modulesByTargets = LinkedHashMap>() // use linked hash map to preserve order + components.targetComponents.forEach { component -> + val target = component.target check(target !in modulesByTargets) - modulesByTargets[target] = components.cache.getAllModules(it.index) + val commonizedModules: List = components.cache.getAllModules(component.index).map(ModuleResult::Commonized) + + val absentModules: List = if (target is InputTarget) + mergeResult.absentModuleInfos.getValue(target).map { ModuleResult.Absent(it.originalLocation) } + else emptyList() + + modulesByTargets[target] = commonizedModules + absentModules } parameters.progressLogger?.invoke("Prepared new descriptors") - return CommonizationPerformed(modulesByTargets) + return Result.Commonized(modulesByTargets) } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt index 3e1d31b6ad2..0a04d57e4f4 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt @@ -91,9 +91,9 @@ class NativeDistributionCommonizer( val stdlib = loadLibrary(stdlibPath) val result = targets.associate { target -> - val inputTarget = InputTarget(target.name, target) + val leafTarget = InputTarget(target.name, target) - val platformLibs = inputTarget.platformLibrariesSource + val platformLibs = leafTarget.platformLibrariesSource .takeIf { it.isDirectory } ?.listFiles() ?.takeIf { it.isNotEmpty() } @@ -103,7 +103,7 @@ class NativeDistributionCommonizer( if (platformLibs.isEmpty()) logger.warning("No platform libraries found for target $target. This target will be excluded from commonization.") - inputTarget to NativeDistributionLibraries(stdlib, platformLibs) + leafTarget to NativeDistributionLibraries(stdlib, platformLibs) } logProgress("Read lazy (uninitialized) libraries") @@ -176,16 +176,16 @@ class NativeDistributionCommonizer( copyCommonStandardLibraries() when (result) { - is NothingToCommonize -> { + is Result.NothingToCommonize -> { // It may happen that all targets to be commonized (or at least all but one target) miss platform libraries. // In such case commonizer will do nothing and return a special result value 'NothingToCommonize'. - // So, let's just copy platform libraries from those target where they are to the new destination. + // So, let's just copy platform libraries from the target where they are to the new destination. originalLibrariesByTargets.forEach { (target, libraries) -> copyTargetAsIs(target, libraries.platformLibs.size) } } - is CommonizationPerformed -> { + is Result.Commonized -> { val serializer = KlibMetadataMonolithicSerializer( languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT, metadataVersion = KlibMetadataVersion.INSTANCE, @@ -193,7 +193,7 @@ class NativeDistributionCommonizer( ) // 'targetsToCopy' are some targets with empty set of platform libraries - val targetsToCopy = originalLibrariesByTargets.keys - result.concreteTargets + val targetsToCopy = originalLibrariesByTargets.keys - result.leafTargets if (targetsToCopy.isNotEmpty()) { targetsToCopy.forEach { target -> val libraries = originalLibrariesByTargets.getValue(target) @@ -201,11 +201,13 @@ class NativeDistributionCommonizer( } } - - val concreteTargetNames = result.concreteTargets.map { it.name } - val targetsToSerialize = result.concreteTargets + result.commonTarget + val leafTargetNames = result.leafTargets.map { it.name } + val targetsToSerialize = result.leafTargets + result.sharedTarget targetsToSerialize.forEach { target -> - val newModules = result.modulesByTargets.getValue(target) + val moduleResults: Collection = result.modulesByTargets.getValue(target) + val newModules: Collection = moduleResults.mapNotNull { (it as? ModuleResult.Commonized)?.module } + val absentModuleLocations: List = + moduleResults.mapNotNull { (it as? ModuleResult.Absent)?.originalLocation } val manifestProvider: NativeManifestDataProvider val starredTarget: String? @@ -220,8 +222,8 @@ class NativeDistributionCommonizer( } } - val targetName = concreteTargetNames.joinToString { if (it == starredTarget) "$it(*)" else it } - serializeTarget(target, targetName, newModules, manifestProvider, serializer) + val targetName = leafTargetNames.joinToString { if (it == starredTarget) "$it(*)" else it } + serializeTarget(target, targetName, newModules, absentModuleLocations, manifestProvider, serializer) } } } @@ -252,24 +254,26 @@ class NativeDistributionCommonizer( } } - private fun copyTargetAsIs(target: InputTarget, platformLibrariesCount: Int) { - val librariesDestination = target.librariesDestination + private fun copyTargetAsIs(leafTarget: InputTarget, librariesCount: Int) { + val librariesDestination = leafTarget.librariesDestination librariesDestination.mkdirs() // always create an empty directory even if there is nothing to copy - val librariesSource = target.platformLibrariesSource + val librariesSource = leafTarget.platformLibrariesSource if (librariesSource.isDirectory) librariesSource.copyRecursively(librariesDestination) - logProgress("Copied $platformLibrariesCount libraries for [${target.name}]") + logProgress("Copied $librariesCount libraries for [${leafTarget.name}]") } private fun serializeTarget( target: Target, targetName: String, newModules: Collection, + absentModuleLocations: List, manifestProvider: NativeManifestDataProvider, serializer: KlibMetadataMonolithicSerializer ) { val librariesDestination = target.librariesDestination + librariesDestination.mkdirs() // always create an empty directory even if there is nothing to copy for (newModule in newModules) { val libraryName = newModule.name @@ -286,6 +290,11 @@ class NativeDistributionCommonizer( writeLibrary(metadata, manifestData, libraryDestination) } + for (absentModuleLocation in absentModuleLocations) { + val libraryName = absentModuleLocation.name + absentModuleLocation.copyRecursively(librariesDestination.resolve(libraryName)) + } + logProgress("Written libraries for [$targetName]") } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt index a1bc2e2202b..c88b2c0631b 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt @@ -10,11 +10,13 @@ import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.descriptors.commonizer.BuiltInsProvider import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider +import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider.ModuleInfo import org.jetbrains.kotlin.descriptors.commonizer.utils.NativeFactories import org.jetbrains.kotlin.descriptors.commonizer.utils.createKotlinNativeForwardDeclarationsModule import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME import org.jetbrains.kotlin.storage.StorageManager +import java.io.File internal class NativeDistributionModulesProvider( private val storageManager: StorageManager, @@ -32,6 +34,15 @@ internal class NativeDistributionModulesProvider( return stdlib.builtIns } + override fun loadModuleInfos(): Map { + return libraries.platformLibs.associate { library -> + val name = library.manifestData.uniqueName + val location = File(library.library.libraryFile.path) + + name to ModuleInfo(name, location) + } + } + override fun loadModules(): Collection { val builtIns = loadBuiltIns() val stdlib = builtIns.builtInsModule diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt index 952151b426b..f02fdb180c4 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt @@ -6,6 +6,8 @@ package org.jetbrains.kotlin.descriptors.commonizer.mergedtree import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.descriptors.commonizer.InputTarget +import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider.ModuleInfo import org.jetbrains.kotlin.descriptors.commonizer.Parameters import org.jetbrains.kotlin.descriptors.commonizer.TargetProvider import org.jetbrains.kotlin.descriptors.commonizer.cir.CirClass @@ -23,26 +25,44 @@ class CirTreeMerger( private val storageManager: StorageManager, private val parameters: Parameters ) { + class CirTreeMergeResult( + val root: CirRootNode, + val absentModuleInfos: Map> + ) + private val size = parameters.targetProviders.size private lateinit var cacheRW: CirClassifiersCacheImpl - fun merge(): CirRootNode { + fun merge(): CirTreeMergeResult { val rootNode: CirRootNode = buildRootNode(storageManager, size) cacheRW = rootNode.cache + val allModuleInfos: List> = parameters.targetProviders.map { it.modulesProvider.loadModuleInfos() } + val commonModuleNames = allModuleInfos.map { it.keys }.reduce { a, b -> a intersect b } + parameters.targetProviders.forEachIndexed { targetIndex, targetProvider -> - processTarget(rootNode, targetIndex, targetProvider) + processTarget(rootNode, targetIndex, targetProvider, commonModuleNames) parameters.progressLogger?.invoke("Loaded declarations for [${targetProvider.target.name}]") System.gc() } - return rootNode + val absentModuleInfos = allModuleInfos.mapIndexed { index, moduleInfos -> + val target = parameters.targetProviders[index].target + val absentInfos = moduleInfos.filterKeys { name -> name !in commonModuleNames }.values + target to absentInfos + }.toMap() + + return CirTreeMergeResult( + root = rootNode, + absentModuleInfos = absentModuleInfos + ) } private fun processTarget( rootNode: CirRootNode, targetIndex: Int, - targetProvider: TargetProvider + targetProvider: TargetProvider, + commonModuleNames: Set ) { rootNode.targetDeclarations[targetIndex] = CirRootFactory.create( targetProvider.target, @@ -54,7 +74,8 @@ class CirTreeMerger( val modules: MutableMap = rootNode.modules moduleDescriptors.forEach { moduleDescriptor -> - processModule(modules, targetIndex, moduleDescriptor) + if (moduleDescriptor.name.asString().removeSurrounding("<", ">") in commonModuleNames) + processModule(modules, targetIndex, moduleDescriptor) } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/stats/AggregatedStatsCollector.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/stats/AggregatedStatsCollector.kt index 87b2ef79ca4..0a7d6267748 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/stats/AggregatedStatsCollector.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/stats/AggregatedStatsCollector.kt @@ -40,8 +40,8 @@ class AggregatedStatsCollector( "Lifted Up, %%", "Commonized", "Commonized, %%", - "Missed in s. targets", - "Missed in s. targets, %%", + "Missing in s. targets", + "Missing in s. targets, %%", "Failed: Other", "Failed: Other, %%", "Total" diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt index 9ef88dc60e1..0656648bada 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt @@ -19,10 +19,8 @@ import org.jetbrains.kotlin.config.languageVersionSettings import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.descriptors.PackageFragmentProvider import org.jetbrains.kotlin.descriptors.commonizer.SourceModuleRoot.Companion.COMMON_TARGET_NAME -import org.jetbrains.kotlin.descriptors.commonizer.utils.assertCommonizationPerformed -import org.jetbrains.kotlin.descriptors.commonizer.utils.assertIsDirectory -import org.jetbrains.kotlin.descriptors.commonizer.utils.assertModulesAreEqual -import org.jetbrains.kotlin.descriptors.commonizer.utils.assertValidModule +import org.jetbrains.kotlin.descriptors.commonizer.utils.* +import org.jetbrains.kotlin.descriptors.commonizer.utils.MockBuiltInsProvider import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.platform.CommonPlatforms @@ -42,10 +40,6 @@ abstract class AbstractCommonizationFromSourcesTest : KtUsefulTestCase() { init { System.setProperty("java.awt.headless", "true") } - - fun Collection.eachModuleAsTarget() = mapIndexed { index, moduleDescriptor -> - InputTarget("target_$index") to moduleDescriptor - }.toMap().toCommonizationParameters() } private fun getTestDataDir(): File { @@ -69,26 +63,28 @@ abstract class AbstractCommonizationFromSourcesTest : KtUsefulTestCase() { val result: Result = runCommonization(analyzedModules.toCommonizationParameters()) assertCommonizationPerformed(result) - val commonTarget: OutputTarget = analyzedModules.commonizedCommonModule.target - assertEquals(commonTarget, result.commonTarget) + val sharedTarget: OutputTarget = analyzedModules.commonizedCommonModule.target + assertEquals(sharedTarget, result.sharedTarget) - val commonModuleAsExpected: ModuleDescriptor = analyzedModules.commonizedCommonModule.module - val commonModuleByCommonizer: ModuleDescriptor = result.modulesByTargets.getValue(commonTarget).single() + val sharedModuleAsExpected: ModuleDescriptor = analyzedModules.commonizedCommonModule.module + val sharedModuleByCommonizer: ModuleDescriptor = + (result.modulesByTargets.getValue(sharedTarget).single() as ModuleResult.Commonized).module - assertValidModule(commonModuleAsExpected) - assertValidModule(commonModuleByCommonizer) - assertModulesAreEqual(commonModuleAsExpected, commonModuleByCommonizer, "\"$commonTarget\" target") + assertValidModule(sharedModuleAsExpected) + assertValidModule(sharedModuleByCommonizer) + assertModulesAreEqual(sharedModuleAsExpected, sharedModuleByCommonizer, "\"$sharedTarget\" target") - val concreteTargets: Set = analyzedModules.commonizedPlatformModules.keys - assertEquals(concreteTargets, result.concreteTargets) + val leafTargets: Set = analyzedModules.commonizedPlatformModules.keys + assertEquals(leafTargets, result.leafTargets) - for (target in concreteTargets) { - val targetModuleAsExpected: ModuleDescriptor = analyzedModules.commonizedPlatformModules.getValue(target).module - val targetModuleByCommonizer: ModuleDescriptor = result.modulesByTargets.getValue(target).single() + for (leafTarget in leafTargets) { + val leafTargetModuleAsExpected: ModuleDescriptor = analyzedModules.commonizedPlatformModules.getValue(leafTarget).module + val leafTargetModuleByCommonizer: ModuleDescriptor = + (result.modulesByTargets.getValue(leafTarget).single() as ModuleResult.Commonized).module - assertValidModule(targetModuleAsExpected) - assertValidModule(targetModuleByCommonizer) - assertModulesAreEqual(targetModuleAsExpected, targetModuleByCommonizer, "\"$target\" target") + assertValidModule(leafTargetModuleAsExpected) + assertValidModule(leafTargetModuleByCommonizer) + assertModulesAreEqual(leafTargetModuleAsExpected, leafTargetModuleByCommonizer, "\"$leafTarget\" target") } } } @@ -327,8 +323,8 @@ private fun Map.toCommonizationParameters(): Para TargetProvider( target = target, builtInsClass = moduleDescriptor.builtIns::class.java, - builtInsProvider = BuiltInsProvider.wrap(moduleDescriptor.builtIns), - modulesProvider = ModulesProvider.wrap(moduleDescriptor) + builtInsProvider = MockBuiltInsProvider(moduleDescriptor.builtIns), + modulesProvider = MockModulesProvider(moduleDescriptor) ) ) } diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt index 71b260b2132..8664742f33b 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt @@ -5,10 +5,9 @@ package org.jetbrains.kotlin.descriptors.commonizer -import org.jetbrains.kotlin.descriptors.ModuleDescriptor -import org.jetbrains.kotlin.descriptors.commonizer.AbstractCommonizationFromSourcesTest.Companion.eachModuleAsTarget +import org.jetbrains.kotlin.builtins.DefaultBuiltIns +import org.jetbrains.kotlin.descriptors.commonizer.utils.MockModulesProvider import org.jetbrains.kotlin.descriptors.commonizer.utils.assertCommonizationPerformed -import org.jetbrains.kotlin.descriptors.commonizer.utils.mockEmptyModule import org.junit.Test import kotlin.contracts.ExperimentalContracts import kotlin.test.assertEquals @@ -17,49 +16,120 @@ import kotlin.test.assertEquals class CommonizerFacadeTest { @Test - fun nothingToCommonize0() { - val modules = listOf() - - val result = runCommonization(modules.eachModuleAsTarget()) - - assertEquals(NothingToCommonize, result) - } + fun nothingToCommonize0() = doTestNothingToCommonize( + emptyMap() + ) @Test - fun nothingToCommonize1() { - val modules = listOf( - mockEmptyModule("") + fun nothingToCommonize1() = doTestNothingToCommonize( + mapOf( + "target1" to listOf("foo") ) - - val result = runCommonization(modules.eachModuleAsTarget()) - - assertEquals(NothingToCommonize, result) - } + ) @Test - fun commonized() { - val modules = listOf( - mockEmptyModule(""), - mockEmptyModule("") + fun commonized1() = doTestSuccessfulCommonization( + mapOf( + "target1" to listOf("foo"), + "target2" to listOf("foo") ) + ) - val result = runCommonization(modules.eachModuleAsTarget()) + @Test + fun commonized2() = doTestSuccessfulCommonization( + mapOf( + "target1" to listOf("foo", "bar"), + "target2" to listOf("bar", "foo") + ) + ) - assertCommonizationPerformed(result) + @Test + fun commonizedWithDifferentModules() = doTestSuccessfulCommonization( + mapOf( + "target1" to listOf("foo"), + "target2" to listOf("bar") + ) + ) - assertSingleModuleForTarget("", result.modulesByTargets.getValue(result.commonTarget)) + @Test + fun commonizedWithAbsentModules() = doTestSuccessfulCommonization( + mapOf( + "target1" to listOf("foo", "bar"), + "target2" to listOf("foo", "qix") + ) + ) - assertEquals(2, result.concreteTargets.size) - for (target in result.concreteTargets) { - assertSingleModuleForTarget("", result.modulesByTargets.getValue(target)) + companion object { + private fun Map>.toCommonizationParameters() = Parameters().also { + forEach { (targetName, moduleNames) -> + it.addTarget( + TargetProvider( + target = InputTarget(targetName), + builtInsClass = DefaultBuiltIns::class.java, + builtInsProvider = BuiltInsProvider.defaultBuiltInsProvider, + modulesProvider = MockModulesProvider(moduleNames) + ) + ) + } + } + + private fun doTestNothingToCommonize(originalModules: Map>) { + val result = runCommonization(originalModules.toCommonizationParameters()) + assertEquals(Result.NothingToCommonize, result) + } + + private fun doTestSuccessfulCommonization(originalModules: Map>) { + val result = runCommonization(originalModules.toCommonizationParameters()) + assertCommonizationPerformed(result) + + val expectedCommonModuleNames = mutableSetOf() + originalModules.values.forEachIndexed { index, moduleNames -> + if (index == 0) + expectedCommonModuleNames.addAll(moduleNames) + else + expectedCommonModuleNames.retainAll(moduleNames) + } + assertModulesMatch( + expectedCommonizedModuleNames = expectedCommonModuleNames, + expectedAbsentModuleNames = emptySet(), + actualModuleResults = result.modulesByTargets.getValue(result.sharedTarget) + ) + + result.leafTargets.forEach { target -> + val allModuleNames = originalModules.getValue(target.name).toSet() + val expectedAbsentModuleNames = allModuleNames - expectedCommonModuleNames + + assertModulesMatch( + expectedCommonizedModuleNames = expectedCommonModuleNames, + expectedAbsentModuleNames = expectedAbsentModuleNames, + actualModuleResults = result.modulesByTargets.getValue(target) + ) + } + } + + private fun assertModulesMatch( + expectedCommonizedModuleNames: Set, + expectedAbsentModuleNames: Set, + actualModuleResults: Collection + ) { + assertEquals(expectedCommonizedModuleNames.size + expectedAbsentModuleNames.size, actualModuleResults.size) + + val actualCommonizedModuleNames = mutableSetOf() + val actualAbsentModuleNames = mutableSetOf() + + actualModuleResults.forEach { moduleResult -> + when (moduleResult) { + is ModuleResult.Commonized -> { + actualCommonizedModuleNames += moduleResult.module.name.asString().removeSurrounding("<", ">") + } + is ModuleResult.Absent -> { + actualAbsentModuleNames += moduleResult.originalLocation.name + } + } + } + + assertEquals(expectedCommonizedModuleNames, actualCommonizedModuleNames) + assertEquals(expectedAbsentModuleNames, actualAbsentModuleNames) } } - - private fun assertSingleModuleForTarget( - @Suppress("SameParameterValue") expectedModuleName: String, - modules: Collection - ) { - assertEquals(1, modules.size) - assertEquals(expectedModuleName, modules.single().name.asString()) - } } diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/core/RootCommonizerTest.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/core/RootCommonizerTest.kt index 9ac04ca9fef..839294ae086 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/core/RootCommonizerTest.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/core/RootCommonizerTest.kt @@ -9,12 +9,12 @@ import org.jetbrains.kotlin.builtins.DefaultBuiltIns import org.jetbrains.kotlin.builtins.KotlinBuiltIns import org.jetbrains.kotlin.builtins.jvm.JvmBuiltIns import org.jetbrains.kotlin.builtins.konan.KonanBuiltIns -import org.jetbrains.kotlin.descriptors.commonizer.BuiltInsProvider import org.jetbrains.kotlin.descriptors.commonizer.InputTarget import org.jetbrains.kotlin.descriptors.commonizer.OutputTarget import org.jetbrains.kotlin.descriptors.commonizer.Target import org.jetbrains.kotlin.descriptors.commonizer.cir.CirRoot import org.jetbrains.kotlin.descriptors.commonizer.cir.factory.CirRootFactory +import org.jetbrains.kotlin.descriptors.commonizer.utils.MockBuiltInsProvider import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.storage.LockBasedStorageManager import org.junit.Test @@ -167,7 +167,7 @@ class RootCommonizerTest : AbstractCommonizerTest() { fun KotlinBuiltIns.toMock(target: Target) = CirRootFactory.create( target = target, builtInsClass = this::class.java.name, - builtInsProvider = BuiltInsProvider.wrap(this) + builtInsProvider = MockBuiltInsProvider(this) ) } } diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/assertions.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/assertions.kt index af22e0c2da0..c9a14652831 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/assertions.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/assertions.kt @@ -6,9 +6,7 @@ package org.jetbrains.kotlin.descriptors.commonizer.utils import org.jetbrains.kotlin.descriptors.* -import org.jetbrains.kotlin.descriptors.commonizer.CommonizationPerformed import org.jetbrains.kotlin.descriptors.commonizer.Result -import org.jetbrains.kotlin.resolve.DescriptorUtils import org.jetbrains.kotlin.resolve.scopes.MemberScope import org.jetbrains.kotlin.test.util.DescriptorValidator.* import org.jetbrains.kotlin.types.ErrorUtils @@ -26,11 +24,11 @@ fun assertIsDirectory(file: File) { @ExperimentalContracts fun assertCommonizationPerformed(result: Result) { contract { - returns() implies (result is CommonizationPerformed) + returns() implies (result is Result.Commonized) } - if (result !is CommonizationPerformed) - fail("$result is not instance of ${CommonizationPerformed::class}") + if (result !is Result.Commonized) + fail("$result is not instance of ${Result.Commonized::class}") } @ExperimentalContracts diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt index 4870a6bd6d6..12db715c5b7 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt @@ -6,9 +6,13 @@ package org.jetbrains.kotlin.descriptors.commonizer.utils import org.jetbrains.kotlin.builtins.DefaultBuiltIns +import org.jetbrains.kotlin.builtins.KotlinBuiltIns import org.jetbrains.kotlin.descriptors.* import org.jetbrains.kotlin.descriptors.annotations.Annotations +import org.jetbrains.kotlin.descriptors.commonizer.BuiltInsProvider import org.jetbrains.kotlin.descriptors.commonizer.InputTarget +import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider +import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider.ModuleInfo import org.jetbrains.kotlin.descriptors.commonizer.builder.* import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirClassNode import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirClassifiersCache @@ -19,6 +23,7 @@ import org.jetbrains.kotlin.resolve.scopes.MemberScope import org.jetbrains.kotlin.storage.LockBasedStorageManager import org.jetbrains.kotlin.test.KotlinTestUtils import org.jetbrains.kotlin.types.* +import java.io.File import kotlin.random.Random // expected special name for module @@ -118,3 +123,28 @@ internal val EMPTY_CLASSIFIERS_CACHE = object : CirClassifiersCache { override val classes: Map get() = emptyMap() override val typeAliases: Map get() = emptyMap() } + +internal class MockBuiltInsProvider(private val builtIns: KotlinBuiltIns) : BuiltInsProvider { + override fun loadBuiltIns() = builtIns +} + +internal class MockModulesProvider : ModulesProvider { + private val moduleInfos: Map + private val modules: Map + + constructor(moduleNames: Collection) { + moduleInfos = moduleNames.associateWith { name -> fakeModuleInfo(name) } + modules = moduleNames.associateWith { name -> mockEmptyModule("<$name>") } + } + + constructor(module: ModuleDescriptor) { + val name = module.name.asString().removeSurrounding("<", ">") + moduleInfos = mapOf(name to fakeModuleInfo(name)) + modules = mapOf(name to module) + } + + override fun loadModuleInfos() = moduleInfos + override fun loadModules() = modules.values + + private fun fakeModuleInfo(name: String) = ModuleInfo(name, File("/tmp/commonizer/mocks/$name")) +}