[Commonizer] Dump every target to disk immediately when it's ready

This commit is contained in:
Dmitriy Dolovov
2021-02-01 11:06:49 +03:00
parent 9c4af5070b
commit 2438265ba8
10 changed files with 292 additions and 240 deletions
@@ -13,7 +13,6 @@ class CommonizerParameters(
) { ) {
// use linked hash map to preserve order // use linked hash map to preserve order
private val _targetProviders = LinkedHashMap<LeafTarget, TargetProvider>() private val _targetProviders = LinkedHashMap<LeafTarget, TargetProvider>()
val targetProviders: List<TargetProvider> get() = _targetProviders.values.toList() val targetProviders: List<TargetProvider> get() = _targetProviders.values.toList()
val sharedTarget: SharedTarget get() = SharedTarget(_targetProviders.keys) val sharedTarget: SharedTarget get() = SharedTarget(_targetProviders.keys)
@@ -31,5 +30,13 @@ class CommonizerParameters(
return this return this
} }
private var _resultsConsumer: ResultsConsumer? = null
var resultsConsumer: ResultsConsumer
get() = _resultsConsumer ?: error("Results consumer has not been set")
set(value) {
check(_resultsConsumer == null)
_resultsConsumer = value
}
fun hasAnythingToCommonize(): Boolean = _targetProviders.size >= 2 fun hasAnythingToCommonize(): Boolean = _targetProviders.size >= 2
} }
@@ -1,30 +0,0 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.descriptors.commonizer
import org.jetbrains.kotlin.library.SerializedMetadata
import java.io.File
sealed class CommonizerResult {
object NothingToDo : CommonizerResult()
class Done(
val modulesByTargets: Map<CommonizerTarget, Collection<ModuleResult>>
) : CommonizerResult() {
val sharedTarget: SharedTarget by lazy { modulesByTargets.keys.filterIsInstance<SharedTarget>().single() }
val leafTargets: Set<LeafTarget> by lazy { modulesByTargets.keys.filterIsInstance<LeafTarget>().toSet() }
}
}
sealed class ModuleResult {
abstract val libraryName: String
class Missing(val originalLocation: File) : ModuleResult() {
override val libraryName: String get() = originalLocation.name
}
class Commonized(override val libraryName: String, val metadata: SerializedMetadata) : ModuleResult()
}
@@ -0,0 +1,27 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.descriptors.commonizer
import org.jetbrains.kotlin.library.SerializedMetadata
import java.io.File
interface ResultsConsumer {
enum class Status { NOTHING_TO_DO, DONE }
sealed class ModuleResult {
abstract val libraryName: String
class Missing(val originalLocation: File) : ModuleResult() {
override val libraryName: String get() = originalLocation.name
}
class Commonized(override val libraryName: String, val metadata: SerializedMetadata) : ModuleResult()
}
fun consumeResults(target: CommonizerTarget, moduleResults: Collection<ModuleResult>)
fun successfullyFinished(status: Status)
}
@@ -6,6 +6,8 @@
package org.jetbrains.kotlin.descriptors.commonizer package org.jetbrains.kotlin.descriptors.commonizer
import kotlinx.metadata.klib.ChunkedKlibModuleFragmentWriteStrategy import kotlinx.metadata.klib.ChunkedKlibModuleFragmentWriteStrategy
import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer.ModuleResult
import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer.Status
import org.jetbrains.kotlin.descriptors.commonizer.core.CommonizationVisitor import org.jetbrains.kotlin.descriptors.commonizer.core.CommonizationVisitor
import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.* import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.*
import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirNode.Companion.dimension import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirNode.Companion.dimension
@@ -15,9 +17,12 @@ import org.jetbrains.kotlin.library.SerializedMetadata
import org.jetbrains.kotlin.storage.LockBasedStorageManager import org.jetbrains.kotlin.storage.LockBasedStorageManager
import org.jetbrains.kotlin.storage.StorageManager import org.jetbrains.kotlin.storage.StorageManager
fun runCommonization(parameters: CommonizerParameters): CommonizerResult { fun runCommonization(parameters: CommonizerParameters) {
if (!parameters.hasAnythingToCommonize()) val resultsConsumer = parameters.resultsConsumer
return CommonizerResult.NothingToDo if (!parameters.hasAnythingToCommonize()) {
resultsConsumer.successfullyFinished(Status.NOTHING_TO_DO)
return
}
val storageManager = LockBasedStorageManager("Declarations commonization") val storageManager = LockBasedStorageManager("Declarations commonization")
@@ -25,7 +30,6 @@ fun runCommonization(parameters: CommonizerParameters): CommonizerResult {
val mergedTree = mergeResult.root val mergedTree = mergeResult.root
// build resulting declarations: // build resulting declarations:
val modulesByTargets = LinkedHashMap<CommonizerTarget, Collection<ModuleResult>>() // use linked hash map to preserve order
val klibFragmentWriteStrategy = ChunkedKlibModuleFragmentWriteStrategy() val klibFragmentWriteStrategy = ChunkedKlibModuleFragmentWriteStrategy()
for (targetIndex in 0 until mergedTree.dimension) { for (targetIndex in 0 until mergedTree.dimension) {
@@ -39,16 +43,15 @@ fun runCommonization(parameters: CommonizerParameters): CommonizerResult {
ModuleResult.Commonized(libraryName, serializedMetadata) ModuleResult.Commonized(libraryName, serializedMetadata)
} }
parameters.progressLogger?.invoke("Built metadata for ${target.prettyCommonizedName(parameters.sharedTarget)}")
val missingModules: List<ModuleResult.Missing> = if (target is LeafTarget) val missingModules: List<ModuleResult.Missing> = if (target is LeafTarget)
mergeResult.missingModuleInfos.getValue(target).map { ModuleResult.Missing(it.originalLocation) } mergeResult.missingModuleInfos.getValue(target).map { ModuleResult.Missing(it.originalLocation) }
else emptyList() else emptyList()
modulesByTargets[target] = commonizedModules + missingModules resultsConsumer.consumeResults(target, commonizedModules + missingModules)
} }
return CommonizerResult.Done(modulesByTargets) resultsConsumer.successfullyFinished(Status.DONE)
} }
private fun mergeAndCommonize(storageManager: StorageManager, parameters: CommonizerParameters): CirTreeMergeResult { private fun mergeAndCommonize(storageManager: StorageManager, parameters: CommonizerParameters): CirTreeMergeResult {
@@ -14,12 +14,7 @@ import org.jetbrains.kotlin.descriptors.commonizer.utils.ResettableClockMark
import org.jetbrains.kotlin.konan.library.* import org.jetbrains.kotlin.konan.library.*
import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.library.KotlinLibrary
import org.jetbrains.kotlin.library.SerializedMetadata
import org.jetbrains.kotlin.library.ToolingSingleFileKlibResolveStrategy import org.jetbrains.kotlin.library.ToolingSingleFileKlibResolveStrategy
import org.jetbrains.kotlin.library.impl.BaseWriterImpl
import org.jetbrains.kotlin.library.impl.BuiltInsPlatform
import org.jetbrains.kotlin.library.impl.KotlinLibraryLayoutForWriter
import org.jetbrains.kotlin.library.impl.KotlinLibraryWriterImpl
import org.jetbrains.kotlin.library.resolveSingleFileKlib import org.jetbrains.kotlin.library.resolveSingleFileKlib
import org.jetbrains.kotlin.storage.LockBasedStorageManager import org.jetbrains.kotlin.storage.LockBasedStorageManager
import org.jetbrains.kotlin.util.Logger import org.jetbrains.kotlin.util.Logger
@@ -35,9 +30,7 @@ class NativeDistributionCommonizer(
private val statsType: StatsType, private val statsType: StatsType,
private val logger: Logger private val logger: Logger
) { ) {
enum class StatsType { enum class StatsType { RAW, AGGREGATED, NONE }
RAW, AGGREGATED, NONE
}
private val clockMark = ResettableClockMark() private val clockMark = ResettableClockMark()
@@ -48,11 +41,8 @@ class NativeDistributionCommonizer(
// 1. load libraries // 1. load libraries
val allLibraries = loadLibraries() val allLibraries = loadLibraries()
// 2. run commonization // 2. run commonization & write new libraries
val result = commonize(allLibraries) commonizeAndSaveResults(allLibraries)
// 3. write new libraries
saveModules(allLibraries, result)
logTotal() logTotal()
} }
@@ -127,7 +117,7 @@ class NativeDistributionCommonizer(
return library return library
} }
private fun commonize(allLibraries: AllNativeLibraries): CommonizerResult { private fun commonizeAndSaveResults(allLibraries: AllNativeLibraries) {
val statsCollector = when (statsType) { val statsCollector = when (statsType) {
RAW -> RawStatsCollector(targets) RAW -> RawStatsCollector(targets)
AGGREGATED -> AggregatedStatsCollector(targets) AGGREGATED -> AggregatedStatsCollector(targets)
@@ -137,6 +127,14 @@ class NativeDistributionCommonizer(
val parameters = CommonizerParameters(statsCollector, ::logProgress).apply { val parameters = CommonizerParameters(statsCollector, ::logProgress).apply {
val storageManager = LockBasedStorageManager("Commonized modules") val storageManager = LockBasedStorageManager("Commonized modules")
resultsConsumer = NativeDistributionResultsConsumer(
repository = repository,
originalLibraries = allLibraries,
destination = destination,
copyStdlib = copyStdlib,
copyEndorsedLibs = copyEndorsedLibs,
logProgress = ::logProgress
)
dependeeModulesProvider = NativeDistributionStdlibProvider(storageManager, allLibraries.stdlib) dependeeModulesProvider = NativeDistributionStdlibProvider(storageManager, allLibraries.stdlib)
allLibraries.librariesByTargets.forEach { (target, librariesToCommonize) -> allLibraries.librariesByTargets.forEach { (target, librariesToCommonize) ->
@@ -154,148 +152,13 @@ class NativeDistributionCommonizer(
} }
} }
val result = runCommonization(parameters) runCommonization(parameters)
statsCollector?.writeTo(FileStatsOutput(destination, statsType.name.toLowerCase())) statsCollector?.writeTo(FileStatsOutput(destination, statsType.name.toLowerCase()))
return result
}
private fun saveModules(originalLibraries: AllNativeLibraries, result: CommonizerResult) {
// optimization: stdlib and endorsed libraries effectively remain the same across all Kotlin/Native targets,
// so they can be just copied to the new destination without running serializer
copyCommonStandardLibraries()
when (result) {
is CommonizerResult.NothingToDo -> {
// It may happen that all targets to be commonized (or at least all but one target) miss platform libraries.
// In such case commonizer will do nothing and return a special result value 'NothingToCommonize'.
// So, let's just copy platform libraries from the target where they are to the new destination.
originalLibraries.librariesByTargets.forEach { (target, librariesToCommonize) ->
copyTargetAsIs(target, librariesToCommonize.libraries.size)
}
}
is CommonizerResult.Done -> {
// 'targetsToCopy' are some targets with empty set of platform libraries
val targetsToCopy = originalLibraries.librariesByTargets.keys - result.leafTargets
if (targetsToCopy.isNotEmpty()) {
targetsToCopy.forEach { target ->
val librariesToCommonize = originalLibraries.librariesByTargets.getValue(target)
copyTargetAsIs(target, librariesToCommonize.libraries.size)
}
}
val targetsToSerialize = result.leafTargets + result.sharedTarget
targetsToSerialize.forEach { target ->
val moduleResults: Collection<ModuleResult> = result.modulesByTargets.getValue(target)
val prettyTargetName = target.prettyCommonizedName(result.sharedTarget)
val manifestProvider = when (target) {
is LeafTarget -> originalLibraries.librariesByTargets.getValue(target)
is SharedTarget -> CommonNativeManifestDataProvider(originalLibraries.librariesByTargets.values)
}
serializeTarget(target, prettyTargetName, moduleResults, manifestProvider)
}
}
}
}
private fun copyCommonStandardLibraries() {
if (copyStdlib || copyEndorsedLibs) {
repository.resolve(KONAN_DISTRIBUTION_KLIB_DIR)
.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR)
.listFiles()
?.filter { it.isDirectory }
?.let {
if (copyStdlib) {
if (copyEndorsedLibs) it else it.filter { dir -> dir.endsWith(KONAN_STDLIB_NAME) }
} else
it.filter { dir -> !dir.endsWith(KONAN_STDLIB_NAME) }
}?.forEach { libraryOrigin ->
val libraryDestination = destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR).resolve(libraryOrigin.name)
libraryOrigin.copyRecursively(libraryDestination)
}
val what = listOfNotNull(
"standard library".takeIf { copyStdlib },
"endorsed libraries".takeIf { copyEndorsedLibs }
).joinToString(separator = " and ")
logProgress("Copied $what")
}
}
private fun copyTargetAsIs(leafTarget: LeafTarget, librariesCount: Int) {
val librariesDestination = leafTarget.librariesDestination
librariesDestination.mkdirs() // always create an empty directory even if there is nothing to copy
val librariesSource = leafTarget.platformLibrariesSource
if (librariesSource.isDirectory) librariesSource.copyRecursively(librariesDestination)
logProgress("Copied $librariesCount libraries for [${leafTarget.name}]")
}
private fun serializeTarget(
target: CommonizerTarget,
prettyTargetName: String,
moduleResults: Collection<ModuleResult>,
manifestProvider: NativeManifestDataProvider
) {
val librariesDestination = target.librariesDestination
librariesDestination.mkdirs() // always create an empty directory even if there is nothing to copy
for (moduleResult in moduleResults) {
when (moduleResult) {
is ModuleResult.Commonized -> {
val libraryName = moduleResult.libraryName
val manifestData = manifestProvider.getManifest(libraryName)
val libraryDestination = librariesDestination.resolve(libraryName)
writeLibrary(moduleResult.metadata, manifestData, libraryDestination)
}
is ModuleResult.Missing -> {
val libraryName = moduleResult.libraryName
val missingModuleLocation = moduleResult.originalLocation
missingModuleLocation.copyRecursively(librariesDestination.resolve(libraryName))
}
}
}
logProgress("Written libraries for $prettyTargetName")
}
private fun writeLibrary(
metadata: SerializedMetadata,
manifestData: NativeSensitiveManifestData,
destination: File
) {
val kDestination = KFile(destination.path)
val layout = KotlinLibraryLayoutForWriter(kDestination, kDestination)
val library = KotlinLibraryWriterImpl(
moduleName = manifestData.uniqueName,
versions = manifestData.versions,
builtInsPlatform = BuiltInsPlatform.NATIVE,
nativeTargets = emptyList(), // will be overwritten with NativeSensitiveManifestData.applyTo() below
nopack = true,
shortName = manifestData.shortName,
layout = layout
)
library.addMetadata(metadata)
manifestData.applyTo(library.base as BaseWriterImpl)
library.commit()
} }
private val LeafTarget.platformLibrariesSource: File private val LeafTarget.platformLibrariesSource: File
get() = repository.resolve(KONAN_DISTRIBUTION_KLIB_DIR) get() = repository.resolve(KONAN_DISTRIBUTION_KLIB_DIR)
.resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR) .resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR)
.resolve(name) .resolve(name)
private val CommonizerTarget.librariesDestination: File
get() = when (this) {
is LeafTarget -> destination.resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR).resolve(name)
is SharedTarget -> destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR)
}
} }
@@ -0,0 +1,163 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.descriptors.commonizer.konan
import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer
import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer.ModuleResult
import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer.Status
import org.jetbrains.kotlin.descriptors.commonizer.CommonizerTarget
import org.jetbrains.kotlin.descriptors.commonizer.LeafTarget
import org.jetbrains.kotlin.descriptors.commonizer.SharedTarget
import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMON_LIBS_DIR
import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_KLIB_DIR
import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR
import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME
import org.jetbrains.kotlin.library.SerializedMetadata
import org.jetbrains.kotlin.library.impl.BaseWriterImpl
import org.jetbrains.kotlin.library.impl.BuiltInsPlatform
import org.jetbrains.kotlin.library.impl.KotlinLibraryLayoutForWriter
import org.jetbrains.kotlin.library.impl.KotlinLibraryWriterImpl
import java.io.File
import org.jetbrains.kotlin.konan.file.File as KFile
internal class NativeDistributionResultsConsumer(
private val repository: File,
private val originalLibraries: AllNativeLibraries,
private val destination: File,
private val copyStdlib: Boolean,
private val copyEndorsedLibs: Boolean,
private val logProgress: (String) -> Unit
) : ResultsConsumer {
private val consumedTargets = LinkedHashSet<CommonizerTarget>()
private val allLeafTargets = originalLibraries.librariesByTargets.keys
private val sharedTarget = SharedTarget(allLeafTargets)
override fun consumeResults(target: CommonizerTarget, moduleResults: Collection<ModuleResult>) {
val added = consumedTargets.add(target)
check(added)
serializeTarget(target, moduleResults)
}
override fun successfullyFinished(status: Status) {
// optimization: stdlib and endorsed libraries effectively remain the same across all Kotlin/Native targets,
// so they can be just copied to the new destination without running serializer
copyCommonStandardLibraries()
when (status) {
Status.NOTHING_TO_DO -> {
// It may happen that all targets to be commonized (or at least all but one target) miss platform libraries.
// In such case commonizer will do nothing and raise a special status value 'NOTHING_TO_DO'.
// So, let's just copy platform libraries from the target where they are to the new destination.
allLeafTargets.forEach(::copyTargetAsIs)
}
Status.DONE -> {
// 'targetsToCopy' are some leaf targets with empty set of platform libraries
val targetsToCopy = allLeafTargets - consumedTargets.filterIsInstance<LeafTarget>()
targetsToCopy.forEach(::copyTargetAsIs)
}
}
}
private fun copyCommonStandardLibraries() {
if (copyStdlib || copyEndorsedLibs) {
repository.resolve(KONAN_DISTRIBUTION_KLIB_DIR)
.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR)
.listFiles()
?.filter { it.isDirectory }
?.let {
if (copyStdlib) {
if (copyEndorsedLibs) it else it.filter { dir -> dir.endsWith(KONAN_STDLIB_NAME) }
} else
it.filter { dir -> !dir.endsWith(KONAN_STDLIB_NAME) }
}?.forEach { libraryOrigin ->
val libraryDestination = destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR).resolve(libraryOrigin.name)
libraryOrigin.copyRecursively(libraryDestination)
}
val what = listOfNotNull(
"standard library".takeIf { copyStdlib },
"endorsed libraries".takeIf { copyEndorsedLibs }
).joinToString(separator = " and ")
logProgress("Copied $what")
}
}
private fun copyTargetAsIs(leafTarget: LeafTarget) {
val librariesCount = originalLibraries.librariesByTargets.getValue(leafTarget).libraries.size
val librariesDestination = leafTarget.librariesDestination
librariesDestination.mkdirs() // always create an empty directory even if there is nothing to copy
val librariesSource = leafTarget.platformLibrariesSource
if (librariesSource.isDirectory) librariesSource.copyRecursively(librariesDestination)
logProgress("Copied $librariesCount libraries for ${leafTarget.prettyName}")
}
private fun serializeTarget(target: CommonizerTarget, moduleResults: Collection<ModuleResult>) {
val librariesDestination = target.librariesDestination
librariesDestination.mkdirs() // always create an empty directory even if there is nothing to copy
val manifestProvider = when (target) {
is LeafTarget -> originalLibraries.librariesByTargets.getValue(target)
is SharedTarget -> CommonNativeManifestDataProvider(originalLibraries.librariesByTargets.values)
}
for (moduleResult in moduleResults) {
when (moduleResult) {
is ModuleResult.Commonized -> {
val libraryName = moduleResult.libraryName
val manifestData = manifestProvider.getManifest(libraryName)
val libraryDestination = librariesDestination.resolve(libraryName)
writeLibrary(moduleResult.metadata, manifestData, libraryDestination)
}
is ModuleResult.Missing -> {
val libraryName = moduleResult.libraryName
val missingModuleLocation = moduleResult.originalLocation
missingModuleLocation.copyRecursively(librariesDestination.resolve(libraryName))
}
}
}
logProgress("Written libraries for ${target.prettyCommonizedName(sharedTarget)}")
}
private fun writeLibrary(
metadata: SerializedMetadata,
manifestData: NativeSensitiveManifestData,
libraryDestination: File
) {
val layout = KFile(libraryDestination.path).let { KotlinLibraryLayoutForWriter(it, it) }
val library = KotlinLibraryWriterImpl(
moduleName = manifestData.uniqueName,
versions = manifestData.versions,
builtInsPlatform = BuiltInsPlatform.NATIVE,
nativeTargets = emptyList(), // will be overwritten with NativeSensitiveManifestData.applyTo() below
nopack = true,
shortName = manifestData.shortName,
layout = layout
)
library.addMetadata(metadata)
manifestData.applyTo(library.base as BaseWriterImpl)
library.commit()
}
private val LeafTarget.platformLibrariesSource: File
get() = repository.resolve(KONAN_DISTRIBUTION_KLIB_DIR)
.resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR)
.resolve(name)
private val CommonizerTarget.librariesDestination: File
get() = when (this) {
is LeafTarget -> destination.resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR).resolve(name)
is SharedTarget -> destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR)
}
}
@@ -23,6 +23,8 @@ import org.jetbrains.kotlin.config.languageVersionSettings
import org.jetbrains.kotlin.descriptors.CallableMemberDescriptor import org.jetbrains.kotlin.descriptors.CallableMemberDescriptor
import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.PackageFragmentProvider import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer.ModuleResult
import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer.Status
import org.jetbrains.kotlin.descriptors.commonizer.SourceModuleRoot.Companion.SHARED_TARGET_NAME import org.jetbrains.kotlin.descriptors.commonizer.SourceModuleRoot.Companion.SHARED_TARGET_NAME
import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ClassCollector import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ClassCollector
import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.FunctionCollector import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.FunctionCollector
@@ -72,25 +74,26 @@ abstract class AbstractCommonizationFromSourcesTest : KtUsefulTestCase() {
val sourceModuleRoots: SourceModuleRoots = SourceModuleRoots.load(getTestDataDir()) val sourceModuleRoots: SourceModuleRoots = SourceModuleRoots.load(getTestDataDir())
val analyzedModules: AnalyzedModules = AnalyzedModules.create(sourceModuleRoots, testRootDisposable) val analyzedModules: AnalyzedModules = AnalyzedModules.create(sourceModuleRoots, testRootDisposable)
val result: CommonizerResult = runCommonization(analyzedModules.toCommonizationParameters()) val results = MockResultsConsumer()
assertCommonizationPerformed(result) runCommonization(analyzedModules.toCommonizerParameters(results))
assertEquals(Status.DONE, results.status)
val sharedTarget: SharedTarget = analyzedModules.sharedTarget val sharedTarget: SharedTarget = analyzedModules.sharedTarget
assertEquals(sharedTarget, result.sharedTarget) assertEquals(sharedTarget, results.sharedTarget)
val sharedModuleAsExpected: SerializedMetadata = analyzedModules.commonizedModules.getValue(sharedTarget) val sharedModuleAsExpected: SerializedMetadata = analyzedModules.commonizedModules.getValue(sharedTarget)
val sharedModuleByCommonizer: SerializedMetadata = val sharedModuleByCommonizer: SerializedMetadata =
(result.modulesByTargets.getValue(sharedTarget).single() as ModuleResult.Commonized).metadata (results.modulesByTargets.getValue(sharedTarget).single() as ModuleResult.Commonized).metadata
assertModulesAreEqual(sharedModuleAsExpected, sharedModuleByCommonizer, sharedTarget) assertModulesAreEqual(sharedModuleAsExpected, sharedModuleByCommonizer, sharedTarget)
val leafTargets: Set<LeafTarget> = analyzedModules.leafTargets val leafTargets: Set<LeafTarget> = analyzedModules.leafTargets
assertEquals(leafTargets, result.leafTargets) assertEquals(leafTargets, results.leafTargets)
for (leafTarget in leafTargets) { for (leafTarget in leafTargets) {
val leafTargetModuleAsExpected: SerializedMetadata = analyzedModules.commonizedModules.getValue(leafTarget) val leafTargetModuleAsExpected: SerializedMetadata = analyzedModules.commonizedModules.getValue(leafTarget)
val leafTargetModuleByCommonizer: SerializedMetadata = val leafTargetModuleByCommonizer: SerializedMetadata =
(result.modulesByTargets.getValue(leafTarget).single() as ModuleResult.Commonized).metadata (results.modulesByTargets.getValue(leafTarget).single() as ModuleResult.Commonized).metadata
assertModulesAreEqual(leafTargetModuleAsExpected, leafTargetModuleByCommonizer, leafTarget) assertModulesAreEqual(leafTargetModuleAsExpected, leafTargetModuleByCommonizer, leafTarget)
} }
@@ -203,26 +206,24 @@ private class AnalyzedModules(
check(allTargets.containsAll(dependeeModules.keys)) check(allTargets.containsAll(dependeeModules.keys))
} }
fun toCommonizationParameters(): CommonizerParameters { fun toCommonizerParameters(resultsConsumer: ResultsConsumer) =
val parameters = CommonizerParameters() CommonizerParameters().also { parameters ->
parameters.resultsConsumer = resultsConsumer
parameters.dependeeModulesProvider = dependeeModules[sharedTarget]?.let(MockModulesProvider::create)
leafTargets.forEach { leafTarget -> leafTargets.forEach { leafTarget ->
val originalModule = originalModules.getValue(leafTarget) val originalModule = originalModules.getValue(leafTarget)
parameters.addTarget( parameters.addTarget(
TargetProvider( TargetProvider(
target = leafTarget, target = leafTarget,
modulesProvider = MockModulesProvider.create(originalModule), modulesProvider = MockModulesProvider.create(originalModule),
dependeeModulesProvider = dependeeModules[leafTarget]?.let(MockModulesProvider::create) dependeeModulesProvider = dependeeModules[leafTarget]?.let(MockModulesProvider::create)
)
) )
) }
} }
parameters.dependeeModulesProvider = dependeeModules[sharedTarget]?.let(MockModulesProvider::create)
return parameters
}
companion object { companion object {
fun create( fun create(
sourceModuleRoots: SourceModuleRoots, sourceModuleRoots: SourceModuleRoots,
@@ -5,11 +5,14 @@
package org.jetbrains.kotlin.descriptors.commonizer package org.jetbrains.kotlin.descriptors.commonizer
import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer.ModuleResult
import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer.Status
import org.jetbrains.kotlin.descriptors.commonizer.utils.MockResultsConsumer
import org.jetbrains.kotlin.descriptors.commonizer.utils.MockModulesProvider import org.jetbrains.kotlin.descriptors.commonizer.utils.MockModulesProvider
import org.jetbrains.kotlin.descriptors.commonizer.utils.assertCommonizationPerformed
import org.junit.Test import org.junit.Test
import kotlin.contracts.ExperimentalContracts import kotlin.contracts.ExperimentalContracts
import kotlin.test.assertEquals import kotlin.test.assertEquals
import kotlin.test.assertTrue
@ExperimentalContracts @ExperimentalContracts
class CommonizerFacadeTest { class CommonizerFacadeTest {
@@ -59,26 +62,32 @@ class CommonizerFacadeTest {
) )
companion object { companion object {
private fun Map<String, List<String>>.toCommonizationParameters() = CommonizerParameters().also { private fun Map<String, List<String>>.toCommonizerParameters(resultsConsumer: ResultsConsumer) =
forEach { (targetName, moduleNames) -> CommonizerParameters().also { parameters ->
it.addTarget( parameters.resultsConsumer = resultsConsumer
TargetProvider(
target = LeafTarget(targetName), forEach { (targetName, moduleNames) ->
modulesProvider = MockModulesProvider.create(moduleNames), parameters.addTarget(
dependeeModulesProvider = null TargetProvider(
target = LeafTarget(targetName),
modulesProvider = MockModulesProvider.create(moduleNames),
dependeeModulesProvider = null
)
) )
) }
} }
}
private fun doTestNothingToCommonize(originalModules: Map<String, List<String>>) { private fun doTestNothingToCommonize(originalModules: Map<String, List<String>>) {
val result = runCommonization(originalModules.toCommonizationParameters()) val results = MockResultsConsumer()
assertEquals(CommonizerResult.NothingToDo, result) runCommonization(originalModules.toCommonizerParameters(results))
assertEquals(results.status, Status.NOTHING_TO_DO)
assertTrue(results.modulesByTargets.isEmpty())
} }
private fun doTestSuccessfulCommonization(originalModules: Map<String, List<String>>) { private fun doTestSuccessfulCommonization(originalModules: Map<String, List<String>>) {
val result = runCommonization(originalModules.toCommonizationParameters()) val results = MockResultsConsumer()
assertCommonizationPerformed(result) runCommonization(originalModules.toCommonizerParameters(results))
assertEquals(results.status, Status.DONE)
val expectedCommonModuleNames = mutableSetOf<String>() val expectedCommonModuleNames = mutableSetOf<String>()
originalModules.values.forEachIndexed { index, moduleNames -> originalModules.values.forEachIndexed { index, moduleNames ->
@@ -90,17 +99,17 @@ class CommonizerFacadeTest {
assertModulesMatch( assertModulesMatch(
expectedCommonizedModuleNames = expectedCommonModuleNames, expectedCommonizedModuleNames = expectedCommonModuleNames,
expectedMissingModuleNames = emptySet(), expectedMissingModuleNames = emptySet(),
actualModuleResults = result.modulesByTargets.getValue(result.sharedTarget) actualModuleResults = results.modulesByTargets.getValue(results.sharedTarget)
) )
result.leafTargets.forEach { target -> results.leafTargets.forEach { target ->
val allModuleNames = originalModules.getValue(target.name).toSet() val allModuleNames = originalModules.getValue(target.name).toSet()
val expectedMissingModuleNames = allModuleNames - expectedCommonModuleNames val expectedMissingModuleNames = allModuleNames - expectedCommonModuleNames
assertModulesMatch( assertModulesMatch(
expectedCommonizedModuleNames = expectedCommonModuleNames, expectedCommonizedModuleNames = expectedCommonModuleNames,
expectedMissingModuleNames = expectedMissingModuleNames, expectedMissingModuleNames = expectedMissingModuleNames,
actualModuleResults = result.modulesByTargets.getValue(target) actualModuleResults = results.modulesByTargets.getValue(target)
) )
} }
} }
@@ -6,7 +6,6 @@
package org.jetbrains.kotlin.descriptors.commonizer.utils package org.jetbrains.kotlin.descriptors.commonizer.utils
import kotlinx.metadata.klib.KlibModuleMetadata import kotlinx.metadata.klib.KlibModuleMetadata
import org.jetbrains.kotlin.descriptors.commonizer.CommonizerResult
import org.jetbrains.kotlin.descriptors.commonizer.CommonizerTarget import org.jetbrains.kotlin.descriptors.commonizer.CommonizerTarget
import org.jetbrains.kotlin.descriptors.commonizer.metadata.utils.MetadataDeclarationsComparator import org.jetbrains.kotlin.descriptors.commonizer.metadata.utils.MetadataDeclarationsComparator
import org.jetbrains.kotlin.descriptors.commonizer.metadata.utils.MetadataDeclarationsComparator.Mismatch import org.jetbrains.kotlin.descriptors.commonizer.metadata.utils.MetadataDeclarationsComparator.Mismatch
@@ -15,7 +14,6 @@ import org.jetbrains.kotlin.descriptors.commonizer.metadata.utils.SerializedMeta
import org.jetbrains.kotlin.library.SerializedMetadata import org.jetbrains.kotlin.library.SerializedMetadata
import java.io.File import java.io.File
import kotlin.contracts.ExperimentalContracts import kotlin.contracts.ExperimentalContracts
import kotlin.contracts.contract
import kotlin.test.fail import kotlin.test.fail
fun assertIsDirectory(file: File) { fun assertIsDirectory(file: File) {
@@ -23,16 +21,6 @@ fun assertIsDirectory(file: File) {
fail("Not a directory: $file") fail("Not a directory: $file")
} }
@ExperimentalContracts
fun assertCommonizationPerformed(result: CommonizerResult) {
contract {
returns() implies (result is CommonizerResult.Done)
}
if (result !is CommonizerResult.Done)
fail("$result is not instance of ${CommonizerResult.Done::class}")
}
@ExperimentalContracts @ExperimentalContracts
fun assertModulesAreEqual(reference: SerializedMetadata, generated: SerializedMetadata, target: CommonizerTarget) { fun assertModulesAreEqual(reference: SerializedMetadata, generated: SerializedMetadata, target: CommonizerTarget) {
val referenceModule = with(reference) { KlibModuleMetadata.read(SerializedMetadataLibraryProvider(module, fragments, fragmentNames)) } val referenceModule = with(reference) { KlibModuleMetadata.read(SerializedMetadataLibraryProvider(module, fragments, fragmentNames)) }
@@ -7,7 +7,8 @@ package org.jetbrains.kotlin.descriptors.commonizer.utils
import org.jetbrains.kotlin.descriptors.* import org.jetbrains.kotlin.descriptors.*
import org.jetbrains.kotlin.descriptors.annotations.Annotations import org.jetbrains.kotlin.descriptors.annotations.Annotations
import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider import org.jetbrains.kotlin.descriptors.commonizer.*
import org.jetbrains.kotlin.descriptors.commonizer.ResultsConsumer.ModuleResult
import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider.ModuleInfo import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider.ModuleInfo
import org.jetbrains.kotlin.descriptors.commonizer.cir.factory.CirClassFactory import org.jetbrains.kotlin.descriptors.commonizer.cir.factory.CirClassFactory
import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.* import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.*
@@ -175,3 +176,23 @@ internal class MockModulesProvider private constructor(
) )
} }
} }
internal class MockResultsConsumer : ResultsConsumer {
private val _modulesByTargets = LinkedHashMap<CommonizerTarget, Collection<ModuleResult>>() // use linked hash map to preserve order
val modulesByTargets: Map<CommonizerTarget, Collection<ModuleResult>>
get() = _modulesByTargets
val sharedTarget: SharedTarget by lazy { modulesByTargets.keys.filterIsInstance<SharedTarget>().single() }
val leafTargets: Set<LeafTarget> by lazy { modulesByTargets.keys.filterIsInstance<LeafTarget>().toSet() }
lateinit var status: ResultsConsumer.Status
override fun consumeResults(target: CommonizerTarget, moduleResults: Collection<ModuleResult>) {
val oldValue = _modulesByTargets.put(target, moduleResults)
check(oldValue == null)
}
override fun successfullyFinished(status: ResultsConsumer.Status) {
this.status = status
}
}