From d610837caff0e6ef59fa9c40aeb2a6da207d8d3d Mon Sep 17 00:00:00 2001 From: Dmitriy Dolovov Date: Mon, 19 Oct 2020 12:31:42 +0300 Subject: [PATCH] [Commonizer] Integrate metadata builder with the commonizer environment --- .../commonizer/CommonizerParameters.kt | 3 + .../commonizer/CommonizerResult.kt | 13 ++++- .../descriptors/commonizer/builder/context.kt | 5 +- .../kotlin/descriptors/commonizer/facade.kt | 55 +++++++++++++++---- .../konan/NativeDistributionCommonizer.kt | 43 +++------------ .../AbstractCommonizationFromSourcesTest.kt | 5 +- .../commonizer/CommonizerFacadeTest.kt | 2 +- 7 files changed, 77 insertions(+), 49 deletions(-) diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerParameters.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerParameters.kt index ca1b747521f..736d29983ec 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerParameters.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerParameters.kt @@ -24,6 +24,9 @@ class CommonizerParameters( field = value } + // To be removed. Used only for tests now. + internal var generateDescriptors: Boolean = false + fun addTarget(targetProvider: TargetProvider): CommonizerParameters { require(targetProvider.target !in _targetProviders) { "Target ${targetProvider.target} is already added" } _targetProviders[targetProvider.target] = targetProvider diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerResult.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerResult.kt index d63906befbc..bca1bb3bb1c 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerResult.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerResult.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.descriptors.commonizer import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.library.SerializedMetadata import java.io.File sealed class CommonizerResult { @@ -21,5 +22,15 @@ sealed class CommonizerResult { sealed class ModuleResult { class Missing(val originalLocation: File) : ModuleResult() - class Commonized(val module: ModuleDescriptor) : ModuleResult() + + class Commonized( + @Deprecated("To be removed. Used only for tests now.") + internal val module: ModuleDescriptor?, + val metadata: LibraryMetadata + ) : ModuleResult() } + +class LibraryMetadata( + val libraryName: String, + val metadata: SerializedMetadata +) diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt index f7304d7e9b3..adca0e6ec3a 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt @@ -218,7 +218,10 @@ class LazyClassifierLookupTable(lazyModules: Map> fun CirRootNode.createGlobalBuilderComponents( storageManager: StorageManager, parameters: CommonizerParameters -): GlobalDeclarationsBuilderComponents { +): GlobalDeclarationsBuilderComponents? { + if (!parameters.generateDescriptors) + return null + val cache = DeclarationsBuilderCache(dimension) val lazyCommonDependeeModules = storageManager.createLazyValue { diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt index 9a8dbbe6f1f..50cbf13dc9a 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt @@ -5,12 +5,19 @@ package org.jetbrains.kotlin.descriptors.commonizer +import kotlinx.metadata.klib.ChunkedKlibModuleFragmentWriteStrategy +import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.descriptors.commonizer.builder.DeclarationsBuilderVisitor1 import org.jetbrains.kotlin.descriptors.commonizer.builder.DeclarationsBuilderVisitor2 import org.jetbrains.kotlin.descriptors.commonizer.builder.createGlobalBuilderComponents import org.jetbrains.kotlin.descriptors.commonizer.core.CommonizationVisitor import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.* +import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirNode.Companion.dimension import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirTreeMerger.CirTreeMergeResult +import org.jetbrains.kotlin.descriptors.commonizer.metadata.MetadataBuilder +import org.jetbrains.kotlin.descriptors.commonizer.utils.strip +import org.jetbrains.kotlin.library.SerializedMetadata +import org.jetbrains.kotlin.serialization.konan.impl.KlibResolvedModuleDescriptorsFactoryImpl.Companion.FORWARD_DECLARATIONS_MODULE_NAME import org.jetbrains.kotlin.storage.LockBasedStorageManager import org.jetbrains.kotlin.storage.StorageManager @@ -24,16 +31,46 @@ fun runCommonization(parameters: CommonizerParameters): CommonizerResult { val mergedTree = mergeResult.root // build resulting descriptors: - val components = mergedTree.createGlobalBuilderComponents(storageManager, parameters) - mergedTree.accept(DeclarationsBuilderVisitor1(components), emptyList()) - mergedTree.accept(DeclarationsBuilderVisitor2(components), emptyList()) - val modulesByTargets = LinkedHashMap>() // use linked hash map to preserve order - components.targetComponents.forEach { component -> - val target = component.target - check(target !in modulesByTargets) + val klibFragmentWriteStrategy = ChunkedKlibModuleFragmentWriteStrategy() - val commonizedModules: List = components.cache.getAllModules(component.index).map(ModuleResult::Commonized) + // optional part for generating descriptors: begin + val components = mergedTree.createGlobalBuilderComponents(storageManager, parameters) + if (components != null) { + mergedTree.accept(DeclarationsBuilderVisitor1(components), emptyList()) + mergedTree.accept(DeclarationsBuilderVisitor2(components), emptyList()) + } + // optional part for generating descriptors: end + + for (targetIndex in 0 until mergedTree.dimension) { + val (target, metadataModules) = MetadataBuilder.build(mergedTree, targetIndex) + + // optional part for generating descriptors: begin + val moduleDescriptors: Map? = components?.targetComponents?.get(targetIndex)?.let { component -> + check(component.target == target) + check(component.index == targetIndex) + + components.cache.getAllModules(targetIndex) + .filter { it.name != FORWARD_DECLARATIONS_MODULE_NAME } + .associateBy { it.name.strip() } + } + // optional part for generating descriptors: end + + val commonizedModules: List = metadataModules.map { metadataModule -> + val libraryName = metadataModule.name + val serializedMetadata = with(metadataModule.write(klibFragmentWriteStrategy)) { + SerializedMetadata(header, fragments, fragmentNames) + } + + val libraryMetadata = LibraryMetadata(libraryName, serializedMetadata) + + // optional part for generating descriptors: begin + val moduleDescriptor = moduleDescriptors?.get(libraryName) + // optional part for generating descriptors: end + + ModuleResult.Commonized(moduleDescriptor, libraryMetadata) + } + parameters.progressLogger?.invoke("Built metadata for target [$target]") val missingModules: List = if (target is LeafTarget) mergeResult.missingModuleInfos.getValue(target).map { ModuleResult.Missing(it.originalLocation) } @@ -42,8 +79,6 @@ fun runCommonization(parameters: CommonizerParameters): CommonizerResult { modulesByTargets[target] = commonizedModules + missingModules } - parameters.progressLogger?.invoke("Prepared new descriptors") - return CommonizerResult.Done(modulesByTargets) } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt index 318fa2a1bd6..581581352dd 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt @@ -5,19 +5,15 @@ package org.jetbrains.kotlin.descriptors.commonizer.konan -import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion import org.jetbrains.kotlin.backend.common.serialization.metadata.metadataVersion import org.jetbrains.kotlin.builtins.konan.KonanBuiltIns -import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl -import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.descriptors.commonizer.* import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeDistributionCommonizer.StatsType.* import org.jetbrains.kotlin.descriptors.commonizer.stats.AggregatedStatsCollector import org.jetbrains.kotlin.descriptors.commonizer.stats.FileStatsOutput import org.jetbrains.kotlin.descriptors.commonizer.stats.RawStatsCollector import org.jetbrains.kotlin.descriptors.commonizer.utils.ResettableClockMark -import org.jetbrains.kotlin.descriptors.konan.NATIVE_STDLIB_MODULE_NAME import org.jetbrains.kotlin.konan.library.* import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.library.KotlinLibrary @@ -28,8 +24,6 @@ import org.jetbrains.kotlin.library.impl.BuiltInsPlatform import org.jetbrains.kotlin.library.impl.KotlinLibraryLayoutForWriter import org.jetbrains.kotlin.library.impl.KotlinLibraryWriterImpl import org.jetbrains.kotlin.library.resolveSingleFileKlib -import org.jetbrains.kotlin.name.Name -import org.jetbrains.kotlin.serialization.konan.impl.KlibResolvedModuleDescriptorsFactoryImpl import org.jetbrains.kotlin.storage.LockBasedStorageManager import org.jetbrains.kotlin.util.Logger import java.io.File @@ -186,13 +180,6 @@ class NativeDistributionCommonizer( } is CommonizerResult.Done -> { - val serializer = KlibMetadataMonolithicSerializer( - languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT, - metadataVersion = KlibMetadataVersion.INSTANCE, - skipExpects = false, - project = null - ) - // 'targetsToCopy' are some targets with empty set of platform libraries val targetsToCopy = originalLibraries.librariesByTargets.keys - result.leafTargets if (targetsToCopy.isNotEmpty()) { @@ -206,7 +193,7 @@ class NativeDistributionCommonizer( val targetsToSerialize = result.leafTargets + result.sharedTarget targetsToSerialize.forEach { target -> val moduleResults: Collection = result.modulesByTargets.getValue(target) - val newModules: Collection = moduleResults.mapNotNull { (it as? ModuleResult.Commonized)?.module } + val newLibraries: Collection = moduleResults.mapNotNull { (it as? ModuleResult.Commonized)?.metadata } val missingModuleLocations: List = moduleResults.mapNotNull { (it as? ModuleResult.Missing)?.originalLocation } @@ -224,7 +211,7 @@ class NativeDistributionCommonizer( } val targetName = leafTargetNames.joinToString { if (it == starredTarget) "$it(*)" else it } - serializeTarget(target, targetName, newModules, missingModuleLocations, manifestProvider, serializer) + serializeTarget(target, targetName, newLibraries, missingModuleLocations, manifestProvider) } } } @@ -268,27 +255,20 @@ class NativeDistributionCommonizer( private fun serializeTarget( target: CommonizerTarget, targetName: String, - newModules: Collection, + newLibraries: Collection, missingModuleLocations: List, - manifestProvider: NativeManifestDataProvider, - serializer: KlibMetadataMonolithicSerializer + manifestProvider: NativeManifestDataProvider ) { val librariesDestination = target.librariesDestination librariesDestination.mkdirs() // always create an empty directory even if there is nothing to copy - for (newModule in newModules) { - val libraryName = newModule.name + for (newLibrary in newLibraries) { + val libraryName = newLibrary.libraryName - if (!shouldBeSerialized(libraryName)) - continue + val manifestData = manifestProvider.getManifest(libraryName) + val libraryDestination = librariesDestination.resolve(libraryName) - val metadata = serializer.serializeModule(newModule) - val plainName = libraryName.asString().removePrefix("<").removeSuffix(">") - - val manifestData = manifestProvider.getManifest(plainName) - val libraryDestination = librariesDestination.resolve(plainName) - - writeLibrary(metadata, manifestData, libraryDestination) + writeLibrary(newLibrary.metadata, manifestData, libraryDestination) } for (missingModuleLocation in missingModuleLocations) { @@ -330,9 +310,4 @@ class NativeDistributionCommonizer( is LeafTarget -> destination.resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR).resolve(name) is SharedTarget -> destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR) } - - private companion object { - fun shouldBeSerialized(libraryName: Name) = - libraryName != NATIVE_STDLIB_MODULE_NAME && libraryName != KlibResolvedModuleDescriptorsFactoryImpl.FORWARD_DECLARATIONS_MODULE_NAME - } } diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt index 098f985a000..d971f5cfd84 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt @@ -76,7 +76,7 @@ abstract class AbstractCommonizationFromSourcesTest : KtUsefulTestCase() { val sharedModuleAsExpected: ModuleDescriptor = analyzedModules.commonizedModules.getValue(sharedTarget) val sharedModuleByCommonizer: ModuleDescriptor = - (result.modulesByTargets.getValue(sharedTarget).single() as ModuleResult.Commonized).module + (result.modulesByTargets.getValue(sharedTarget).single() as ModuleResult.Commonized).module!! assertValidModule(sharedModuleAsExpected) assertValidModule(sharedModuleByCommonizer) @@ -88,7 +88,7 @@ abstract class AbstractCommonizationFromSourcesTest : KtUsefulTestCase() { for (leafTarget in leafTargets) { val leafTargetModuleAsExpected: ModuleDescriptor = analyzedModules.commonizedModules.getValue(leafTarget) val leafTargetModuleByCommonizer: ModuleDescriptor = - (result.modulesByTargets.getValue(leafTarget).single() as ModuleResult.Commonized).module + (result.modulesByTargets.getValue(leafTarget).single() as ModuleResult.Commonized).module!! assertValidModule(leafTargetModuleAsExpected) assertValidModule(leafTargetModuleByCommonizer) @@ -205,6 +205,7 @@ private class AnalyzedModules( fun toCommonizationParameters(): CommonizerParameters { val parameters = CommonizerParameters() + parameters.generateDescriptors = true leafTargets.forEach { leafTarget -> val originalModule = originalModules.getValue(leafTarget) diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt index 34a02d43c75..7a438c7ba64 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt @@ -121,7 +121,7 @@ class CommonizerFacadeTest { actualModuleResults.forEach { moduleResult -> when (moduleResult) { is ModuleResult.Commonized -> { - actualCommonizedModuleNames += moduleResult.module.name.asString().removeSurrounding("<", ">") + actualCommonizedModuleNames += moduleResult.metadata.libraryName } is ModuleResult.Missing -> { actualMissingModuleNames += moduleResult.originalLocation.name