Support generating a klib from common code with K2MetadataCompiler

This commit is contained in:
Sergey Igushkin
2019-12-11 16:14:49 +03:00
parent 28e0089259
commit ed6034e449
7 changed files with 313 additions and 43 deletions
+4
View File
@@ -25,6 +25,10 @@ dependencies {
compile(project(":compiler:fir:jvm"))
compile(project(":compiler:fir:java"))
compile(project(":compiler:fir:fir2ir"))
compile(project(":kotlin-util-klib"))
compile(project(":kotlin-util-io"))
compile(project(":compiler:ir.serialization.common"))
compile(toolsJar())
compileOnly(intellijCoreDep()) { includeJars("intellij-core") }
compileOnly(intellijDep()) { includeIntellijCoreJarDependencies(project) }
@@ -17,10 +17,8 @@
package org.jetbrains.kotlin.analyzer.common
import com.intellij.openapi.components.ServiceManager
import com.intellij.openapi.project.Project
import com.intellij.psi.search.GlobalSearchScope
import org.jetbrains.kotlin.analyzer.*
import org.jetbrains.kotlin.builtins.DefaultBuiltIns
import org.jetbrains.kotlin.config.LanguageFeature
import org.jetbrains.kotlin.config.LanguageVersionSettings
import org.jetbrains.kotlin.container.StorageComponentContainer
@@ -30,6 +28,7 @@ import org.jetbrains.kotlin.container.useInstance
import org.jetbrains.kotlin.context.ModuleContext
import org.jetbrains.kotlin.context.ProjectContext
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
import org.jetbrains.kotlin.descriptors.impl.CompositePackageFragmentProvider
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.frontend.di.configureModule
@@ -38,7 +37,6 @@ import org.jetbrains.kotlin.load.kotlin.MetadataFinderFactory
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.platform.CommonPlatforms
import org.jetbrains.kotlin.platform.TargetPlatform
import org.jetbrains.kotlin.platform.TargetPlatformVersion
import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.resolve.*
import org.jetbrains.kotlin.resolve.checkers.ExpectedActualDeclarationChecker
@@ -60,14 +58,16 @@ class CommonResolverForModuleFactory(
private val platformParameters: CommonAnalysisParameters,
private val targetEnvironment: TargetEnvironment,
private val targetPlatform: TargetPlatform,
private val shouldCheckExpectActual: Boolean
private val shouldCheckExpectActual: Boolean,
private val commonDependenciesContainer: CommonDependenciesContainer? = null
) : ResolverForModuleFactory() {
private class SourceModuleInfo(
override val name: Name,
override val capabilities: Map<ModuleDescriptor.Capability<*>, Any?>,
private val dependencies: Iterable<ModuleInfo>,
private val dependOnOldBuiltIns: Boolean
) : ModuleInfo {
override fun dependencies() = listOf(this)
override fun dependencies() = listOf(this, *dependencies.toList().toTypedArray() )
override fun dependencyOnBuiltIns(): ModuleInfo.DependencyOnBuiltIns =
if (dependOnOldBuiltIns) ModuleInfo.DependencyOnBuiltIns.LAST else ModuleInfo.DependencyOnBuiltIns.NONE
@@ -101,10 +101,13 @@ class CommonResolverForModuleFactory(
languageVersionSettings, CommonPlatforms.defaultCommonPlatform, CommonPlatformAnalyzerServices, shouldCheckExpectActual
)
val packageFragmentProviders = listOf(
container.get<ResolveSession>().packageFragmentProvider,
container.get<MetadataPackageFragmentProvider>()
)
val packageFragmentProviders =
/** If this is a dependency module that [commonDependenciesContainer] knows about, get the package fragments from there */
commonDependenciesContainer?.packageFragmentProviderForModuleInfo(moduleInfo)?.let(::listOf)
?: listOf(
container.get<ResolveSession>().packageFragmentProvider,
container.get<MetadataPackageFragmentProvider>()
)
return ResolverForModule(CompositePackageFragmentProvider(packageFragmentProviders), container)
}
@@ -113,9 +116,15 @@ class CommonResolverForModuleFactory(
fun analyzeFiles(
files: Collection<KtFile>, moduleName: Name, dependOnBuiltIns: Boolean, languageVersionSettings: LanguageVersionSettings,
capabilities: Map<ModuleDescriptor.Capability<*>, Any?> = emptyMap(),
dependenciesContainer: CommonDependenciesContainer? = null,
metadataPartProviderFactory: (ModuleContent<ModuleInfo>) -> MetadataPartProvider
): AnalysisResult {
val moduleInfo = SourceModuleInfo(moduleName, capabilities, dependOnBuiltIns)
val moduleInfo = SourceModuleInfo(
moduleName,
capabilities,
dependenciesContainer?.moduleInfos?.toList().orEmpty(),
dependOnBuiltIns
)
val project = files.firstOrNull()?.project ?: throw AssertionError("No files to analyze")
val multiplatformLanguageSettings = object : LanguageVersionSettings by languageVersionSettings {
@@ -128,21 +137,24 @@ class CommonResolverForModuleFactory(
CommonAnalysisParameters(metadataPartProviderFactory),
CompilerEnvironment,
CommonPlatforms.defaultCommonPlatform,
shouldCheckExpectActual = false
shouldCheckExpectActual = false,
dependenciesContainer
)
@Suppress("NAME_SHADOWING")
val resolver = ResolverForSingleModuleProject(
val resolver = ResolverForSingleModuleProject<ModuleInfo>(
"sources for metadata serializer",
ProjectContext(project, "metadata serializer"),
moduleInfo,
resolverForModuleFactory,
GlobalSearchScope.allScope(project),
languageVersionSettings = multiplatformLanguageSettings,
syntheticFiles = files
syntheticFiles = files,
dependencyModules = dependenciesContainer?.moduleInfos ?: emptyList()
)
val moduleDescriptor = resolver.descriptorForModule(moduleInfo)
val container = resolver.resolverForModule(moduleInfo).componentProvider
container.get<LazyTopDownAnalyzer>().analyzeDeclarations(TopDownAnalysisMode.TopLevelDeclarations, files)
@@ -152,6 +164,12 @@ class CommonResolverForModuleFactory(
}
}
interface CommonDependenciesContainer {
val moduleInfos: List<ModuleInfo>
fun packageFragmentProviderForModuleInfo(moduleInfo: ModuleInfo): PackageFragmentProvider?
}
private fun createContainerToResolveCommonCode(
moduleContext: ModuleContext,
bindingTrace: BindingTrace,
@@ -0,0 +1,59 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.cli.metadata
import org.jetbrains.kotlin.analyzer.common.CommonDependenciesContainer
import org.jetbrains.kotlin.analyzer.common.CommonResolverForModuleFactory
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
import org.jetbrains.kotlin.config.CommonConfigurationKeys
import org.jetbrains.kotlin.config.languageVersionSettings
import org.jetbrains.kotlin.name.Name
import java.io.File
internal val KotlinCoreEnvironment.destDir: File?
get() = configuration.get(CLIConfigurationKeys.METADATA_DESTINATION_DIRECTORY)
internal fun runCommonAnalysisForSerialization(
environment: KotlinCoreEnvironment,
dependOnBuiltins: Boolean,
dependencyContainer: CommonDependenciesContainer?
): AnalyzerWithCompilerReport? {
if (environment.destDir == null) {
val configuration = environment.configuration
val messageCollector = configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)
messageCollector.report(CompilerMessageSeverity.ERROR, "Specify destination via -d")
return null
}
return runCommonAnalysis(environment, dependOnBuiltins, dependencyContainer)
}
private fun runCommonAnalysis(
environment: KotlinCoreEnvironment,
dependOnBuiltins: Boolean,
dependencyContainer: CommonDependenciesContainer?
): AnalyzerWithCompilerReport {
val configuration = environment.configuration
val messageCollector = configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)
val files = environment.getSourceFiles()
val moduleName = Name.special("<${configuration.getNotNull(CommonConfigurationKeys.MODULE_NAME)}>")
val analyzer = AnalyzerWithCompilerReport(messageCollector, configuration.languageVersionSettings)
analyzer.analyzeAndReport(files) {
CommonResolverForModuleFactory.analyzeFiles(
files, moduleName, dependOnBuiltins, configuration.languageVersionSettings,
dependenciesContainer = dependencyContainer
) { content ->
environment.createPackagePartProvider(content.moduleContentScope)
}
}
return analyzer
}
@@ -101,7 +101,11 @@ class K2MetadataCompiler : CLICompiler<K2MetadataCompilerArguments>() {
try {
val metadataVersion =
configuration.get(CommonConfigurationKeys.METADATA_VERSION) as? BuiltInsBinaryVersion ?: BuiltInsBinaryVersion.INSTANCE
MetadataSerializer(metadataVersion, true).serialize(environment)
if (arguments.klibBasedMpp) {
K2MetadataKlibSerializer(metadataVersion).serialize(environment)
} else {
MetadataSerializer(metadataVersion, true).serialize(environment)
}
} catch (e: CompilationException) {
collector.report(EXCEPTION, OutputMessageUtil.renderException(e), MessageUtil.psiElementToMessageLocation(e.element))
return ExitCode.INTERNAL_ERROR
@@ -0,0 +1,202 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.cli.metadata
import org.jetbrains.kotlin.analyzer.ModuleInfo
import org.jetbrains.kotlin.analyzer.common.CommonDependenciesContainer
import org.jetbrains.kotlin.analyzer.common.CommonPlatformAnalyzerServices
import org.jetbrains.kotlin.backend.common.serialization.DescriptorTable
import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer
import org.jetbrains.kotlin.builtins.DefaultBuiltIns
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
import org.jetbrains.kotlin.cli.jvm.config.JvmClasspathRoot
import org.jetbrains.kotlin.cli.jvm.config.JvmContentRoot
import org.jetbrains.kotlin.config.CommonConfigurationKeys
import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.config.KotlinCompilerVersion
import org.jetbrains.kotlin.config.languageVersionSettings
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.descriptors.konan.DeserializedKlibModuleOrigin
import org.jetbrains.kotlin.konan.util.KlibMetadataFactories
import org.jetbrains.kotlin.library.KotlinAbiVersion
import org.jetbrains.kotlin.library.KotlinLibrary
import org.jetbrains.kotlin.library.KotlinLibraryVersioning
import org.jetbrains.kotlin.library.SerializedMetadata
import org.jetbrains.kotlin.library.impl.buildKoltinLibrary
import org.jetbrains.kotlin.library.impl.createKotlinLibrary
import org.jetbrains.kotlin.library.metadata.parseModuleHeader
import org.jetbrains.kotlin.metadata.builtins.BuiltInsBinaryVersion
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.platform.CommonPlatforms
import org.jetbrains.kotlin.platform.TargetPlatform
import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
import org.jetbrains.kotlin.resolve.PlatformDependentAnalyzerServices
import org.jetbrains.kotlin.serialization.konan.impl.KlibMetadataModuleDescriptorFactoryImpl
import org.jetbrains.kotlin.storage.LockBasedStorageManager
import org.jetbrains.kotlin.storage.StorageManager
import org.jetbrains.kotlin.utils.keysToMap
import java.io.File
internal class K2MetadataKlibSerializer(private val metadataVersion: BuiltInsBinaryVersion) {
fun serialize(environment: KotlinCoreEnvironment) {
val configuration = environment.configuration
val dependencyContainer = KlibMetadataDependencyContainer(
configuration,
LockBasedStorageManager("K2MetadataKlibSerializer")
)
val analyzer = runCommonAnalysisForSerialization(environment, false, dependencyContainer)
if (analyzer == null || analyzer.hasErrors()) return
val (_, moduleDescriptor) = analyzer.analysisResult
val destDir = checkNotNull(environment.destDir)
performSerialization(configuration, moduleDescriptor, destDir)
}
private fun performSerialization(
configuration: CompilerConfiguration,
module: ModuleDescriptor,
destDir: File
) {
val serializedMetadata: SerializedMetadata = KlibMetadataMonolithicSerializer(
configuration.languageVersionSettings,
metadataVersion,
DescriptorTable.createDefault(),
skipExpects = false,
includeOnlyModuleContent = true
).serializeModule(module)
val versions = KotlinLibraryVersioning(
abiVersion = KotlinAbiVersion.CURRENT,
libraryVersion = null,
compilerVersion = KotlinCompilerVersion.getVersion(),
metadataVersion = null,
irVersion = null
)
buildKoltinLibrary(
emptyList(),
serializedMetadata,
null,
versions,
destDir.absolutePath,
configuration[CommonConfigurationKeys.MODULE_NAME]!!,
nopack = true,
manifestProperties = null,
dataFlowGraph = null
)
}
}
private class KlibMetadataDependencyContainer(
private val configuration: CompilerConfiguration,
private val storageManager: StorageManager
) : CommonDependenciesContainer {
private val kotlinLibraries = run {
val classpathFiles =
configuration.getList(CLIConfigurationKeys.CONTENT_ROOTS).filterIsInstance<JvmClasspathRoot>().map(JvmContentRoot::file)
val klibFiles = classpathFiles
.filter { it.extension == "klib" || it.isDirectory }
klibFiles.map { createKotlinLibrary(org.jetbrains.kotlin.konan.file.File(it.absolutePath)) }
}
private val builtIns
get() = DefaultBuiltIns.Instance
private class KlibModuleInfo(
override val name: Name,
val kotlinLibrary: KotlinLibrary,
private val dependOnKlibModules: List<ModuleInfo>
) : ModuleInfo {
override fun dependencies(): List<ModuleInfo> = dependOnKlibModules
override fun dependencyOnBuiltIns(): ModuleInfo.DependencyOnBuiltIns = ModuleInfo.DependencyOnBuiltIns.LAST
override val platform: TargetPlatform
get() = CommonPlatforms.defaultCommonPlatform
override val analyzerServices: PlatformDependentAnalyzerServices
get() = CommonPlatformAnalyzerServices
}
private val moduleDescriptorsForKotlinLibraries: Map<KotlinLibrary, ModuleDescriptorImpl> =
kotlinLibraries.keysToMap { library ->
val moduleHeader = parseModuleHeader(library.moduleHeaderData)
val moduleName = Name.special(moduleHeader.moduleName)
val moduleOrigin = DeserializedKlibModuleOrigin(library)
MetadataFactories.DefaultDescriptorFactory.createDescriptor(
moduleName, storageManager, builtIns, moduleOrigin
)
}.also { result ->
val resultValues = result.values.toList()
val dependenciesForModuleDescriptors = resultValues + builtIns.builtInsModule
resultValues.forEach { module ->
module.setDependencies(dependenciesForModuleDescriptors)
}
}
override val moduleInfos: List<ModuleInfo> = mutableListOf<KlibModuleInfo>().apply {
addAll(
moduleDescriptorsForKotlinLibraries.map { (kotlinLibrary, moduleDescriptor) ->
KlibModuleInfo(moduleDescriptor.name, kotlinLibrary, this@apply)
}
)
}
override fun packageFragmentProviderForModuleInfo(
moduleInfo: ModuleInfo
): PackageFragmentProvider? {
if (moduleInfo !in moduleInfos)
return null
moduleInfo as KlibModuleInfo
return packageFragmentProviderForKotlinLibrary(moduleInfo.kotlinLibrary)
}
private val klibMetadataModuleDescriptorFactory by lazy {
KlibMetadataModuleDescriptorFactoryImpl(
MetadataFactories.DefaultDescriptorFactory,
MetadataFactories.DefaultPackageFragmentsFactory,
MetadataFactories.flexibleTypeDeserializer
)
}
private fun packageFragmentProviderForKotlinLibrary(
library: KotlinLibrary
): PackageFragmentProvider {
val languageVersionSettings = configuration.languageVersionSettings
val libraryModuleDescriptor = moduleDescriptorsForKotlinLibraries.getValue(library)
val packageFragmentNames = parseModuleHeader(library.moduleHeaderData).packageFragmentNameList
return klibMetadataModuleDescriptorFactory.createPackageFragmentProvider(
library,
packageAccessHandler = null,
packageFragmentNames = packageFragmentNames,
storageManager = LockBasedStorageManager("KlibMetadataPackageFragmentProvider"),
moduleDescriptor = libraryModuleDescriptor,
configuration = CompilerDeserializationConfiguration(languageVersionSettings),
compositePackageFragmentAddend = null
).also {
libraryModuleDescriptor.initialize(it)
}
}
}
private val MetadataFactories =
KlibMetadataFactories(
{ DefaultBuiltIns.Instance },
org.jetbrains.kotlin.serialization.konan.NullFlexibleTypeDeserializer
)
@@ -16,14 +16,8 @@
package org.jetbrains.kotlin.cli.metadata
import org.jetbrains.kotlin.analyzer.common.CommonResolverForModuleFactory
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
import org.jetbrains.kotlin.codegen.JvmCodegenUtil
import org.jetbrains.kotlin.config.CommonConfigurationKeys
import org.jetbrains.kotlin.config.languageVersionSettings
import org.jetbrains.kotlin.descriptors.ClassDescriptor
import org.jetbrains.kotlin.descriptors.ClassKind
import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
@@ -37,7 +31,6 @@ import org.jetbrains.kotlin.metadata.jvm.deserialization.PackageParts
import org.jetbrains.kotlin.metadata.jvm.deserialization.serializeToByteArray
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.psi.*
import org.jetbrains.kotlin.resolve.BindingContext
import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter
@@ -56,28 +49,14 @@ open class MetadataSerializer(
protected var totalFiles = 0
fun serialize(environment: KotlinCoreEnvironment) {
val configuration = environment.configuration
val messageCollector = configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)
val files = environment.getSourceFiles()
val moduleName = Name.special("<${configuration.getNotNull(CommonConfigurationKeys.MODULE_NAME)}>")
val analyzer = runCommonAnalysisForSerialization(environment, dependOnOldBuiltIns, dependencyContainer = null)
val destDir = configuration.get(CLIConfigurationKeys.METADATA_DESTINATION_DIRECTORY) ?: run {
messageCollector.report(CompilerMessageSeverity.ERROR, "Specify destination via -d")
return
}
val analyzer = AnalyzerWithCompilerReport(messageCollector, configuration.languageVersionSettings)
analyzer.analyzeAndReport(files) {
CommonResolverForModuleFactory.analyzeFiles(files, moduleName, dependOnOldBuiltIns, configuration.languageVersionSettings) { content ->
environment.createPackagePartProvider(content.moduleContentScope)
}
}
if (analyzer.hasErrors()) return
if (analyzer == null || analyzer.hasErrors()) return
val (bindingContext, moduleDescriptor) = analyzer.analysisResult
performSerialization(files, bindingContext, moduleDescriptor, destDir)
val destDir = checkNotNull(environment.destDir)
performSerialization(environment.getSourceFiles(), bindingContext, moduleDescriptor, destDir)
}
protected open fun performSerialization(
@@ -19,24 +19,28 @@ import org.jetbrains.kotlin.psi.KtFile
class ResolverForSingleModuleProject<M : ModuleInfo>(
debugName: String,
projectContext: ProjectContext,
module: M,
private val module: M,
private val resolverForModuleFactory: ResolverForModuleFactory,
private val searchScope: GlobalSearchScope,
private val builtIns: KotlinBuiltIns = DefaultBuiltIns.Instance,
private val languageVersionSettings: LanguageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
private val syntheticFiles: Collection<KtFile> = emptyList(),
private val sdkDependency: M? = null
private val sdkDependency: M? = null,
dependencyModules: Iterable<M> = emptyList()
) : AbstractResolverForProject<M>(
debugName,
projectContext,
listOf(module),
listOf(module) + dependencyModules,
null,
EmptyResolverForProject(),
PackageOracleFactory.OptimisticFactory
) {
override fun sdkDependency(module: M): M? = sdkDependency
override fun modulesContent(module: M): ModuleContent<M> = ModuleContent(module, syntheticFiles, searchScope)
override fun modulesContent(module: M): ModuleContent<M> = when (module) {
this.module -> ModuleContent(module, syntheticFiles, searchScope)
else -> ModuleContent(module, emptyList(), searchScope)
}
override fun builtInsForModule(module: M): KotlinBuiltIns = builtIns