From ce376f49fdce350feae753a030be2ae7f589517f Mon Sep 17 00:00:00 2001 From: Dmitriy Dolovov Date: Wed, 9 Oct 2019 13:43:58 +0700 Subject: [PATCH] [Commonizer] CLI for Kotlin/Native KLIBs --- ...libResolvedModuleDescriptorsFactoryImpl.kt | 35 ++- .../library/impl/KotlinLibraryWriterImpl.kt | 2 +- konan/commonizer/build.gradle.kts | 32 +- .../builder/DeclarationsBuilderVisitor1.kt | 6 - .../descriptors/commonizer/builder/context.kt | 23 +- .../kotlin/descriptors/commonizer/cli.kt | 283 ++++++++++++++++++ .../kotlin/descriptors/commonizer/utils.kt | 2 +- 7 files changed, 350 insertions(+), 33 deletions(-) create mode 100644 konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli.kt diff --git a/compiler/util-klib-metadata/src/org/jetbrains/kotlin/library/metadata/impl/KlibResolvedModuleDescriptorsFactoryImpl.kt b/compiler/util-klib-metadata/src/org/jetbrains/kotlin/library/metadata/impl/KlibResolvedModuleDescriptorsFactoryImpl.kt index 2cf3aa1e40b..ad58d39464e 100644 --- a/compiler/util-klib-metadata/src/org/jetbrains/kotlin/library/metadata/impl/KlibResolvedModuleDescriptorsFactoryImpl.kt +++ b/compiler/util-klib-metadata/src/org/jetbrains/kotlin/library/metadata/impl/KlibResolvedModuleDescriptorsFactoryImpl.kt @@ -71,26 +71,27 @@ class KlibResolvedModuleDescriptorsFactoryImpl( } fun createForwardDeclarationsModule( - builtIns: KotlinBuiltIns?, - storageManager: StorageManager): ModuleDescriptorImpl { + builtIns: KotlinBuiltIns?, + storageManager: StorageManager + ): ModuleDescriptorImpl { - val name = Name.special("") - val module = createDescriptorOptionalBuiltsIns(name, storageManager, builtIns, SyntheticModulesOrigin) + val module = createDescriptorOptionalBuiltsIns(FORWARD_DECLARATIONS_MODULE_NAME, storageManager, builtIns, SyntheticModulesOrigin) fun createPackage(fqName: FqName, supertypeName: String, classKind: ClassKind) = - ForwardDeclarationsPackageFragmentDescriptor( - storageManager, - module, - fqName, - Name.identifier(supertypeName), - classKind) + ForwardDeclarationsPackageFragmentDescriptor( + storageManager, + module, + fqName, + Name.identifier(supertypeName), + classKind + ) val packageFragmentProvider = PackageFragmentProviderImpl( - listOf( - createPackage(ForwardDeclarationsFqNames.cNamesStructs, "COpaque", ClassKind.CLASS), - createPackage(ForwardDeclarationsFqNames.objCNamesClasses, "ObjCObjectBase", ClassKind.CLASS), - createPackage(ForwardDeclarationsFqNames.objCNamesProtocols, "ObjCObject", ClassKind.INTERFACE) - ) + listOf( + createPackage(ForwardDeclarationsFqNames.cNamesStructs, "COpaque", ClassKind.CLASS), + createPackage(ForwardDeclarationsFqNames.objCNamesClasses, "ObjCObjectBase", ClassKind.CLASS), + createPackage(ForwardDeclarationsFqNames.objCNamesProtocols, "ObjCObject", ClassKind.INTERFACE) + ) ) module.initialize(packageFragmentProvider) @@ -119,6 +120,10 @@ class KlibResolvedModuleDescriptorsFactoryImpl( moduleDescriptorFactory.createDescriptor(library, languageVersionSettings, storageManager, builtIns, packageAccessHandler) else moduleDescriptorFactory.createDescriptorAndNewBuiltIns(library, languageVersionSettings, storageManager, packageAccessHandler) + + companion object { + val FORWARD_DECLARATIONS_MODULE_NAME = Name.special("") + } } /** diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryWriterImpl.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryWriterImpl.kt index 88755b16aeb..41fde845e5f 100644 --- a/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryWriterImpl.kt +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/impl/KotlinLibraryWriterImpl.kt @@ -79,7 +79,7 @@ class KoltinLibraryWriterImpl( val layout: KotlinLibraryLayoutForWriter = KotlinLibraryLayoutForWriter(libDir), - base: BaseWriter = BaseWriterImpl(layout, moduleName, versions, nopack), + val base: BaseWriter = BaseWriterImpl(layout, moduleName, versions, nopack), metadata: MetadataWriter = MetadataWriterImpl(layout), ir: IrWriter = IrMonoliticWriterImpl(layout) // ir: IrWriter = IrPerFileWriterImpl(layout) diff --git a/konan/commonizer/build.gradle.kts b/konan/commonizer/build.gradle.kts index d64f18700de..b8c3907ffb6 100644 --- a/konan/commonizer/build.gradle.kts +++ b/konan/commonizer/build.gradle.kts @@ -1,14 +1,28 @@ +import org.gradle.api.artifacts.maven.Conf2ScopeMappingContainer.COMPILE + plugins { + maven kotlin("jvm") id("jps-compatible") } +val mavenCompileScope by configurations.creating { + the() + .conf2ScopeMappings + .addMapping(0, this, COMPILE) +} + +description = "Kotlin/Native library commonizer" + dependencies { - compileOnly(project(":core:util.runtime")) - compileOnly(project(":core:descriptors")) + compileOnly(project(":compiler:frontend")) { isTransitive = true } + + // This dependency is necessary to keep the right dependency record inside of POM file: + mavenCompileScope(project(":kotlin-compiler")) compile(kotlinStdlib()) + compile(project(":kotlin-util-klib-metadata")) compile(project(":kotlin-native:kotlin-native-utils")) { isTransitive = false } compile(project(":kotlin-native:kotlin-native-library-reader")) { isTransitive = false } @@ -44,8 +58,16 @@ dependencies { } } +val runCommonizer by tasks.registering(NoDebugJavaExec::class) { + classpath(sourceSets.main.get().runtimeClasspath) + main = "org.jetbrains.kotlin.descriptors.commonizer.CliKt" +} + sourceSets { - "main" { projectDefault() } + "main" { + projectDefault() + runtimeClasspath += configurations.compileOnly + } "test" { projectDefault() } } @@ -53,3 +75,7 @@ projectTest(parallel = true) { dependsOn(":dist") workingDir = rootDir } + +publish() + +standardPublicJars() diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/DeclarationsBuilderVisitor1.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/DeclarationsBuilderVisitor1.kt index 9df6b8300ce..ba6ac834cb1 100644 --- a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/DeclarationsBuilderVisitor1.kt +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/DeclarationsBuilderVisitor1.kt @@ -41,12 +41,6 @@ internal class DeclarationsBuilderVisitor1( } } - // set cross-module dependencies: - modulesByTargets.values.forEach { modulesSameTarget -> - for (module in modulesSameTarget) - module.setDependencies(modulesSameTarget) - } - // return result (preserving order of targets): allTargets.forEachIndexed { index, target -> components.cache.cache(index, modulesByTargets.getValue(target)) diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt index d83b8b29d1f..115e1bee2d9 100644 --- a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt @@ -7,7 +7,6 @@ package org.jetbrains.kotlin.descriptors.commonizer.builder import org.jetbrains.kotlin.builtins.KotlinBuiltIns import org.jetbrains.kotlin.descriptors.* -import org.jetbrains.kotlin.descriptors.commonizer.* import org.jetbrains.kotlin.descriptors.commonizer.CommonizedGroup import org.jetbrains.kotlin.descriptors.commonizer.CommonizedGroupMap import org.jetbrains.kotlin.descriptors.commonizer.Target @@ -16,6 +15,7 @@ import org.jetbrains.kotlin.descriptors.commonizer.isUnderKotlinNativeSyntheticP import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ir.CirRootNode import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ir.dimension import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ir.indexOfCommon +import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl import org.jetbrains.kotlin.incremental.components.NoLookupLocation import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name @@ -29,12 +29,13 @@ class DeclarationsBuilderCache(dimension: Int) { check(dimension > 0) } - private val modules = CommonizedGroup>(dimension) + private val modules = CommonizedGroup>(dimension) private val packageFragments = CommonizedGroupMap, CommonizedPackageFragmentDescriptor>(dimension) private val classes = CommonizedGroupMap(dimension) private val typeAliases = CommonizedGroupMap(dimension) - private val forwardDeclarationsModules = CommonizedGroup(dimension) + private val forwardDeclarationsModules = CommonizedGroup(dimension) + private val allModulesWithDependencies = CommonizedGroup>(dimension) fun getCachedPackageFragments(moduleName: Name, packageFqName: FqName): List = packageFragments.getOrFail(moduleName to packageFqName) @@ -44,7 +45,7 @@ class DeclarationsBuilderCache(dimension: Int) { fun getCachedClassifier(fqName: FqName, index: Int): ClassifierDescriptorWithTypeParameters? = classes.getOrNull(fqName)?.get(index) ?: typeAliases.getOrNull(fqName)?.get(index) - fun cache(index: Int, modules: Collection) { + fun cache(index: Int, modules: List) { this.modules[index] = modules } @@ -60,7 +61,7 @@ class DeclarationsBuilderCache(dimension: Int) { typeAliases[fqName][index] = descriptor } - fun computeIfAbsentForwardDeclarationsModule(index: Int, computable: () -> ModuleDescriptor): ModuleDescriptor { + fun computeIfAbsentForwardDeclarationsModule(index: Int, computable: () -> ModuleDescriptorImpl): ModuleDescriptorImpl { forwardDeclarationsModules[index]?.let { return it } val module = computable() @@ -68,16 +69,24 @@ class DeclarationsBuilderCache(dimension: Int) { return module } - fun getAllModules(index: Int): Collection { + fun getAllModules(index: Int): List { + allModulesWithDependencies[index]?.let { return it } + val modulesForTarget = modules[index] ?: error("No module descriptors found for index $index") val forwardDeclarationsModule = forwardDeclarationsModules[index] // forward declarations module is created on demand (and only when commonizing Kotlin/Native target) // so, don't return it if it's not necessary - return if (forwardDeclarationsModule != null) + val allModules = if (forwardDeclarationsModule != null) modulesForTarget + forwardDeclarationsModule else modulesForTarget + + // set dependencies for target modules and cache them + modulesForTarget.forEach { it.setDependencies(allModules) } + allModulesWithDependencies[index] = allModules + + return allModules } companion object { diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli.kt new file mode 100644 index 00000000000..71b60770da3 --- /dev/null +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli.kt @@ -0,0 +1,283 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer + +import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl +import org.jetbrains.kotlin.descriptors.commonizer.ModuleForCommonization.DeserializedModule +import org.jetbrains.kotlin.descriptors.commonizer.ModuleForCommonization.SyntheticModule +import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl +import org.jetbrains.kotlin.konan.library.* +import org.jetbrains.kotlin.konan.library.KonanFactories.DefaultDeserializedDescriptorFactory +import org.jetbrains.kotlin.konan.properties.propertyList +import org.jetbrains.kotlin.konan.target.HostManager +import org.jetbrains.kotlin.konan.target.KonanTarget +import org.jetbrains.kotlin.library.* +import org.jetbrains.kotlin.library.impl.BaseWriterImpl +import org.jetbrains.kotlin.library.impl.KoltinLibraryWriterImpl +import org.jetbrains.kotlin.library.impl.createKotlinLibrary +import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.serialization.konan.impl.KlibResolvedModuleDescriptorsFactoryImpl.Companion.FORWARD_DECLARATIONS_MODULE_NAME +import org.jetbrains.kotlin.storage.LockBasedStorageManager +import java.io.File +import kotlin.system.exitProcess +import org.jetbrains.kotlin.konan.file.File as KFile + +fun main(args: Array) { + if (args.isEmpty()) printUsageAndExit() + + val parsedArgs = parseArgs(args) + + val repository = parsedArgs["-repository"]?.firstOrNull()?.let(::File) ?: printUsageAndExit("repository not specified") + if (!repository.isDirectory) printErrorAndExit("repository does not exist: $repository") + + val targets = with(HostManager()) { + val targetNames = parsedArgs["-target"]?.toSet() ?: printUsageAndExit("no targets specified") + targetNames.map { targetName -> + targets[targetName] ?: printUsageAndExit("unknown target name: $targetName") + } + } + + val destination = parsedArgs["-output"]?.firstOrNull()?.let(::File) ?: printUsageAndExit("output not specified") + when { + !destination.exists() -> destination.mkdirs() + !destination.isDirectory -> printErrorAndExit("output already exists: $destination") + destination.walkTopDown().any { it != destination } -> printErrorAndExit("output is not empty: $destination") + } + + val modulesByTargets = loadModules(repository, targets) + val result = commonize(modulesByTargets) + saveModules(modulesByTargets, destination, result) + + println("Done.") + println() +} + +private fun parseArgs(args: Array): Map> { + val commandLine = mutableMapOf>() + for (index in args.indices step 2) { + val key = args[index] + if (key[0] != '-') printUsageAndExit("Expected a flag with initial dash: $key") + if (index + 1 == args.size) printUsageAndExit("Expected a value after $key") + val value = args[index + 1] + commandLine.computeIfAbsent(key) { mutableListOf() }.add(value) + } + return commandLine +} + +private fun printErrorAndExit(errorMessage: String): Nothing { + println("Error: $errorMessage") + println() + + exitProcess(1) +} + +private fun printUsageAndExit(errorMessage: String? = null): Nothing { + if (errorMessage != null) { + println("Error: $errorMessage") + println() + } + + println("Usage: commonizer ") + println("where possible options include:") + println("\t-repository \tWork with the specified Kotlin/Native repository") + println("\t-target \t\tAdd hardware target to commonization") + println("\t-output \t\tDestination of commonized KLIBs") + println() + + exitProcess(if (errorMessage != null) 1 else 0) +} + +private fun loadModules(repository: File, targets: List): Map> { + val stdlibPath = repository.resolve(konanCommonLibraryPath(KONAN_STDLIB_NAME)) + val stdlib = loadLibrary(stdlibPath) + + val librariesByTargets = targets.map { target -> + val platformLibsPath = repository.resolve(KONAN_DISTRIBUTION_KLIB_DIR) + .resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR) + .resolve(target.name) + + val platformLibs = platformLibsPath.takeIf { it.isDirectory } + ?.listFiles() + ?.takeIf { it.isNotEmpty() } + ?.map { loadLibrary(it) } + ?: printErrorAndExit("no platform libraries found for target $target in $platformLibsPath") + + InputTarget(target.name, target) to platformLibs + }.toMap() + + return librariesByTargets.mapValues { (target, libraries) -> + val storageManager = LockBasedStorageManager("Target $target") + + val rawStdlibModule = DefaultDeserializedDescriptorFactory.createDescriptorAndNewBuiltIns( + library = stdlib, + languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT, + storageManager = storageManager, + packageAccessHandler = null + ) + + val otherModules = libraries.map { library -> + val rawModule = DefaultDeserializedDescriptorFactory.createDescriptorOptionalBuiltIns( + library = library, + languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT, + storageManager = storageManager, + builtIns = rawStdlibModule.builtIns, + packageAccessHandler = null + ) + val data = SensitiveManifestData.readFrom(library) + DeserializedModule(rawModule, data, File(library.libraryFile.path)) + } + + val rawForwardDeclarationsModule = createKotlinNativeForwardDeclarationsModule( + storageManager = storageManager, + builtIns = rawStdlibModule.builtIns + ) + + val onlyDeserializedModules = listOf(rawStdlibModule) + otherModules.map { it.module } + val allModules = onlyDeserializedModules + rawForwardDeclarationsModule + onlyDeserializedModules.forEach { it.setDependencies(allModules) } + + val stdlibModule = DeserializedModule(rawStdlibModule, SensitiveManifestData.readFrom(stdlib), File(stdlib.libraryFile.path)) + val forwardDeclarationsModule = SyntheticModule(rawForwardDeclarationsModule) + + listOf(stdlibModule) + otherModules + forwardDeclarationsModule + } +} + +private fun loadLibrary(location: File): KotlinLibrary { + if (!location.isDirectory) printErrorAndExit("library not found: $location") + return createKotlinLibrary(KFile(location.path)) +} + +private fun commonize(modulesByTargets: Map>): CommonizationPerformed { + val parameters = CommonizationParameters().apply { + modulesByTargets.forEach { (target, modules) -> + addTarget(target, modules.map { it.module }) + } + } + + val result = runCommonization(parameters) + return when (result) { + is NothingToCommonize -> printUsageAndExit("too few targets specified: ${modulesByTargets.keys}") + is CommonizationPerformed -> result + } +} + +private fun saveModules( + originalModulesByTargets: Map>, + destination: File, + result: CommonizationPerformed +) { + // optimization: stdlib effectively remains the same across all Kotlin/Native targets, + // so it can be just copied to the new destination without running serializer + val stdlibOrigin = originalModulesByTargets.values.asSequence() + .flatten() + .filterIsInstance() + .map { it.location } + .first { it.endsWith(KONAN_STDLIB_NAME) } + + val stdlibDestination = destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR).resolve(KONAN_STDLIB_NAME) + stdlibOrigin.copyRecursively(stdlibDestination) + + val originalModulesManifestData = originalModulesByTargets.mapValues { (_, modules) -> + modules.asSequence() + .filterIsInstance() + .filter { !it.location.endsWith(KONAN_STDLIB_NAME) } + .associate { it.module.name to it.data } + } + + val stdlibName = Name.special("<$KONAN_STDLIB_NAME>") + + result.concreteTargets.forEach { target -> + val konanTarget = target.konanTarget!! + val targetLibsDestination = destination.resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR).resolve(konanTarget.name) + + val newModulesManifestData = originalModulesManifestData.getValue(target) + val newModules = result.modulesByTargets.getValue(target) + + for (newModule in newModules) { + val libraryName = newModule.name + if (libraryName == stdlibName || libraryName == FORWARD_DECLARATIONS_MODULE_NAME) continue + + // TODO: obtain serialization data +// val metadata = TODO() + val manifestData = newModulesManifestData.getValue(newModule.name) + val libraryDestination = targetLibsDestination.resolve(libraryName.asString().trimStart('<').trimEnd('>')) + +// writeLibrary(metadata, manifestData, libraryDestination) + println("$manifestData - $libraryDestination") + } + } + + TODO("serialize common target") +} + +private sealed class ModuleForCommonization(val module: ModuleDescriptorImpl) { + class DeserializedModule( + module: ModuleDescriptorImpl, + val data: SensitiveManifestData, + val location: File + ) : ModuleForCommonization(module) + + class SyntheticModule(module: ModuleDescriptorImpl) : ModuleForCommonization(module) +} + +private data class SensitiveManifestData( + val uniqueName: String, + val versions: KotlinLibraryVersioning, + val dependencies: List, + val isInterop: Boolean, + val packageFqName: String?, + val exportForwardDeclarations: List +) { + fun applyTo(library: BaseWriterImpl) { + library.manifestProperties[KLIB_PROPERTY_UNIQUE_NAME] = uniqueName + + // note: versions can't be added here + + if (dependencies.isNotEmpty()) + library.manifestProperties[KLIB_PROPERTY_DEPENDS] = dependencies.joinToString(separator = " ") + else + library.manifestProperties.remove(KLIB_PROPERTY_DEPENDS) + + if (isInterop) + library.manifestProperties[KLIB_PROPERTY_INTEROP] = "true" + else + library.manifestProperties.remove(KLIB_PROPERTY_INTEROP) + + if (packageFqName != null) + library.manifestProperties[KLIB_PROPERTY_PACKAGE] = packageFqName + else + library.manifestProperties.remove(KLIB_PROPERTY_PACKAGE) + + if (exportForwardDeclarations.isNotEmpty()) + library.manifestProperties[KLIB_PROPERTY_EXPORT_FORWARD_DECLARATIONS] = + exportForwardDeclarations.joinToString(separator = " ") + else + library.manifestProperties.remove(KLIB_PROPERTY_EXPORT_FORWARD_DECLARATIONS) + } + + companion object { + fun readFrom(library: KotlinLibrary) = SensitiveManifestData( + uniqueName = library.uniqueName, + versions = library.versions, + dependencies = library.manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true), + isInterop = library.isInterop, + packageFqName = library.packageFqName, + exportForwardDeclarations = library.exportForwardDeclarations + ) + } +} + +private fun writeLibrary( + metadata: SerializedMetadata, + manifestData: SensitiveManifestData, + destination: File +) { + val library = KoltinLibraryWriterImpl(KFile(destination.path), manifestData.uniqueName, manifestData.versions, nopack = false) + library.addMetadata(metadata) + manifestData.applyTo(library.base as BaseWriterImpl) + library.commit() +} diff --git a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils.kt b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils.kt index c691658b83d..1e214398da3 100644 --- a/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils.kt +++ b/konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils.kt @@ -122,7 +122,7 @@ internal val ModuleDescriptor.packageFragmentProvider internal fun createKotlinNativeForwardDeclarationsModule( storageManager: StorageManager, builtIns: KotlinBuiltIns -) = +): ModuleDescriptorImpl = (createDefaultKonanResolvedModuleDescriptorsFactory(DefaultDeserializedDescriptorFactory) as KlibResolvedModuleDescriptorsFactoryImpl) .createForwardDeclarationsModule( builtIns = builtIns,