[Commonizer] CLI for Kotlin/Native KLIBs

2019-10-09 13:43:58 +07:00
parent 5bc2a4e1e8
commit ce376f49fd
7 changed files with 350 additions and 33 deletions
@@ -71,26 +71,27 @@ class KlibResolvedModuleDescriptorsFactoryImpl(
    }

    fun createForwardDeclarationsModule(
-            builtIns: KotlinBuiltIns?,
-            storageManager: StorageManager): ModuleDescriptorImpl {
+        builtIns: KotlinBuiltIns?,
+        storageManager: StorageManager
+    ): ModuleDescriptorImpl {

-        val name = Name.special("<forward declarations>")
-        val module = createDescriptorOptionalBuiltsIns(name, storageManager, builtIns, SyntheticModulesOrigin)
+        val module = createDescriptorOptionalBuiltsIns(FORWARD_DECLARATIONS_MODULE_NAME, storageManager, builtIns, SyntheticModulesOrigin)

        fun createPackage(fqName: FqName, supertypeName: String, classKind: ClassKind) =
-                ForwardDeclarationsPackageFragmentDescriptor(
-                        storageManager,
-                        module,
-                        fqName,
-                        Name.identifier(supertypeName),
-                        classKind)
+            ForwardDeclarationsPackageFragmentDescriptor(
+                storageManager,
+                module,
+                fqName,
+                Name.identifier(supertypeName),
+                classKind
+            )

        val packageFragmentProvider = PackageFragmentProviderImpl(
-                listOf(
-                        createPackage(ForwardDeclarationsFqNames.cNamesStructs, "COpaque", ClassKind.CLASS),
-                        createPackage(ForwardDeclarationsFqNames.objCNamesClasses, "ObjCObjectBase", ClassKind.CLASS),
-                        createPackage(ForwardDeclarationsFqNames.objCNamesProtocols, "ObjCObject", ClassKind.INTERFACE)
-                )
+            listOf(
+                createPackage(ForwardDeclarationsFqNames.cNamesStructs, "COpaque", ClassKind.CLASS),
+                createPackage(ForwardDeclarationsFqNames.objCNamesClasses, "ObjCObjectBase", ClassKind.CLASS),
+                createPackage(ForwardDeclarationsFqNames.objCNamesProtocols, "ObjCObject", ClassKind.INTERFACE)
+            )
        )

        module.initialize(packageFragmentProvider)
@@ -119,6 +120,10 @@ class KlibResolvedModuleDescriptorsFactoryImpl(
        moduleDescriptorFactory.createDescriptor(library, languageVersionSettings, storageManager, builtIns, packageAccessHandler)
    else
        moduleDescriptorFactory.createDescriptorAndNewBuiltIns(library, languageVersionSettings, storageManager, packageAccessHandler)
+
+    companion object {
+        val FORWARD_DECLARATIONS_MODULE_NAME = Name.special("<forward declarations>")
+    }
 }

 /**
@@ -79,7 +79,7 @@ class KoltinLibraryWriterImpl(

    val layout: KotlinLibraryLayoutForWriter = KotlinLibraryLayoutForWriter(libDir),

-    base: BaseWriter = BaseWriterImpl(layout, moduleName, versions, nopack),
+    val base: BaseWriter = BaseWriterImpl(layout, moduleName, versions, nopack),
    metadata: MetadataWriter = MetadataWriterImpl(layout),
    ir: IrWriter = IrMonoliticWriterImpl(layout)
 //    ir: IrWriter = IrPerFileWriterImpl(layout)
@@ -1,14 +1,28 @@
+import org.gradle.api.artifacts.maven.Conf2ScopeMappingContainer.COMPILE
+
 plugins {
+    maven
    kotlin("jvm")
    id("jps-compatible")
 }

+val mavenCompileScope by configurations.creating {
+    the<MavenPluginConvention>()
+        .conf2ScopeMappings
+        .addMapping(0, this, COMPILE)
+}
+
+description = "Kotlin/Native library commonizer"
+
 dependencies {
-    compileOnly(project(":core:util.runtime"))
-    compileOnly(project(":core:descriptors"))
+    compileOnly(project(":compiler:frontend")) { isTransitive = true }
+
+    // This dependency is necessary to keep the right dependency record inside of POM file:
+    mavenCompileScope(project(":kotlin-compiler"))

    compile(kotlinStdlib())

+    compile(project(":kotlin-util-klib-metadata"))
    compile(project(":kotlin-native:kotlin-native-utils")) { isTransitive = false }
    compile(project(":kotlin-native:kotlin-native-library-reader")) { isTransitive = false }

@@ -44,8 +58,16 @@ dependencies {
    }
 }

+val runCommonizer by tasks.registering(NoDebugJavaExec::class) {
+    classpath(sourceSets.main.get().runtimeClasspath)
+    main = "org.jetbrains.kotlin.descriptors.commonizer.CliKt"
+}
+
 sourceSets {
-    "main" { projectDefault() }
+    "main" {
+        projectDefault()
+        runtimeClasspath += configurations.compileOnly
+    }
    "test" { projectDefault() }
 }

@@ -53,3 +75,7 @@ projectTest(parallel = true) {
    dependsOn(":dist")
    workingDir = rootDir
 }
+
+publish()
+
+standardPublicJars()
@@ -41,12 +41,6 @@ internal class DeclarationsBuilderVisitor1(
            }
        }

-        // set cross-module dependencies:
-        modulesByTargets.values.forEach { modulesSameTarget ->
-            for (module in modulesSameTarget)
-                module.setDependencies(modulesSameTarget)
-        }
-
        // return result (preserving order of targets):
        allTargets.forEachIndexed { index, target ->
            components.cache.cache(index, modulesByTargets.getValue(target))
@@ -7,7 +7,6 @@ package org.jetbrains.kotlin.descriptors.commonizer.builder

 import org.jetbrains.kotlin.builtins.KotlinBuiltIns
 import org.jetbrains.kotlin.descriptors.*
-import org.jetbrains.kotlin.descriptors.commonizer.*
 import org.jetbrains.kotlin.descriptors.commonizer.CommonizedGroup
 import org.jetbrains.kotlin.descriptors.commonizer.CommonizedGroupMap
 import org.jetbrains.kotlin.descriptors.commonizer.Target
@@ -16,6 +15,7 @@ import org.jetbrains.kotlin.descriptors.commonizer.isUnderKotlinNativeSyntheticP
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ir.CirRootNode
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ir.dimension
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ir.indexOfCommon
+import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.incremental.components.NoLookupLocation
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
@@ -29,12 +29,13 @@ class DeclarationsBuilderCache(dimension: Int) {
        check(dimension > 0)
    }

-    private val modules = CommonizedGroup<Collection<ModuleDescriptor>>(dimension)
+    private val modules = CommonizedGroup<List<ModuleDescriptorImpl>>(dimension)
    private val packageFragments = CommonizedGroupMap<Pair<Name, FqName>, CommonizedPackageFragmentDescriptor>(dimension)
    private val classes = CommonizedGroupMap<FqName, CommonizedClassDescriptor>(dimension)
    private val typeAliases = CommonizedGroupMap<FqName, CommonizedTypeAliasDescriptor>(dimension)

-    private val forwardDeclarationsModules = CommonizedGroup<ModuleDescriptor>(dimension)
+    private val forwardDeclarationsModules = CommonizedGroup<ModuleDescriptorImpl>(dimension)
+    private val allModulesWithDependencies = CommonizedGroup<List<ModuleDescriptor>>(dimension)

    fun getCachedPackageFragments(moduleName: Name, packageFqName: FqName): List<CommonizedPackageFragmentDescriptor?> =
        packageFragments.getOrFail(moduleName to packageFqName)
@@ -44,7 +45,7 @@ class DeclarationsBuilderCache(dimension: Int) {
    fun getCachedClassifier(fqName: FqName, index: Int): ClassifierDescriptorWithTypeParameters? =
        classes.getOrNull(fqName)?.get(index) ?: typeAliases.getOrNull(fqName)?.get(index)

-    fun cache(index: Int, modules: Collection<ModuleDescriptor>) {
+    fun cache(index: Int, modules: List<ModuleDescriptorImpl>) {
        this.modules[index] = modules
    }

@@ -60,7 +61,7 @@ class DeclarationsBuilderCache(dimension: Int) {
        typeAliases[fqName][index] = descriptor
    }

-    fun computeIfAbsentForwardDeclarationsModule(index: Int, computable: () -> ModuleDescriptor): ModuleDescriptor {
+    fun computeIfAbsentForwardDeclarationsModule(index: Int, computable: () -> ModuleDescriptorImpl): ModuleDescriptorImpl {
        forwardDeclarationsModules[index]?.let { return it }

        val module = computable()
@@ -68,16 +69,24 @@ class DeclarationsBuilderCache(dimension: Int) {
        return module
    }

-    fun getAllModules(index: Int): Collection<ModuleDescriptor> {
+    fun getAllModules(index: Int): List<ModuleDescriptor> {
+        allModulesWithDependencies[index]?.let { return it }
+
        val modulesForTarget = modules[index] ?: error("No module descriptors found for index $index")
        val forwardDeclarationsModule = forwardDeclarationsModules[index]

        // forward declarations module is created on demand (and only when commonizing Kotlin/Native target)
        // so, don't return it if it's not necessary
-        return if (forwardDeclarationsModule != null)
+        val allModules = if (forwardDeclarationsModule != null)
            modulesForTarget + forwardDeclarationsModule
        else
            modulesForTarget
+
+        // set dependencies for target modules and cache them
+        modulesForTarget.forEach { it.setDependencies(allModules) }
+        allModulesWithDependencies[index] = allModules
+
+        return allModules
    }

    companion object {
@@ -0,0 +1,283 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.descriptors.commonizer
+
+import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl
+import org.jetbrains.kotlin.descriptors.commonizer.ModuleForCommonization.DeserializedModule
+import org.jetbrains.kotlin.descriptors.commonizer.ModuleForCommonization.SyntheticModule
+import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
+import org.jetbrains.kotlin.konan.library.*
+import org.jetbrains.kotlin.konan.library.KonanFactories.DefaultDeserializedDescriptorFactory
+import org.jetbrains.kotlin.konan.properties.propertyList
+import org.jetbrains.kotlin.konan.target.HostManager
+import org.jetbrains.kotlin.konan.target.KonanTarget
+import org.jetbrains.kotlin.library.*
+import org.jetbrains.kotlin.library.impl.BaseWriterImpl
+import org.jetbrains.kotlin.library.impl.KoltinLibraryWriterImpl
+import org.jetbrains.kotlin.library.impl.createKotlinLibrary
+import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.serialization.konan.impl.KlibResolvedModuleDescriptorsFactoryImpl.Companion.FORWARD_DECLARATIONS_MODULE_NAME
+import org.jetbrains.kotlin.storage.LockBasedStorageManager
+import java.io.File
+import kotlin.system.exitProcess
+import org.jetbrains.kotlin.konan.file.File as KFile
+
+fun main(args: Array<String>) {
+    if (args.isEmpty()) printUsageAndExit()
+
+    val parsedArgs = parseArgs(args)
+
+    val repository = parsedArgs["-repository"]?.firstOrNull()?.let(::File) ?: printUsageAndExit("repository not specified")
+    if (!repository.isDirectory) printErrorAndExit("repository does not exist: $repository")
+
+    val targets = with(HostManager()) {
+        val targetNames = parsedArgs["-target"]?.toSet() ?: printUsageAndExit("no targets specified")
+        targetNames.map { targetName ->
+            targets[targetName] ?: printUsageAndExit("unknown target name: $targetName")
+        }
+    }
+
+    val destination = parsedArgs["-output"]?.firstOrNull()?.let(::File) ?: printUsageAndExit("output not specified")
+    when {
+        !destination.exists() -> destination.mkdirs()
+        !destination.isDirectory -> printErrorAndExit("output already exists: $destination")
+        destination.walkTopDown().any { it != destination } -> printErrorAndExit("output is not empty: $destination")
+    }
+
+    val modulesByTargets = loadModules(repository, targets)
+    val result = commonize(modulesByTargets)
+    saveModules(modulesByTargets, destination, result)
+
+    println("Done.")
+    println()
+}
+
+private fun parseArgs(args: Array<String>): Map<String, List<String>> {
+    val commandLine = mutableMapOf<String, MutableList<String>>()
+    for (index in args.indices step 2) {
+        val key = args[index]
+        if (key[0] != '-') printUsageAndExit("Expected a flag with initial dash: $key")
+        if (index + 1 == args.size) printUsageAndExit("Expected a value after $key")
+        val value = args[index + 1]
+        commandLine.computeIfAbsent(key) { mutableListOf() }.add(value)
+    }
+    return commandLine
+}
+
+private fun printErrorAndExit(errorMessage: String): Nothing {
+    println("Error: $errorMessage")
+    println()
+
+    exitProcess(1)
+}
+
+private fun printUsageAndExit(errorMessage: String? = null): Nothing {
+    if (errorMessage != null) {
+        println("Error: $errorMessage")
+        println()
+    }
+
+    println("Usage: commonizer <options>")
+    println("where possible options include:")
+    println("\t-repository <path>\tWork with the specified Kotlin/Native repository")
+    println("\t-target <name>\t\tAdd hardware target to commonization")
+    println("\t-output <path>\t\tDestination of commonized KLIBs")
+    println()
+
+    exitProcess(if (errorMessage != null) 1 else 0)
+}
+
+private fun loadModules(repository: File, targets: List<KonanTarget>): Map<InputTarget, List<ModuleForCommonization>> {
+    val stdlibPath = repository.resolve(konanCommonLibraryPath(KONAN_STDLIB_NAME))
+    val stdlib = loadLibrary(stdlibPath)
+
+    val librariesByTargets = targets.map { target ->
+        val platformLibsPath = repository.resolve(KONAN_DISTRIBUTION_KLIB_DIR)
+            .resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR)
+            .resolve(target.name)
+
+        val platformLibs = platformLibsPath.takeIf { it.isDirectory }
+            ?.listFiles()
+            ?.takeIf { it.isNotEmpty() }
+            ?.map { loadLibrary(it) }
+            ?: printErrorAndExit("no platform libraries found for target $target in $platformLibsPath")
+
+        InputTarget(target.name, target) to platformLibs
+    }.toMap()
+
+    return librariesByTargets.mapValues { (target, libraries) ->
+        val storageManager = LockBasedStorageManager("Target $target")
+
+        val rawStdlibModule = DefaultDeserializedDescriptorFactory.createDescriptorAndNewBuiltIns(
+            library = stdlib,
+            languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
+            storageManager = storageManager,
+            packageAccessHandler = null
+        )
+
+        val otherModules = libraries.map { library ->
+            val rawModule = DefaultDeserializedDescriptorFactory.createDescriptorOptionalBuiltIns(
+                library = library,
+                languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
+                storageManager = storageManager,
+                builtIns = rawStdlibModule.builtIns,
+                packageAccessHandler = null
+            )
+            val data = SensitiveManifestData.readFrom(library)
+            DeserializedModule(rawModule, data, File(library.libraryFile.path))
+        }
+
+        val rawForwardDeclarationsModule = createKotlinNativeForwardDeclarationsModule(
+            storageManager = storageManager,
+            builtIns = rawStdlibModule.builtIns
+        )
+
+        val onlyDeserializedModules = listOf(rawStdlibModule) + otherModules.map { it.module }
+        val allModules = onlyDeserializedModules + rawForwardDeclarationsModule
+        onlyDeserializedModules.forEach { it.setDependencies(allModules) }
+
+        val stdlibModule = DeserializedModule(rawStdlibModule, SensitiveManifestData.readFrom(stdlib), File(stdlib.libraryFile.path))
+        val forwardDeclarationsModule = SyntheticModule(rawForwardDeclarationsModule)
+
+        listOf(stdlibModule) + otherModules + forwardDeclarationsModule
+    }
+}
+
+private fun loadLibrary(location: File): KotlinLibrary {
+    if (!location.isDirectory) printErrorAndExit("library not found: $location")
+    return createKotlinLibrary(KFile(location.path))
+}
+
+private fun commonize(modulesByTargets: Map<InputTarget, List<ModuleForCommonization>>): CommonizationPerformed {
+    val parameters = CommonizationParameters().apply {
+        modulesByTargets.forEach { (target, modules) ->
+            addTarget(target, modules.map { it.module })
+        }
+    }
+
+    val result = runCommonization(parameters)
+    return when (result) {
+        is NothingToCommonize -> printUsageAndExit("too few targets specified: ${modulesByTargets.keys}")
+        is CommonizationPerformed -> result
+    }
+}
+
+private fun saveModules(
+    originalModulesByTargets: Map<InputTarget, List<ModuleForCommonization>>,
+    destination: File,
+    result: CommonizationPerformed
+) {
+    // optimization: stdlib effectively remains the same across all Kotlin/Native targets,
+    // so it can be just copied to the new destination without running serializer
+    val stdlibOrigin = originalModulesByTargets.values.asSequence()
+        .flatten()
+        .filterIsInstance<DeserializedModule>()
+        .map { it.location }
+        .first { it.endsWith(KONAN_STDLIB_NAME) }
+
+    val stdlibDestination = destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR).resolve(KONAN_STDLIB_NAME)
+    stdlibOrigin.copyRecursively(stdlibDestination)
+
+    val originalModulesManifestData = originalModulesByTargets.mapValues { (_, modules) ->
+        modules.asSequence()
+            .filterIsInstance<DeserializedModule>()
+            .filter { !it.location.endsWith(KONAN_STDLIB_NAME) }
+            .associate { it.module.name to it.data }
+    }
+
+    val stdlibName = Name.special("<$KONAN_STDLIB_NAME>")
+
+    result.concreteTargets.forEach { target ->
+        val konanTarget = target.konanTarget!!
+        val targetLibsDestination = destination.resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR).resolve(konanTarget.name)
+
+        val newModulesManifestData = originalModulesManifestData.getValue(target)
+        val newModules = result.modulesByTargets.getValue(target)
+
+        for (newModule in newModules) {
+            val libraryName = newModule.name
+            if (libraryName == stdlibName || libraryName == FORWARD_DECLARATIONS_MODULE_NAME) continue
+
+            // TODO: obtain serialization data
+//            val metadata = TODO()
+            val manifestData = newModulesManifestData.getValue(newModule.name)
+            val libraryDestination = targetLibsDestination.resolve(libraryName.asString().trimStart('<').trimEnd('>'))
+
+//            writeLibrary(metadata, manifestData, libraryDestination)
+            println("$manifestData - $libraryDestination")
+        }
+    }
+
+    TODO("serialize common target")
+}
+
+private sealed class ModuleForCommonization(val module: ModuleDescriptorImpl) {
+    class DeserializedModule(
+        module: ModuleDescriptorImpl,
+        val data: SensitiveManifestData,
+        val location: File
+    ) : ModuleForCommonization(module)
+
+    class SyntheticModule(module: ModuleDescriptorImpl) : ModuleForCommonization(module)
+}
+
+private data class SensitiveManifestData(
+    val uniqueName: String,
+    val versions: KotlinLibraryVersioning,
+    val dependencies: List<String>,
+    val isInterop: Boolean,
+    val packageFqName: String?,
+    val exportForwardDeclarations: List<String>
+) {
+    fun applyTo(library: BaseWriterImpl) {
+        library.manifestProperties[KLIB_PROPERTY_UNIQUE_NAME] = uniqueName
+
+        // note: versions can't be added here
+
+        if (dependencies.isNotEmpty())
+            library.manifestProperties[KLIB_PROPERTY_DEPENDS] = dependencies.joinToString(separator = " ")
+        else
+            library.manifestProperties.remove(KLIB_PROPERTY_DEPENDS)
+
+        if (isInterop)
+            library.manifestProperties[KLIB_PROPERTY_INTEROP] = "true"
+        else
+            library.manifestProperties.remove(KLIB_PROPERTY_INTEROP)
+
+        if (packageFqName != null)
+            library.manifestProperties[KLIB_PROPERTY_PACKAGE] = packageFqName
+        else
+            library.manifestProperties.remove(KLIB_PROPERTY_PACKAGE)
+
+        if (exportForwardDeclarations.isNotEmpty())
+            library.manifestProperties[KLIB_PROPERTY_EXPORT_FORWARD_DECLARATIONS] =
+                exportForwardDeclarations.joinToString(separator = " ")
+        else
+            library.manifestProperties.remove(KLIB_PROPERTY_EXPORT_FORWARD_DECLARATIONS)
+    }
+
+    companion object {
+        fun readFrom(library: KotlinLibrary) = SensitiveManifestData(
+            uniqueName = library.uniqueName,
+            versions = library.versions,
+            dependencies = library.manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true),
+            isInterop = library.isInterop,
+            packageFqName = library.packageFqName,
+            exportForwardDeclarations = library.exportForwardDeclarations
+        )
+    }
+}
+
+private fun writeLibrary(
+    metadata: SerializedMetadata,
+    manifestData: SensitiveManifestData,
+    destination: File
+) {
+    val library = KoltinLibraryWriterImpl(KFile(destination.path), manifestData.uniqueName, manifestData.versions, nopack = false)
+    library.addMetadata(metadata)
+    manifestData.applyTo(library.base as BaseWriterImpl)
+    library.commit()
+}
@@ -122,7 +122,7 @@ internal val ModuleDescriptor.packageFragmentProvider
 internal fun createKotlinNativeForwardDeclarationsModule(
    storageManager: StorageManager,
    builtIns: KotlinBuiltIns
-) =
+): ModuleDescriptorImpl =
    (createDefaultKonanResolvedModuleDescriptorsFactory(DefaultDeserializedDescriptorFactory) as KlibResolvedModuleDescriptorsFactoryImpl)
        .createForwardDeclarationsModule(
            builtIns = builtIns,