[Gradle] Implement support for hierarchical commonization

2021-03-26 15:53:45 +01:00
parent 68c3e39058
commit 8c941fc203
21 changed files with 269 additions and 59 deletions
@@ -31,7 +31,7 @@ internal class KotlinNativeCommonizerToolRunner(project: Project) : KotlinToolRu

    override val defaultMaxHeapSize: String get() = "4G"

-    override val mustRunViaExec get() = false // because it's not enough the standard Gradle wrapper's heap size
+    override val mustRunViaExec get() = true // because it's not enough the standard Gradle wrapper's heap size

    override fun getCustomJvmArgs() = PropertiesProvider(project).commonizerJvmArgs?.split("\\s+".toRegex()).orEmpty()
 }
@@ -224,6 +224,12 @@ internal class PropertiesProvider private constructor(private val project: Proje
    val enableCInteropCommonization: Boolean
        get() = booleanProperty("kotlin.mpp.enableCInteropCommonization") ?: false

+    /**
+     * Enables experimental commonization of 'higher level' shared native source sets
+     */
+    val enableHierarchicalCommonization: Boolean
+        get() = booleanProperty("kotlin.mpp.enableHierarchicalCommonization") ?: false
+
    /**
     * Dependencies caching strategy for all targets that support caches.
     */
@@ -27,9 +27,10 @@ internal abstract class AbstractCInteropCommonizerTask : DefaultTask() {
    internal abstract fun getCommonizationParameters(compilation: KotlinSharedNativeCompilation): CInteropCommonizationParameters?

    internal fun getLibraries(compilation: KotlinSharedNativeCompilation): FileCollection {
+        val compilationCommonizerTarget = project.getCommonizerTarget(compilation) ?: return project.files()
        val fileProvider = project.provider<Set<File>> {
            val parameters = getCommonizationParameters(compilation) ?: return@provider emptySet()
-            HierarchicalCommonizerOutputLayout.getTargetDirectory(outputDirectory(parameters), parameters.commonizerTarget)
+            HierarchicalCommonizerOutputLayout.getTargetDirectory(outputDirectory(parameters), compilationCommonizerTarget)
                .listFiles().orEmpty().toSet()
        }

@@ -24,7 +24,6 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.kotlinSourceSetsIncludingDefault
 import org.jetbrains.kotlin.gradle.plugin.sources.resolveAllDependsOnSourceSets
 import org.jetbrains.kotlin.gradle.targets.native.internal.CInteropCommonizerTask.CInteropGist
 import org.jetbrains.kotlin.gradle.tasks.CInteropProcess
-import org.jetbrains.kotlin.commonizer.util.transitiveClosure
 import org.jetbrains.kotlin.gradle.utils.fileProvider
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import java.io.File
@@ -53,11 +52,11 @@ internal open class CInteropCommonizerTask : AbstractCInteropCommonizerTask() {
     * All library files produced by the [Project.commonizeNativeDistributionTask] that are relevant for commonization
     */
    @get:Classpath
-    internal val nativeDistributionLibraries: Set<File>
+    internal val nonHierarchicalNativeDistributionLibraries: Set<File>
        get() {
            val commonizeNativeDistribution = project.commonizeNativeDistributionTask.get()
            return getCommonizationParameters().flatMapTo(mutableSetOf()) { parameters ->
-                parameters.commonizerTarget.withAllTransitiveTargets().flatMap { target ->
+                parameters.commonizerTarget.withAllAncestors().flatMap { target ->
                    commonizeNativeDistribution.commonizerTargetOutputDirectories.flatMap { outputDirectory ->
                        NativeDistributionCommonizerOutputLayout.getTargetDirectory(outputDirectory, target)
                            .listFiles().orEmpty().toList()
@@ -111,11 +110,24 @@ internal open class CInteropCommonizerTask : AbstractCInteropCommonizerTask() {
            konanHome = project.file(project.konanHome),
            outputCommonizerTarget = parameters.commonizerTarget,
            inputLibraries = cinteropsForTarget.map { it.libraryFile.get() }.toSet(),
-            dependencyLibraries = cinteropsForTarget.flatMap { it.dependencies.files }.toSet() + nativeDistributionLibraries,
+            dependencyLibraries = cinteropsForTarget.flatMap { it.dependencies.files }.toSet() + nativeDistributionLibraries(parameters),
            outputDirectory = outputDirectory(parameters)
        )
    }

+    private fun nativeDistributionLibraries(parameters: CInteropCommonizationParameters): Set<File> {
+        val task = project.commonizeNativeDistributionHierarchicalTask?.get() ?: return nonHierarchicalNativeDistributionLibraries
+
+        val rootTarget = task.rootCommonizerTargets
+            .firstOrNull { rootTarget -> parameters.commonizerTarget in rootTarget } ?: return emptySet()
+
+        val rootTargetOutput = task.getRootOutputDirectory(rootTarget)
+
+        return parameters.commonizerTarget.withAllAncestors().flatMap { target ->
+            HierarchicalCommonizerOutputLayout.getTargetDirectory(rootTargetOutput, target).listFiles().orEmpty().toList()
+        }.toSet()
+    }
+
    @Nested
    internal fun getCommonizationParameters(): Set<CInteropCommonizationParameters> {
        val sharedNativeCompilations = (project.multiplatformExtensionOrNull ?: return emptySet())
@@ -136,7 +148,7 @@ internal open class CInteropCommonizerTask : AbstractCInteropCommonizerTask() {
        return sharedNativeCompilations.mapNotNull(::getCommonizationParameters).toSet()
            .run(::removeNotRegisteredInterops)
            .run(::removeEmptyInterops)
-            .run(::removeHierarchicalParameters)
+            .run(if (project.isHierarchicalCommonizationEnabled) ::identity else ::removeHierarchicalParameters)
            .run(::removeRedundantParameters)
    }

@@ -193,6 +205,8 @@ private fun removeEmptyInterops(parameters: Set<CInteropCommonizationParameters>
    return parameters.filterTo(mutableSetOf()) { it.interops.isNotEmpty() }
 }

+private fun identity(parameters: Set<CInteropCommonizationParameters>) = parameters
+
 private fun removeHierarchicalParameters(parameters: Set<CInteropCommonizationParameters>): Set<CInteropCommonizationParameters> {
    return parameters.filterTo(mutableSetOf()) { it.commonizerTarget.level <= 1 }
 }
@@ -210,10 +224,6 @@ private operator fun CommonizerTarget.contains(other: CommonizerTarget): Boolean
    return this.isAncestorOf(other)
 }

-private fun SharedCommonizerTarget.withAllTransitiveTargets(): Set<CommonizerTarget> {
-    return setOf(this) + transitiveClosure<CommonizerTarget>(this) { if (this is SharedCommonizerTarget) this.targets else emptySet() }
-}
-
 private fun Project.getDependingNativeCompilations(compilation: KotlinSharedNativeCompilation): Set<KotlinNativeCompilation> {
    /**
     * Some implementations of [KotlinCompilation] do not contain the default source set in
@@ -16,6 +16,8 @@ import org.jetbrains.kotlin.gradle.tasks.registerTask

 internal val Project.isCInteropCommonizationEnabled: Boolean get() = PropertiesProvider(this).enableCInteropCommonization

+internal val Project.isHierarchicalCommonizationEnabled: Boolean get() = PropertiesProvider(this).enableHierarchicalCommonization
+
 internal val Project.commonizeTask: TaskProvider<Task>
    get() = locateOrRegisterTask(
        "commonize",
@@ -26,7 +28,6 @@ internal val Project.commonizeTask: TaskProvider<Task>
        }
    )

-
 /**
 * Keeping this task/task name for IDE compatibility which is invoking 'runCommonizer' during sync
 */
@@ -46,7 +47,10 @@ internal val Project.commonizeCInteropTask: TaskProvider<CInteropCommonizerTask>
        if (isCInteropCommonizationEnabled) {
            return locateOrRegisterTask(
                "commonizeCInterop",
-                invokeWhenRegistered = { commonizeTask.dependsOn(this); dependsOn(commonizeNativeDistributionTask) },
+                invokeWhenRegistered = {
+                    commonizeTask.dependsOn(this)
+                    commonizeNativeDistributionHierarchicalTask?.let(this::dependsOn) ?: dependsOn(commonizeNativeDistributionTask)
+                },
                configureTask = {
                    group = "interop"
                    description = "Invokes the commonizer on c-interop bindings of the project"
@@ -82,6 +86,19 @@ internal val Project.commonizeNativeDistributionTask: TaskProvider<NativeDistrib
        }
    )

+internal val Project.commonizeNativeDistributionHierarchicalTask: TaskProvider<HierarchicalNativeDistributionCommonizerTask>?
+    get() {
+        if (!isHierarchicalCommonizationEnabled) return null
+        return locateOrRegisterTask(
+            "commonizeNativeDistributionHierarchically",
+            invokeWhenRegistered = { commonizeTask.dependsOn(this) },
+            configureTask = {
+                group = "interop"
+                description = "Invokes the commonizer on platform libraries provided by the Kotlin/Native distribution"
+            }
+        )
+    }
+
 private inline fun <reified T : Task> Project.locateOrRegisterTask(
    name: String,
    args: List<Any> = emptyList(),
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.targets.native.internal
+
+import org.gradle.api.Project
+import org.gradle.api.file.ConfigurableFileCollection
+import org.gradle.api.file.FileCollection
+import org.jetbrains.kotlin.commonizer.HierarchicalCommonizerOutputLayout
+import org.jetbrains.kotlin.commonizer.KonanDistribution
+import org.jetbrains.kotlin.commonizer.isAncestorOf
+import org.jetbrains.kotlin.commonizer.stdlib
+import org.jetbrains.kotlin.compilerRunner.konanHome
+import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull
+import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
+import org.jetbrains.kotlin.gradle.targets.metadata.getMetadataCompilationForSourceSet
+import java.io.File
+
+internal fun Project.setUpHierarchicalKotlinNativePlatformDependencies() {
+    val task = commonizeNativeDistributionHierarchicalTask?.get() ?: return
+    val kotlin = multiplatformExtensionOrNull ?: return
+    kotlin.sourceSets.forEach { sourceSet ->
+        val target = getCommonizerTarget(sourceSet) ?: return@forEach
+
+        val rootTarget = task.rootCommonizerTargets
+            .firstOrNull { rootTarget -> rootTarget == target || rootTarget.isAncestorOf(target) }
+            ?: return@forEach
+
+        val rootOutputDirectory = task.getRootOutputDirectory(rootTarget)
+        val targetOutputDirectory = HierarchicalCommonizerOutputLayout.getTargetDirectory(rootOutputDirectory, target)
+
+        val dependencies = project.lazyFiles { targetOutputDirectory.listFiles().orEmpty().toList() }.builtBy(task)
+        val stdlib = project.lazyFiles { listOf(konanDistribution.stdlib) }
+        addDependencies(sourceSet, dependencies)
+        addDependencies(sourceSet, stdlib)
+    }
+}
+
+private fun Project.addDependencies(sourceSet: KotlinSourceSet, libraries: FileCollection) {
+    getMetadataCompilationForSourceSet(sourceSet)?.let { compilation ->
+        compilation.compileDependencyFiles += libraries
+    }
+    dependencies.add(sourceSet.implementationMetadataConfigurationName, libraries)
+}
+
+
+private fun Project.lazyFiles(provider: () -> Iterable<File>): ConfigurableFileCollection {
+    return project.files(project.provider { provider() })
+}
+
+private val Project.konanDistribution: KonanDistribution
+    get() = KonanDistribution(project.file(konanHome))
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.targets.native.internal
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.Project
+import org.gradle.api.tasks.*
+import org.jetbrains.kotlin.commonizer.SharedCommonizerTarget
+import org.jetbrains.kotlin.commonizer.identityString
+import org.jetbrains.kotlin.commonizer.isAncestorOf
+import org.jetbrains.kotlin.compilerRunner.KotlinNativeCommonizerToolRunner
+import org.jetbrains.kotlin.compilerRunner.konanHome
+import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull
+import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion
+import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMONIZED_LIBS_DIR
+import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMON_LIBS_DIR
+import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_KLIB_DIR
+import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR
+import java.io.File
+import java.nio.charset.StandardCharsets
+import java.util.*
+
+internal open class HierarchicalNativeDistributionCommonizerTask : DefaultTask() {
+
+    private val konanHome = project.file(project.konanHome)
+
+    @get:Input
+    internal val rootCommonizerTargets: Set<SharedCommonizerTarget>
+        get() = project.getRootCommonizerTargets()
+
+    @get:PathSensitive(PathSensitivity.ABSOLUTE)
+    @get:InputDirectory
+    @Suppress("unused") // Only for up-to-date checker. The directory with the original common libs.
+    val originalCommonLibrariesDirectory = konanHome
+        .resolve(KONAN_DISTRIBUTION_KLIB_DIR)
+        .resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR)
+
+    @get:PathSensitive(PathSensitivity.ABSOLUTE)
+    @get:InputDirectory
+    @Suppress("unused") // Only for up-to-date checker. The directory with the original platform libs.
+    val originalPlatformLibrariesDirectory = konanHome
+        .resolve(KONAN_DISTRIBUTION_KLIB_DIR)
+        .resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR)
+
+    @get:OutputDirectories
+    @Suppress("unused") // Only for up-to-date checker. The directory with the original platform libs.
+    val outputDirectories: Set<File>
+        get() = rootCommonizerTargets.map(::getRootOutputDirectory).toSet()
+
+    internal fun getRootOutputDirectory(target: SharedCommonizerTarget): File {
+        val kotlinVersion = checkNotNull(project.getKotlinPluginVersion()) { "Missing Kotlin Plugin version" }
+
+        val discriminator = buildString {
+            append(target.identityString)
+            append("-")
+            append(kotlinVersion.toLowerCase().base64)
+        }
+
+        return project.file(konanHome)
+            .resolve(KONAN_DISTRIBUTION_KLIB_DIR)
+            .resolve(KONAN_DISTRIBUTION_COMMONIZED_LIBS_DIR)
+            .resolve(discriminator)
+    }
+
+    @TaskAction
+    protected fun run() {
+        for (target in rootCommonizerTargets) {
+            getRootOutputDirectory(target).deleteRecursively()
+            KotlinNativeCommonizerToolRunner(project).run(getCommandLineArguments(target))
+        }
+    }
+
+    private fun getCommandLineArguments(target: SharedCommonizerTarget): List<String> {
+        return mutableListOf<String>().apply {
+            this += "native-dist-commonize"
+            this += "-distribution-path"
+            this += konanHome.absolutePath
+            this += "-output-path"
+            this += getRootOutputDirectory(target).absolutePath
+            this += "-output-commonizer-target"
+            this += target.identityString
+        }
+    }
+}
+
+private fun Project.getRootCommonizerTargets(): Set<SharedCommonizerTarget> {
+    val kotlin = multiplatformExtensionOrNull ?: return emptySet()
+    val allTargets = kotlin.sourceSets
+        .mapNotNull { sourceSet -> getCommonizerTarget(sourceSet) }
+        .filterIsInstance<SharedCommonizerTarget>()
+    return allTargets.filter { target -> allTargets.none { otherTarget -> otherTarget isAncestorOf target } }.toSet()
+}
+
+private val String.base64
+    get() = base64Encoder.encodeToString(toByteArray(StandardCharsets.UTF_8))
+
+private val base64Encoder = Base64.getEncoder().withoutPadding()
@@ -47,11 +47,13 @@ internal fun Project.setUpKotlinNativePlatformDependencies() {
        // not a multiplatform project, nothing to set up
        return
    }
-
    val kotlinVersion = getKotlinPluginVersion()!!
    val allowCommonizer = isAllowCommonizer(kotlinVersion)
-    val dependencyResolver = NativePlatformDependencyResolver(this, kotlinVersion)
+    if (allowCommonizer && isHierarchicalCommonizationEnabled) {
+        return setUpHierarchicalKotlinNativePlatformDependencies()
+    }

+    val dependencyResolver = NativePlatformDependencyResolver(this, kotlinVersion)
    findSourceSetsToAddDependencies(allowCommonizer).forEach { (sourceSet: KotlinSourceSet, sourceSetDeps: Set<NativePlatformDependency>) ->
        sourceSetDeps.forEach { sourceSetDep: NativePlatformDependency ->
            dependencyResolver.addForResolve(sourceSetDep) { resolvedFiles: FileCollection ->
@@ -188,7 +188,7 @@ internal fun Project.createTempNativeDistributionCommonizerOutputDirectory(targe
    ).toFile()
 }

-fun callCommonizerCLI(project: Project, commandLineArguments: List<String>) {
+internal fun callCommonizerCLI(project: Project, commandLineArguments: List<String>) {
    if (commandLineArguments.isEmpty()) return

    KotlinNativeCommonizerToolRunner(project).run(commandLineArguments)