From 8c941fc203881b58a3d792ffe0f21f8c4d6d814f Mon Sep 17 00:00:00 2001 From: "sebastian.sellmair" Date: Fri, 26 Mar 2021 15:53:45 +0100 Subject: [PATCH] [Gradle] Implement support for hierarchical commonization --- .../KotlinNativeCommonizerToolRunner.kt | 2 +- .../kotlin/gradle/plugin/KotlinProperties.kt | 6 ++ .../AbstractCInteropCommonizerTask.kt | 3 +- .../native/internal/CInteropCommonizerTask.kt | 28 +++-- .../native/internal/CommonizerTasks.kt | 21 +++- ...rchicalKotlinNativePlatformDependencies.kt | 54 ++++++++++ ...rchicalNativeDistributionCommonizerTask.kt | 100 ++++++++++++++++++ .../KotlinNativePlatformDependencies.kt | 6 +- .../NativeDistributionCommonizerTask.kt | 2 +- .../kotlin/commonizer/KonanDistribution.kt | 10 +- .../kotlin/commonizer/CommonizerParameters.kt | 5 +- .../kotlin/commonizer/TargetDependent.kt | 3 + .../commonizer/cli/NativeTargetsOptionType.kt | 3 +- .../cli/OutputCommonizerTargetOptionType.kt | 2 +- .../jetbrains/kotlin/commonizer/cli/Task.kt | 2 +- .../kotlin/commonizer/cli/TaskType.kt | 1 + .../kotlin/commonizer/cli/nativeTasks.kt | 42 ++++---- .../kotlin/commonizer/core/RootCommonizer.kt | 10 +- .../org/jetbrains/kotlin/commonizer/facade.kt | 2 +- .../mergedtree/CirProvidedClassifiers.kt | 4 - .../kotlin/commonizer/CommonizerFacadeTest.kt | 22 ++-- 21 files changed, 269 insertions(+), 59 deletions(-) create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalKotlinNativePlatformDependencies.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalNativeDistributionCommonizerTask.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/KotlinNativeCommonizerToolRunner.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/KotlinNativeCommonizerToolRunner.kt index 5cfae29e47c..da433a19e43 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/KotlinNativeCommonizerToolRunner.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/KotlinNativeCommonizerToolRunner.kt @@ -31,7 +31,7 @@ internal class KotlinNativeCommonizerToolRunner(project: Project) : KotlinToolRu override val defaultMaxHeapSize: String get() = "4G" - override val mustRunViaExec get() = false // because it's not enough the standard Gradle wrapper's heap size + override val mustRunViaExec get() = true // because it's not enough the standard Gradle wrapper's heap size override fun getCustomJvmArgs() = PropertiesProvider(project).commonizerJvmArgs?.split("\\s+".toRegex()).orEmpty() } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt index abebdce7a6a..fc868145653 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt @@ -224,6 +224,12 @@ internal class PropertiesProvider private constructor(private val project: Proje val enableCInteropCommonization: Boolean get() = booleanProperty("kotlin.mpp.enableCInteropCommonization") ?: false + /** + * Enables experimental commonization of 'higher level' shared native source sets + */ + val enableHierarchicalCommonization: Boolean + get() = booleanProperty("kotlin.mpp.enableHierarchicalCommonization") ?: false + /** * Dependencies caching strategy for all targets that support caches. */ diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/AbstractCInteropCommonizerTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/AbstractCInteropCommonizerTask.kt index 764f7a2a62a..ce84def1a0b 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/AbstractCInteropCommonizerTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/AbstractCInteropCommonizerTask.kt @@ -27,9 +27,10 @@ internal abstract class AbstractCInteropCommonizerTask : DefaultTask() { internal abstract fun getCommonizationParameters(compilation: KotlinSharedNativeCompilation): CInteropCommonizationParameters? internal fun getLibraries(compilation: KotlinSharedNativeCompilation): FileCollection { + val compilationCommonizerTarget = project.getCommonizerTarget(compilation) ?: return project.files() val fileProvider = project.provider> { val parameters = getCommonizationParameters(compilation) ?: return@provider emptySet() - HierarchicalCommonizerOutputLayout.getTargetDirectory(outputDirectory(parameters), parameters.commonizerTarget) + HierarchicalCommonizerOutputLayout.getTargetDirectory(outputDirectory(parameters), compilationCommonizerTarget) .listFiles().orEmpty().toSet() } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CInteropCommonizerTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CInteropCommonizerTask.kt index 1746df6d4e6..a44e5d73813 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CInteropCommonizerTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CInteropCommonizerTask.kt @@ -24,7 +24,6 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.kotlinSourceSetsIncludingDefault import org.jetbrains.kotlin.gradle.plugin.sources.resolveAllDependsOnSourceSets import org.jetbrains.kotlin.gradle.targets.native.internal.CInteropCommonizerTask.CInteropGist import org.jetbrains.kotlin.gradle.tasks.CInteropProcess -import org.jetbrains.kotlin.commonizer.util.transitiveClosure import org.jetbrains.kotlin.gradle.utils.fileProvider import org.jetbrains.kotlin.konan.target.KonanTarget import java.io.File @@ -53,11 +52,11 @@ internal open class CInteropCommonizerTask : AbstractCInteropCommonizerTask() { * All library files produced by the [Project.commonizeNativeDistributionTask] that are relevant for commonization */ @get:Classpath - internal val nativeDistributionLibraries: Set + internal val nonHierarchicalNativeDistributionLibraries: Set get() { val commonizeNativeDistribution = project.commonizeNativeDistributionTask.get() return getCommonizationParameters().flatMapTo(mutableSetOf()) { parameters -> - parameters.commonizerTarget.withAllTransitiveTargets().flatMap { target -> + parameters.commonizerTarget.withAllAncestors().flatMap { target -> commonizeNativeDistribution.commonizerTargetOutputDirectories.flatMap { outputDirectory -> NativeDistributionCommonizerOutputLayout.getTargetDirectory(outputDirectory, target) .listFiles().orEmpty().toList() @@ -111,11 +110,24 @@ internal open class CInteropCommonizerTask : AbstractCInteropCommonizerTask() { konanHome = project.file(project.konanHome), outputCommonizerTarget = parameters.commonizerTarget, inputLibraries = cinteropsForTarget.map { it.libraryFile.get() }.toSet(), - dependencyLibraries = cinteropsForTarget.flatMap { it.dependencies.files }.toSet() + nativeDistributionLibraries, + dependencyLibraries = cinteropsForTarget.flatMap { it.dependencies.files }.toSet() + nativeDistributionLibraries(parameters), outputDirectory = outputDirectory(parameters) ) } + private fun nativeDistributionLibraries(parameters: CInteropCommonizationParameters): Set { + val task = project.commonizeNativeDistributionHierarchicalTask?.get() ?: return nonHierarchicalNativeDistributionLibraries + + val rootTarget = task.rootCommonizerTargets + .firstOrNull { rootTarget -> parameters.commonizerTarget in rootTarget } ?: return emptySet() + + val rootTargetOutput = task.getRootOutputDirectory(rootTarget) + + return parameters.commonizerTarget.withAllAncestors().flatMap { target -> + HierarchicalCommonizerOutputLayout.getTargetDirectory(rootTargetOutput, target).listFiles().orEmpty().toList() + }.toSet() + } + @Nested internal fun getCommonizationParameters(): Set { val sharedNativeCompilations = (project.multiplatformExtensionOrNull ?: return emptySet()) @@ -136,7 +148,7 @@ internal open class CInteropCommonizerTask : AbstractCInteropCommonizerTask() { return sharedNativeCompilations.mapNotNull(::getCommonizationParameters).toSet() .run(::removeNotRegisteredInterops) .run(::removeEmptyInterops) - .run(::removeHierarchicalParameters) + .run(if (project.isHierarchicalCommonizationEnabled) ::identity else ::removeHierarchicalParameters) .run(::removeRedundantParameters) } @@ -193,6 +205,8 @@ private fun removeEmptyInterops(parameters: Set return parameters.filterTo(mutableSetOf()) { it.interops.isNotEmpty() } } +private fun identity(parameters: Set) = parameters + private fun removeHierarchicalParameters(parameters: Set): Set { return parameters.filterTo(mutableSetOf()) { it.commonizerTarget.level <= 1 } } @@ -210,10 +224,6 @@ private operator fun CommonizerTarget.contains(other: CommonizerTarget): Boolean return this.isAncestorOf(other) } -private fun SharedCommonizerTarget.withAllTransitiveTargets(): Set { - return setOf(this) + transitiveClosure(this) { if (this is SharedCommonizerTarget) this.targets else emptySet() } -} - private fun Project.getDependingNativeCompilations(compilation: KotlinSharedNativeCompilation): Set { /** * Some implementations of [KotlinCompilation] do not contain the default source set in diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt index 543f9497c39..0e8abc1ca89 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt @@ -16,6 +16,8 @@ import org.jetbrains.kotlin.gradle.tasks.registerTask internal val Project.isCInteropCommonizationEnabled: Boolean get() = PropertiesProvider(this).enableCInteropCommonization +internal val Project.isHierarchicalCommonizationEnabled: Boolean get() = PropertiesProvider(this).enableHierarchicalCommonization + internal val Project.commonizeTask: TaskProvider get() = locateOrRegisterTask( "commonize", @@ -26,7 +28,6 @@ internal val Project.commonizeTask: TaskProvider } ) - /** * Keeping this task/task name for IDE compatibility which is invoking 'runCommonizer' during sync */ @@ -46,7 +47,10 @@ internal val Project.commonizeCInteropTask: TaskProvider if (isCInteropCommonizationEnabled) { return locateOrRegisterTask( "commonizeCInterop", - invokeWhenRegistered = { commonizeTask.dependsOn(this); dependsOn(commonizeNativeDistributionTask) }, + invokeWhenRegistered = { + commonizeTask.dependsOn(this) + commonizeNativeDistributionHierarchicalTask?.let(this::dependsOn) ?: dependsOn(commonizeNativeDistributionTask) + }, configureTask = { group = "interop" description = "Invokes the commonizer on c-interop bindings of the project" @@ -82,6 +86,19 @@ internal val Project.commonizeNativeDistributionTask: TaskProvider? + get() { + if (!isHierarchicalCommonizationEnabled) return null + return locateOrRegisterTask( + "commonizeNativeDistributionHierarchically", + invokeWhenRegistered = { commonizeTask.dependsOn(this) }, + configureTask = { + group = "interop" + description = "Invokes the commonizer on platform libraries provided by the Kotlin/Native distribution" + } + ) + } + private inline fun Project.locateOrRegisterTask( name: String, args: List = emptyList(), diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalKotlinNativePlatformDependencies.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalKotlinNativePlatformDependencies.kt new file mode 100644 index 00000000000..224197866ff --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalKotlinNativePlatformDependencies.kt @@ -0,0 +1,54 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.targets.native.internal + +import org.gradle.api.Project +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.FileCollection +import org.jetbrains.kotlin.commonizer.HierarchicalCommonizerOutputLayout +import org.jetbrains.kotlin.commonizer.KonanDistribution +import org.jetbrains.kotlin.commonizer.isAncestorOf +import org.jetbrains.kotlin.commonizer.stdlib +import org.jetbrains.kotlin.compilerRunner.konanHome +import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull +import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet +import org.jetbrains.kotlin.gradle.targets.metadata.getMetadataCompilationForSourceSet +import java.io.File + +internal fun Project.setUpHierarchicalKotlinNativePlatformDependencies() { + val task = commonizeNativeDistributionHierarchicalTask?.get() ?: return + val kotlin = multiplatformExtensionOrNull ?: return + kotlin.sourceSets.forEach { sourceSet -> + val target = getCommonizerTarget(sourceSet) ?: return@forEach + + val rootTarget = task.rootCommonizerTargets + .firstOrNull { rootTarget -> rootTarget == target || rootTarget.isAncestorOf(target) } + ?: return@forEach + + val rootOutputDirectory = task.getRootOutputDirectory(rootTarget) + val targetOutputDirectory = HierarchicalCommonizerOutputLayout.getTargetDirectory(rootOutputDirectory, target) + + val dependencies = project.lazyFiles { targetOutputDirectory.listFiles().orEmpty().toList() }.builtBy(task) + val stdlib = project.lazyFiles { listOf(konanDistribution.stdlib) } + addDependencies(sourceSet, dependencies) + addDependencies(sourceSet, stdlib) + } +} + +private fun Project.addDependencies(sourceSet: KotlinSourceSet, libraries: FileCollection) { + getMetadataCompilationForSourceSet(sourceSet)?.let { compilation -> + compilation.compileDependencyFiles += libraries + } + dependencies.add(sourceSet.implementationMetadataConfigurationName, libraries) +} + + +private fun Project.lazyFiles(provider: () -> Iterable): ConfigurableFileCollection { + return project.files(project.provider { provider() }) +} + +private val Project.konanDistribution: KonanDistribution + get() = KonanDistribution(project.file(konanHome)) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalNativeDistributionCommonizerTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalNativeDistributionCommonizerTask.kt new file mode 100644 index 00000000000..6af1b96343a --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalNativeDistributionCommonizerTask.kt @@ -0,0 +1,100 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.targets.native.internal + +import org.gradle.api.DefaultTask +import org.gradle.api.Project +import org.gradle.api.tasks.* +import org.jetbrains.kotlin.commonizer.SharedCommonizerTarget +import org.jetbrains.kotlin.commonizer.identityString +import org.jetbrains.kotlin.commonizer.isAncestorOf +import org.jetbrains.kotlin.compilerRunner.KotlinNativeCommonizerToolRunner +import org.jetbrains.kotlin.compilerRunner.konanHome +import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull +import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion +import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMONIZED_LIBS_DIR +import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMON_LIBS_DIR +import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_KLIB_DIR +import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR +import java.io.File +import java.nio.charset.StandardCharsets +import java.util.* + +internal open class HierarchicalNativeDistributionCommonizerTask : DefaultTask() { + + private val konanHome = project.file(project.konanHome) + + @get:Input + internal val rootCommonizerTargets: Set + get() = project.getRootCommonizerTargets() + + @get:PathSensitive(PathSensitivity.ABSOLUTE) + @get:InputDirectory + @Suppress("unused") // Only for up-to-date checker. The directory with the original common libs. + val originalCommonLibrariesDirectory = konanHome + .resolve(KONAN_DISTRIBUTION_KLIB_DIR) + .resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR) + + @get:PathSensitive(PathSensitivity.ABSOLUTE) + @get:InputDirectory + @Suppress("unused") // Only for up-to-date checker. The directory with the original platform libs. + val originalPlatformLibrariesDirectory = konanHome + .resolve(KONAN_DISTRIBUTION_KLIB_DIR) + .resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR) + + @get:OutputDirectories + @Suppress("unused") // Only for up-to-date checker. The directory with the original platform libs. + val outputDirectories: Set + get() = rootCommonizerTargets.map(::getRootOutputDirectory).toSet() + + internal fun getRootOutputDirectory(target: SharedCommonizerTarget): File { + val kotlinVersion = checkNotNull(project.getKotlinPluginVersion()) { "Missing Kotlin Plugin version" } + + val discriminator = buildString { + append(target.identityString) + append("-") + append(kotlinVersion.toLowerCase().base64) + } + + return project.file(konanHome) + .resolve(KONAN_DISTRIBUTION_KLIB_DIR) + .resolve(KONAN_DISTRIBUTION_COMMONIZED_LIBS_DIR) + .resolve(discriminator) + } + + @TaskAction + protected fun run() { + for (target in rootCommonizerTargets) { + getRootOutputDirectory(target).deleteRecursively() + KotlinNativeCommonizerToolRunner(project).run(getCommandLineArguments(target)) + } + } + + private fun getCommandLineArguments(target: SharedCommonizerTarget): List { + return mutableListOf().apply { + this += "native-dist-commonize" + this += "-distribution-path" + this += konanHome.absolutePath + this += "-output-path" + this += getRootOutputDirectory(target).absolutePath + this += "-output-commonizer-target" + this += target.identityString + } + } +} + +private fun Project.getRootCommonizerTargets(): Set { + val kotlin = multiplatformExtensionOrNull ?: return emptySet() + val allTargets = kotlin.sourceSets + .mapNotNull { sourceSet -> getCommonizerTarget(sourceSet) } + .filterIsInstance() + return allTargets.filter { target -> allTargets.none { otherTarget -> otherTarget isAncestorOf target } }.toSet() +} + +private val String.base64 + get() = base64Encoder.encodeToString(toByteArray(StandardCharsets.UTF_8)) + +private val base64Encoder = Base64.getEncoder().withoutPadding() diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt index 7aef1a2f21f..653e4ee8c72 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt @@ -47,11 +47,13 @@ internal fun Project.setUpKotlinNativePlatformDependencies() { // not a multiplatform project, nothing to set up return } - val kotlinVersion = getKotlinPluginVersion()!! val allowCommonizer = isAllowCommonizer(kotlinVersion) - val dependencyResolver = NativePlatformDependencyResolver(this, kotlinVersion) + if (allowCommonizer && isHierarchicalCommonizationEnabled) { + return setUpHierarchicalKotlinNativePlatformDependencies() + } + val dependencyResolver = NativePlatformDependencyResolver(this, kotlinVersion) findSourceSetsToAddDependencies(allowCommonizer).forEach { (sourceSet: KotlinSourceSet, sourceSetDeps: Set) -> sourceSetDeps.forEach { sourceSetDep: NativePlatformDependency -> dependencyResolver.addForResolve(sourceSetDep) { resolvedFiles: FileCollection -> diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt index dcc3daac3ab..6618fc1abae 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt @@ -188,7 +188,7 @@ internal fun Project.createTempNativeDistributionCommonizerOutputDirectory(targe ).toFile() } -fun callCommonizerCLI(project: Project, commandLineArguments: List) { +internal fun callCommonizerCLI(project: Project, commandLineArguments: List) { if (commandLineArguments.isEmpty()) return KotlinNativeCommonizerToolRunner(project).run(commandLineArguments) diff --git a/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/KonanDistribution.kt b/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/KonanDistribution.kt index 66f58b9b48a..9eb10c4a8bb 100644 --- a/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/KonanDistribution.kt +++ b/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/KonanDistribution.kt @@ -5,13 +5,19 @@ package org.jetbrains.kotlin.commonizer -import org.jetbrains.kotlin.konan.library.* +import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMON_LIBS_DIR +import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_KLIB_DIR +import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR +import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME import java.io.File public data class KonanDistribution(val root: File) +public val KonanDistribution.konanCommonLibraries: File + get() = root.resolve(KONAN_DISTRIBUTION_KLIB_DIR).resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR) + public val KonanDistribution.stdlib: File - get() = root.resolve(konanCommonLibraryPath(KONAN_STDLIB_NAME)) + get() = konanCommonLibraries.resolve(KONAN_STDLIB_NAME) public val KonanDistribution.klibDir: File get() = root.resolve(KONAN_DISTRIBUTION_KLIB_DIR) diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/CommonizerParameters.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/CommonizerParameters.kt index 957b2bca550..f080c4a5456 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/CommonizerParameters.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/CommonizerParameters.kt @@ -32,8 +32,11 @@ fun CommonizerParameters.getCommonModuleNames(): Set { internal fun CommonizerParameters.dependencyClassifiers(target: CommonizerTarget): CirProvidedClassifiers { val modules = outputTarget.withAllAncestors() + .sortedBy { it.level } .filter { it == target || it isAncestorOf target } .mapNotNull { compatibleTarget -> dependenciesProvider[compatibleTarget] } - return CirProvidedClassifiers.of(modules.map { module -> CirProvidedClassifiers.by(module) } + CirFictitiousFunctionClassifiers) + return modules.fold(CirFictitiousFunctionClassifiers) { classifiers, module -> + CirProvidedClassifiers.of(classifiers, CirProvidedClassifiers.by(module)) + } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/TargetDependent.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/TargetDependent.kt index 0aa7d0c7a67..03eb8ed32b4 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/TargetDependent.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/TargetDependent.kt @@ -63,6 +63,9 @@ internal fun TargetDependent(keys: Iterable, factory: (tar return FactoryBasedTargetDependent(keys.toList(), factory) } +internal fun EagerTargetDependent(keys: Iterable, factory: (target: CommonizerTarget) -> T): TargetDependent { + return keys.associateWith(factory).toTargetDependent() +} private class MapBasedTargetDependent(private val map: Map) : TargetDependent { override val targets: List = map.keys.toList() override fun get(target: CommonizerTarget): T = map.getValue(target) diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/NativeTargetsOptionType.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/NativeTargetsOptionType.kt index 6205b095a28..dac8d705299 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/NativeTargetsOptionType.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/NativeTargetsOptionType.kt @@ -8,7 +8,8 @@ package org.jetbrains.kotlin.commonizer.cli import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.target.KonanTarget.Companion.predefinedTargets -internal object NativeTargetsOptionType : OptionType>("targets", "Comma-separated list of hardware targets") { +internal object NativeTargetsOptionType : OptionType>( + "targets", "Comma-separated list of hardware targets", mandatory = false) { override fun parse(rawValue: String, onError: (reason: String) -> Nothing): Option> { val targetNames = rawValue.split(',') if (targetNames.isEmpty()) onError("No hardware targets specified: $rawValue") diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/OutputCommonizerTargetOptionType.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/OutputCommonizerTargetOptionType.kt index 6ae336e15e1..0bf004f1b44 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/OutputCommonizerTargetOptionType.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/OutputCommonizerTargetOptionType.kt @@ -11,7 +11,7 @@ import org.jetbrains.kotlin.commonizer.parseCommonizerTarget internal object OutputCommonizerTargetOptionType : OptionType( alias = "output-commonizer-target", description = "Shared commonizer target representing the commonized output hierarchy", - mandatory = true + mandatory = false ) { override fun parse(rawValue: String, onError: (reason: String) -> Nothing): Option { return try { diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/Task.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/Task.kt index 5850dd9eb81..ceb43e5b894 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/Task.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/Task.kt @@ -35,7 +35,7 @@ internal abstract class Task(private val options: Collection>) : Compa return option.value as T } - protected inline fun > getOptional(nameFilter: (String) -> Boolean = { true }): T? { + internal inline fun > getOptional(nameFilter: (String) -> Boolean = { true }): T? { val option = options.filter { it.type is O }.singleOrNull { nameFilter(it.type.alias) } if (option != null) check(!option.type.mandatory) diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/TaskType.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/TaskType.kt index 4f77534e46e..e0ccc8ae22c 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/TaskType.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/TaskType.kt @@ -18,6 +18,7 @@ internal enum class TaskType( NativeDistributionOptionType, OutputOptionType, NativeTargetsOptionType, + OutputCommonizerTargetOptionType, BooleanOptionType( "copy-stdlib", "Boolean (default false);\nwhether to copy Kotlin/Native endorsed libraries to the destination", diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/nativeTasks.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/nativeTasks.kt index 950555d3439..ed68d1cb369 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/nativeTasks.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/nativeTasks.kt @@ -48,7 +48,7 @@ internal class NativeKlibCommonize(options: Collection>) : Task(option val destination = getMandatory() val targetLibraries = getMandatory, InputLibrariesOptionType>() val dependencyLibraries = getOptional, DependencyLibrariesOptionType>().orEmpty() - val outputCommonizerTarget = getMandatory() + val outputCommonizerTarget = compatGetOutputTarget() val statsType = getOptional { it == "log-stats" } ?: StatsType.NONE val konanTargets = outputCommonizerTarget.konanTargets @@ -87,8 +87,12 @@ internal class NativeDistributionCommonize(options: Collection>) : Tas override fun execute(logPrefix: String) { val distribution = KonanDistribution(getMandatory()) val destination = getMandatory() - val konanTargets = getMandatory, NativeTargetsOptionType>() - val commonizerTargets = konanTargets.map(::CommonizerTarget) + + val outputTarget = compatGetOutputTarget() + val outputLayout = if (getOptional() != null) + HierarchicalCommonizerOutputLayout + else NativeDistributionCommonizerOutputLayout + val copyStdlib = getOptional { it == "copy-stdlib" } ?: false val copyEndorsedLibs = getOptional { it == "copy-endorsed-libs" } ?: false @@ -96,30 +100,24 @@ internal class NativeDistributionCommonize(options: Collection>) : Tas val progressLogger = ProgressLogger(CliLoggerAdapter(2), startImmediately = true) val libraryLoader = DefaultNativeLibraryLoader(progressLogger) - val repository = KonanDistributionRepository(distribution, commonizerTargets.toSet(), libraryLoader) - val existingTargets = commonizerTargets.filter { repository.getLibraries(it).isNotEmpty() }.toSet() - val statsCollector = StatsCollector(statsType, commonizerTargets) + val repository = KonanDistributionRepository(distribution, outputTarget.allLeaves(), libraryLoader) + val statsCollector = StatsCollector(statsType, outputTarget.withAllAncestors().toList()) val resultsConsumer = buildResultsConsumer { - this add ModuleSerializer(destination, NativeDistributionCommonizerOutputLayout) - this add CopyUnconsumedModulesAsIsConsumer( - repository, destination, commonizerTargets.toSet(), NativeDistributionCommonizerOutputLayout, progressLogger - ) + this add ModuleSerializer(destination, outputLayout) + this add CopyUnconsumedModulesAsIsConsumer(repository, destination, outputTarget.allLeaves(), outputLayout, progressLogger) if (copyStdlib) this add CopyStdlibResultsConsumer(distribution, destination, progressLogger) if (copyEndorsedLibs) this add CopyEndorsedLibrairesResultsConsumer(distribution, destination, progressLogger) - - SharedCommonizerTarget.ifNotEmpty(existingTargets)?.let { sharedTargetForLogger -> - this add LoggingResultsConsumer(sharedTargetForLogger, progressLogger) - } + this add LoggingResultsConsumer(outputTarget, progressLogger) } - val targetNames = commonizerTargets.joinToString { it.prettyName } - val descriptionSuffix = estimateLibrariesCount(repository, commonizerTargets).let { " ($it items)" } + val targetNames = outputTarget.allLeaves().joinToString { it.prettyName } + val descriptionSuffix = estimateLibrariesCount(repository, outputTarget.allLeaves()).let { " ($it items)" } val description = "${logPrefix}Preparing commonized Kotlin/Native libraries for targets $targetNames$descriptionSuffix" println(description) LibraryCommonizer( - outputTarget = SharedCommonizerTarget(commonizerTargets.toSet()), + outputTarget = outputTarget, repository = repository, dependencies = StdlibRepository(distribution, libraryLoader), resultsConsumer = resultsConsumer, @@ -133,8 +131,16 @@ internal class NativeDistributionCommonize(options: Collection>) : Tas } companion object { - private fun estimateLibrariesCount(repository: Repository, targets: List): Int { + private fun estimateLibrariesCount(repository: Repository, targets: Iterable): Int { return targets.flatMap { repository.getLibraries(it) }.count() } } } + +private fun Task.compatGetOutputTarget(): SharedCommonizerTarget { + getOptional()?.let { return it } + val konanTargets = getOptional, NativeTargetsOptionType>() ?: throw IllegalArgumentException( + "Missing ${OutputCommonizerTargetOptionType.alias} or deprecated ${NativeTargetsOptionType.alias} option was specified" + ) + return SharedCommonizerTarget(konanTargets) +} diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/core/RootCommonizer.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/core/RootCommonizer.kt index d30acd828ce..a739ea3a500 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/core/RootCommonizer.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/core/RootCommonizer.kt @@ -5,23 +5,23 @@ package org.jetbrains.kotlin.commonizer.core -import org.jetbrains.kotlin.commonizer.LeafCommonizerTarget +import org.jetbrains.kotlin.commonizer.CommonizerTarget import org.jetbrains.kotlin.commonizer.SharedCommonizerTarget import org.jetbrains.kotlin.commonizer.cir.CirRoot class RootCommonizer : AbstractStandardCommonizer() { - private val leafTargets = mutableSetOf() + private val targets = mutableSetOf() override fun commonizationResult() = CirRoot.create( - target = SharedCommonizerTarget(leafTargets) + target = SharedCommonizerTarget(targets) ) override fun initialize(first: CirRoot) { - leafTargets += first.target as LeafCommonizerTarget + targets += first.target } override fun doCommonizeWith(next: CirRoot): Boolean { - leafTargets += next.target as LeafCommonizerTarget + targets += next.target return true } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/facade.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/facade.kt index 0277822d7f0..3c5773b0c7c 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/facade.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/facade.kt @@ -61,7 +61,7 @@ private fun commonize( private fun getCirTree( parameters: CommonizerParameters, storageManager: StorageManager, target: SharedCommonizerTarget ): TargetDependent { - return TargetDependent(target.targets) { childTarget -> + return EagerTargetDependent(target.targets) { childTarget -> when (childTarget) { is LeafCommonizerTarget -> deserializeCirTree(parameters, parameters.targetProviders[childTarget]) is SharedCommonizerTarget -> commonize(parameters, storageManager, childTarget).assembleCirTree() diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/mergedtree/CirProvidedClassifiers.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/mergedtree/CirProvidedClassifiers.kt index 68de9a650d4..e5a1c67acfc 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/mergedtree/CirProvidedClassifiers.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/mergedtree/CirProvidedClassifiers.kt @@ -50,10 +50,6 @@ interface CirProvidedClassifiers { } } - fun of(delegates: Iterable): CirProvidedClassifiers { - return of(*delegates.filterNotNull().toTypedArray()) - } - fun by(modulesProvider: ModulesProvider?): CirProvidedClassifiers = if (modulesProvider != null) CirProvidedClassifiersByModules.load(modulesProvider) else EMPTY } diff --git a/native/commonizer/tests/org/jetbrains/kotlin/commonizer/CommonizerFacadeTest.kt b/native/commonizer/tests/org/jetbrains/kotlin/commonizer/CommonizerFacadeTest.kt index 953acbe6f14..ecf10290ca3 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/commonizer/CommonizerFacadeTest.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/commonizer/CommonizerFacadeTest.kt @@ -68,19 +68,23 @@ class CommonizerFacadeTest { private fun Map>.toCommonizerParameters( resultsConsumer: ResultsConsumer, manifestDataProvider: NativeManifestDataProvider = MockNativeManifestDataProvider() - ) = CommonizerParameters( - targetProviders = mapKeys { (targetName, _) -> LeafCommonizerTarget(targetName) }.toTargetDependent() - .map { target, moduleNames -> + ): CommonizerParameters { + val targetDependentModuleNames = mapKeys { (targetName, _) -> LeafCommonizerTarget(targetName) }.toTargetDependent() + val sharedTarget = SharedCommonizerTarget(targetDependentModuleNames.targets.toSet()) + + return CommonizerParameters( + outputTarget = sharedTarget, + dependenciesProvider = TargetDependent(sharedTarget.withAllAncestors()) { null }, + manifestProvider = TargetDependent(sharedTarget.withAllAncestors()) { manifestDataProvider }, + targetProviders = targetDependentModuleNames.map { target, moduleNames -> TargetProvider( target = target, - modulesProvider = MockModulesProvider.create(moduleNames), - dependencyModulesProvider = null, - manifestProvider = manifestDataProvider, + modulesProvider = MockModulesProvider.create(moduleNames) ) }, - resultsConsumer = resultsConsumer, - commonManifestProvider = manifestDataProvider - ) + resultsConsumer = resultsConsumer, + ) + } private fun doTestNothingToCommonize(originalModules: Map>) { val results = MockResultsConsumer()