From 73ffc0d18091008da22f8456deded814ff5b1397 Mon Sep 17 00:00:00 2001 From: "sebastian.sellmair" Date: Thu, 17 Jun 2021 13:25:26 +0200 Subject: [PATCH] [Gradle] NativeDistributionCommonizationCache: Support associative commonization ^KT-47301 --- .../jetbrains/kotlin/gradle/CommonizerIT.kt | 23 ++++++++++++------- .../native/internal/CommonizerTasks.kt | 16 ++++++++++++- ...rchicalKotlinNativePlatformDependencies.kt | 8 +++++-- .../NativeDistributionCommonizationCache.kt | 14 +++++------ .../org/jetbrains/kotlin/commonizer/facade.kt | 8 +++---- 5 files changed, 47 insertions(+), 22 deletions(-) diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/CommonizerIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/CommonizerIT.kt index 091b52fd55b..f203608bff1 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/CommonizerIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/CommonizerIT.kt @@ -8,7 +8,10 @@ package org.jetbrains.kotlin.gradle import org.gradle.internal.os.OperatingSystem import org.jetbrains.kotlin.gradle.internals.DISABLED_NATIVE_TARGETS_REPORTER_WARNING_PREFIX import org.jetbrains.kotlin.incremental.testingUtils.assertEqualDirectories -import kotlin.test.* +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import kotlin.test.fail class CommonizerIT : BaseGradleIT() { override val defaultGradleVersion: GradleVersionRequired = GradleVersionRequired.FOR_MPP_SUPPORT @@ -17,26 +20,30 @@ class CommonizerIT : BaseGradleIT() { private const val commonizerOutput = "Preparing commonized Kotlin/Native libraries" } - @Ignore // TODO NOW @Test fun `test commonizeNativeDistributionWithIosLinuxWindows`() { with(Project("commonizeNativeDistributionWithIosLinuxWindows")) { - build(":p1:commonize", "-Pkotlin.mpp.enableNativeDistributionCommonizationCache=false") { - assertTasksExecuted(":p1:commonizeNativeDistribution") + build(":cleanNativeDistributionCommonization") { + assertSuccessful() + } + + build("commonize", "-Pkotlin.mpp.enableNativeDistributionCommonizationCache=false") { + assertTasksExecuted(":commonizeNativeDistribution") assertContains(DISABLED_NATIVE_TARGETS_REPORTER_WARNING_PREFIX) assertContains(commonizerOutput) assertSuccessful() } - build(":p1:commonize", "--rerun-tasks", "-Pkotlin.mpp.enableNativeDistributionCommonizationCache=true") { - assertTasksExecuted(":p1:commonizeNativeDistribution") + build("commonize", "--rerun-tasks", "-Pkotlin.mpp.enableNativeDistributionCommonizationCache=true") { + assertTasksExecuted(":commonizeNativeDistribution") assertContains("Native Distribution Commonization: Cache hit") + assertContains("Native Distribution Commonization: All available targets are commonized already") assertNotContains(commonizerOutput) assertSuccessful() } - build(":p1:commonize", "--rerun-tasks", "-Pkotlin.mpp.enableNativeDistributionCommonizationCache=false") { - assertTasksExecuted(":p1:commonizeNativeDistribution") + build("commonize", "--rerun-tasks", "-Pkotlin.mpp.enableNativeDistributionCommonizationCache=false") { + assertTasksExecuted(":commonizeNativeDistribution") assertContains("Native Distribution Commonization: Cache disabled") assertContains(commonizerOutput) assertSuccessful() diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt index d2574efca76..cfe8b017a8a 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt @@ -7,6 +7,7 @@ package org.jetbrains.kotlin.gradle.targets.native.internal import org.gradle.api.Project import org.gradle.api.Task +import org.gradle.api.tasks.Delete import org.gradle.api.tasks.TaskProvider import org.jetbrains.kotlin.gradle.internal.isInIdeaSync import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider @@ -98,7 +99,7 @@ internal val Project.commonizeNativeDistributionHierarchicallyTask: TaskProvider if (!isHierarchicalCommonizationEnabled) return null return rootProject.locateOrRegisterTask( "commonizeNativeDistribution", - invokeWhenRegistered = { commonizeTask.dependsOn(this) }, + invokeWhenRegistered = { rootProject.commonizeTask.dependsOn(this); cleanNativeDistributionCommonizationTask }, configureTask = { group = "interop" description = "Invokes the commonizer on platform libraries provided by the Kotlin/Native distribution" @@ -106,6 +107,19 @@ internal val Project.commonizeNativeDistributionHierarchicallyTask: TaskProvider ) } +internal val Project.cleanNativeDistributionCommonizationTask: TaskProvider? + get() { + val commonizeNativeDistributionTask = commonizeNativeDistributionHierarchicallyTask ?: return null + return rootProject.locateOrRegisterTask( + "cleanNativeDistributionCommonization", + configureTask = { + group = "interop" + description = "Deletes all previously commonized klib's from the Kotlin/Native distribution" + delete(commonizeNativeDistributionTask.map { it.getRootOutputDirectory() }) + } + ) + } + private inline fun Project.locateOrRegisterTask( name: String, args: List = emptyList(), diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalKotlinNativePlatformDependencies.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalKotlinNativePlatformDependencies.kt index 07a4319e12c..16488d8b725 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalKotlinNativePlatformDependencies.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalKotlinNativePlatformDependencies.kt @@ -14,6 +14,7 @@ import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet import org.jetbrains.kotlin.gradle.plugin.sources.DefaultKotlinSourceSet import org.jetbrains.kotlin.gradle.targets.metadata.getMetadataCompilationForSourceSet import org.jetbrains.kotlin.gradle.utils.filesProvider +import java.io.File internal fun Project.setUpHierarchicalKotlinNativePlatformDependencies() { val kotlin = multiplatformExtensionOrNull ?: return @@ -35,12 +36,12 @@ internal fun Project.getNativeDistributionDependencies(target: CommonizerTarget) } private fun Project.getOriginalPlatformLibrariesFor(target: LeafCommonizerTarget): FileCollection { - return project.filesProvider { konanDistribution.platformLibsDir.resolve(target.konanTarget.name).listFiles().orEmpty().toSet() } + return project.filesProvider { konanDistribution.platformLibsDir.resolve(target.konanTarget.name).listLibraryFiles().toSet() } } private fun HierarchicalNativeDistributionCommonizerTask.getCommonizedPlatformLibrariesFor(target: SharedCommonizerTarget): FileCollection { val targetOutputDirectory = CommonizerOutputFileLayout.resolveCommonizedDirectory(getRootOutputDirectory(), target) - return project.filesProvider { targetOutputDirectory.listFiles().orEmpty().toList() }.builtBy(this) + return project.filesProvider { targetOutputDirectory.listLibraryFiles() }.builtBy(this) } private fun HierarchicalNativeDistributionCommonizerTask.getCommonizedDependenciesFor(target: CommonizerTarget): FileCollection { @@ -64,3 +65,6 @@ private fun Project.addDependencies(sourceSet: KotlinSourceSet, libraries: FileC private val Project.konanDistribution: KonanDistribution get() = KonanDistribution(project.file(konanHome)) + +private fun File.listLibraryFiles(): List = listFiles().orEmpty() + .filter { it.isDirectory || it.extension == "klib" } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizationCache.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizationCache.kt index b5b17cfcfe8..a7ae463602f 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizationCache.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizationCache.kt @@ -29,7 +29,6 @@ internal class NativeDistributionCommonizationCache( ) { if (!project.isNativeDistributionCommonizationCacheEnabled) { logInfo("Cache disabled") - return commonizer.commonizeNativeDistribution(konanHome, outputDirectory, outputTargets, logLevel) } val cachedOutputTargets = outputTargets @@ -37,25 +36,26 @@ internal class NativeDistributionCommonizationCache( .onEach { outputTarget -> logInfo("Cache hit: $outputTarget already commonized") } .toSet() - val missingOutputTargets = outputTargets - cachedOutputTargets + val enqueuedOutputTargets = if (project.isNativeDistributionCommonizationCacheEnabled) outputTargets - cachedOutputTargets + else outputTargets - if (canReturnFast(konanHome, missingOutputTargets)) { + if (canReturnFast(konanHome, enqueuedOutputTargets)) { logInfo("All available targets are commonized already - Nothing to do") return } - missingOutputTargets + enqueuedOutputTargets .map { outputTarget -> resolveCommonizedDirectory(outputDirectory, outputTarget) } .forEach { commonizedDirectory -> if (commonizedDirectory.exists()) commonizedDirectory.deleteRecursively() } commonizer.commonizeNativeDistribution( - konanHome, outputDirectory, missingOutputTargets, logLevel + konanHome, outputDirectory, enqueuedOutputTargets, logLevel ) - missingOutputTargets + enqueuedOutputTargets .map { outputTarget -> resolveCommonizedDirectory(outputDirectory, outputTarget) } .filter { commonizedDirectory -> commonizedDirectory.isDirectory } - .forEach { commonizedDirectory -> commonizedDirectory.resolve(".success").isFile } + .forEach { commonizedDirectory -> commonizedDirectory.resolve(".success").createNewFile() } } private fun isCached(directory: File): Boolean { diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/facade.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/facade.kt index 9fe2e0a4945..391608c9ad5 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/facade.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/facade.kt @@ -46,11 +46,11 @@ internal fun commonizeTarget( inputs: TargetDependent, output: CommonizerTarget ): CirRootNode? { - parameters.logger.progress(output, "Commonized declarations from ${inputs.targets}") { - val availableTrees = inputs.filterNonNull() - /* Nothing to merge */ - if (availableTrees.size == 0) return null + val availableTrees = inputs.filterNonNull() + /* Nothing to merge */ + if (availableTrees.size == 0) return null + parameters.logger.progress(output, "Commonized declarations from ${inputs.targets}") { val classifiers = CirKnownClassifiers( commonizedNodes = CirCommonizedClassifierNodes.default(), commonDependencies = parameters.dependencyClassifiers(output)