[Gradle] NativeDistributionCommonizationCache: Support associative commonization

^KT-47301

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/CommonizerIT.kt

@@ -8,7 +8,10 @@ package org.jetbrains.kotlin.gradle
 import org.gradle.internal.os.OperatingSystem
 import org.jetbrains.kotlin.gradle.internals.DISABLED_NATIVE_TARGETS_REPORTER_WARNING_PREFIX
 import org.jetbrains.kotlin.incremental.testingUtils.assertEqualDirectories
-import kotlin.test.*
+import kotlin.test.Test
+import kotlin.test.assertFalse
+import kotlin.test.assertTrue
+import kotlin.test.fail
 
 class CommonizerIT : BaseGradleIT() {
     override val defaultGradleVersion: GradleVersionRequired = GradleVersionRequired.FOR_MPP_SUPPORT
@@ -17,26 +20,30 @@ class CommonizerIT : BaseGradleIT() {
         private const val commonizerOutput = "Preparing commonized Kotlin/Native libraries"
     }
 
-    @Ignore // TODO NOW
     @Test
     fun `test commonizeNativeDistributionWithIosLinuxWindows`() {
         with(Project("commonizeNativeDistributionWithIosLinuxWindows")) {
-            build(":p1:commonize", "-Pkotlin.mpp.enableNativeDistributionCommonizationCache=false") {
+            build(":cleanNativeDistributionCommonization") {
-                assertTasksExecuted(":p1:commonizeNativeDistribution")
+                assertSuccessful()
+            }
+
+            build("commonize", "-Pkotlin.mpp.enableNativeDistributionCommonizationCache=false") {
+                assertTasksExecuted(":commonizeNativeDistribution")
                 assertContains(DISABLED_NATIVE_TARGETS_REPORTER_WARNING_PREFIX)
                 assertContains(commonizerOutput)
                 assertSuccessful()
             }
 
-            build(":p1:commonize", "--rerun-tasks", "-Pkotlin.mpp.enableNativeDistributionCommonizationCache=true") {
+            build("commonize", "--rerun-tasks", "-Pkotlin.mpp.enableNativeDistributionCommonizationCache=true") {
-                assertTasksExecuted(":p1:commonizeNativeDistribution")
+                assertTasksExecuted(":commonizeNativeDistribution")
                 assertContains("Native Distribution Commonization: Cache hit")
+                assertContains("Native Distribution Commonization: All available targets are commonized already")
                 assertNotContains(commonizerOutput)
                 assertSuccessful()
             }
 
-            build(":p1:commonize", "--rerun-tasks", "-Pkotlin.mpp.enableNativeDistributionCommonizationCache=false") {
+            build("commonize", "--rerun-tasks", "-Pkotlin.mpp.enableNativeDistributionCommonizationCache=false") {
-                assertTasksExecuted(":p1:commonizeNativeDistribution")
+                assertTasksExecuted(":commonizeNativeDistribution")
                 assertContains("Native Distribution Commonization: Cache disabled")
                 assertContains(commonizerOutput)
                 assertSuccessful()

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt

@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.gradle.targets.native.internal
 
 import org.gradle.api.Project
 import org.gradle.api.Task
+import org.gradle.api.tasks.Delete
 import org.gradle.api.tasks.TaskProvider
 import org.jetbrains.kotlin.gradle.internal.isInIdeaSync
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider
@@ -98,7 +99,7 @@ internal val Project.commonizeNativeDistributionHierarchicallyTask: TaskProvider
         if (!isHierarchicalCommonizationEnabled) return null
         return rootProject.locateOrRegisterTask(
             "commonizeNativeDistribution",
-            invokeWhenRegistered = { commonizeTask.dependsOn(this) },
+            invokeWhenRegistered = { rootProject.commonizeTask.dependsOn(this); cleanNativeDistributionCommonizationTask },
             configureTask = {
                 group = "interop"
                 description = "Invokes the commonizer on platform libraries provided by the Kotlin/Native distribution"
@@ -106,6 +107,19 @@ internal val Project.commonizeNativeDistributionHierarchicallyTask: TaskProvider
         )
     }
 
+internal val Project.cleanNativeDistributionCommonizationTask: TaskProvider<Delete>?
+    get() {
+        val commonizeNativeDistributionTask = commonizeNativeDistributionHierarchicallyTask ?: return null
+        return rootProject.locateOrRegisterTask(
+            "cleanNativeDistributionCommonization",
+            configureTask = {
+                group = "interop"
+                description = "Deletes all previously commonized klib's from the Kotlin/Native distribution"
+                delete(commonizeNativeDistributionTask.map { it.getRootOutputDirectory() })
+            }
+        )
+    }
+
 private inline fun <reified T : Task> Project.locateOrRegisterTask(
     name: String,
     args: List<Any> = emptyList(),

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/HierarchicalKotlinNativePlatformDependencies.kt

@@ -14,6 +14,7 @@ import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
 import org.jetbrains.kotlin.gradle.plugin.sources.DefaultKotlinSourceSet
 import org.jetbrains.kotlin.gradle.targets.metadata.getMetadataCompilationForSourceSet
 import org.jetbrains.kotlin.gradle.utils.filesProvider
+import java.io.File
 
 internal fun Project.setUpHierarchicalKotlinNativePlatformDependencies() {
     val kotlin = multiplatformExtensionOrNull ?: return
@@ -35,12 +36,12 @@ internal fun Project.getNativeDistributionDependencies(target: CommonizerTarget)
 }
 
 private fun Project.getOriginalPlatformLibrariesFor(target: LeafCommonizerTarget): FileCollection {
-    return project.filesProvider { konanDistribution.platformLibsDir.resolve(target.konanTarget.name).listFiles().orEmpty().toSet() }
+    return project.filesProvider { konanDistribution.platformLibsDir.resolve(target.konanTarget.name).listLibraryFiles().toSet() }
 }
 
 private fun HierarchicalNativeDistributionCommonizerTask.getCommonizedPlatformLibrariesFor(target: SharedCommonizerTarget): FileCollection {
     val targetOutputDirectory = CommonizerOutputFileLayout.resolveCommonizedDirectory(getRootOutputDirectory(), target)
-    return project.filesProvider { targetOutputDirectory.listFiles().orEmpty().toList() }.builtBy(this)
+    return project.filesProvider { targetOutputDirectory.listLibraryFiles() }.builtBy(this)
 }
 
 private fun HierarchicalNativeDistributionCommonizerTask.getCommonizedDependenciesFor(target: CommonizerTarget): FileCollection {
@@ -64,3 +65,6 @@ private fun Project.addDependencies(sourceSet: KotlinSourceSet, libraries: FileC
 
 private val Project.konanDistribution: KonanDistribution
     get() = KonanDistribution(project.file(konanHome))
+
+private fun File.listLibraryFiles(): List<File> = listFiles().orEmpty()
+    .filter { it.isDirectory || it.extension == "klib" }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizationCache.kt

@@ -29,7 +29,6 @@ internal class NativeDistributionCommonizationCache(
     ) {
         if (!project.isNativeDistributionCommonizationCacheEnabled) {
             logInfo("Cache disabled")
-            return commonizer.commonizeNativeDistribution(konanHome, outputDirectory, outputTargets, logLevel)
         }
 
         val cachedOutputTargets = outputTargets
@@ -37,25 +36,26 @@ internal class NativeDistributionCommonizationCache(
             .onEach { outputTarget -> logInfo("Cache hit: $outputTarget already commonized") }
             .toSet()
 
-        val missingOutputTargets = outputTargets - cachedOutputTargets
+        val enqueuedOutputTargets = if (project.isNativeDistributionCommonizationCacheEnabled) outputTargets - cachedOutputTargets
+        else outputTargets
 
-        if (canReturnFast(konanHome, missingOutputTargets)) {
+        if (canReturnFast(konanHome, enqueuedOutputTargets)) {
             logInfo("All available targets are commonized already - Nothing to do")
             return
         }
 
-        missingOutputTargets
+        enqueuedOutputTargets
             .map { outputTarget -> resolveCommonizedDirectory(outputDirectory, outputTarget) }
             .forEach { commonizedDirectory -> if (commonizedDirectory.exists()) commonizedDirectory.deleteRecursively() }
 
         commonizer.commonizeNativeDistribution(
-            konanHome, outputDirectory, missingOutputTargets, logLevel
+            konanHome, outputDirectory, enqueuedOutputTargets, logLevel
         )
 
-        missingOutputTargets
+        enqueuedOutputTargets
             .map { outputTarget -> resolveCommonizedDirectory(outputDirectory, outputTarget) }
             .filter { commonizedDirectory -> commonizedDirectory.isDirectory }
-            .forEach { commonizedDirectory -> commonizedDirectory.resolve(".success").isFile }
+            .forEach { commonizedDirectory -> commonizedDirectory.resolve(".success").createNewFile() }
     }
 
     private fun isCached(directory: File): Boolean {

native/commonizer/src/org/jetbrains/kotlin/commonizer/facade.kt

@@ -46,11 +46,11 @@ internal fun commonizeTarget(
     inputs: TargetDependent<CirTreeRoot?>,
     output: CommonizerTarget
 ): CirRootNode? {
-    parameters.logger.progress(output, "Commonized declarations from ${inputs.targets}") {
+    val availableTrees = inputs.filterNonNull()
-        val availableTrees = inputs.filterNonNull()
+    /* Nothing to merge */
-        /* Nothing to merge */
+    if (availableTrees.size == 0) return null
-        if (availableTrees.size == 0) return null
 
+    parameters.logger.progress(output, "Commonized declarations from ${inputs.targets}") {
         val classifiers = CirKnownClassifiers(
             commonizedNodes = CirCommonizedClassifierNodes.default(),
             commonDependencies = parameters.dependencyClassifiers(output)