[Gradle][MPP] Mark 'commonizeNativeDistribution' as UP-TO-DATE when cache hits

^KT-52632 Verification Pending

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/CommonizerIT.kt

@@ -39,7 +39,7 @@ class CommonizerIT : BaseGradleIT() {
             }
 
             build("commonize", "-Pkotlin.mpp.enableNativeDistributionCommonizationCache=true") {
-                assertTasksExecuted(":commonizeNativeDistribution")
+                assertTasksUpToDate(":commonizeNativeDistribution")
                 assertNativeDistributionCommonizationCacheHit()
                 assertContains("Native Distribution Commonization: All available targets are commonized already")
                 assertContains("Native Distribution Commonization: Lock acquired")
@@ -552,6 +552,7 @@ class CommonizerIT : BaseGradleIT() {
             build(":assemble", options = testSourceSetsDependingOnMainParameterOption) {
                 assertTestSourceSetsDependingOnMainParameter()
                 assertSuccessful()
+                assertTasksUpToDate(":commonizeNativeDistribution")
                 assertContains("Native Distribution Commonization: Cache hit")
                 assertTasksUpToDate(":commonizeCInterop")
             }

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt

@@ -129,7 +129,7 @@ internal val Project.cleanNativeDistributionCommonizerTask: TaskProvider<Delete>
             configureTask = {
                 group = "interop"
                 description = "Deletes all previously commonized klib's from the Kotlin/Native distribution"
-                delete(commonizeNativeDistributionTask.map { it.getRootOutputDirectory() })
+                delete(commonizeNativeDistributionTask.map { it.rootOutputDirectory })
             }
         )
     }

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt

@@ -81,7 +81,7 @@ private fun Project.getOriginalPlatformLibrariesFor(target: LeafCommonizerTarget
 }
 
 private fun NativeDistributionCommonizerTask.getCommonizedPlatformLibrariesFor(target: SharedCommonizerTarget): FileCollection {
-    val targetOutputDirectory = CommonizerOutputFileLayout.resolveCommonizedDirectory(getRootOutputDirectory(), target)
+    val targetOutputDirectory = CommonizerOutputFileLayout.resolveCommonizedDirectory(rootOutputDirectory, target)
     return project.filesProvider { targetOutputDirectory.listLibraryFiles() }.builtBy(this)
 }
 

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizationCache.kt

@@ -7,59 +7,77 @@
 
 package org.jetbrains.kotlin.gradle.targets.native.internal
 
-import org.gradle.api.Project
+import org.gradle.api.logging.Logger
 import org.jetbrains.kotlin.commonizer.*
 import org.jetbrains.kotlin.commonizer.CommonizerOutputFileLayout.resolveCommonizedDirectory
-import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider
 import java.io.File
 import java.io.FileOutputStream
-
+import java.util.concurrent.locks.ReentrantLock
-internal val Project.isNativeDistributionCommonizationCacheEnabled: Boolean
+import kotlin.concurrent.withLock
-    get() = PropertiesProvider(this).enableNativeDistributionCommonizationCache
 
 internal class NativeDistributionCommonizationCache(
-    private val project: Project,
+    private val logger: Logger,
+    private val isCachingEnabled: Boolean,
     private val commonizer: NativeDistributionCommonizer
 ) : NativeDistributionCommonizer {
 
+    fun isUpToDate(
+        konanHome: File, outputDirectory: File, outputTargets: Set<SharedCommonizerTarget>
+    ): Boolean = lock.withLock(outputDirectory) {
+        todoTargets(konanHome, outputDirectory, outputTargets)
+    }.isEmpty()
+
     override fun commonizeNativeDistribution(
         konanHome: File,
         outputDirectory: File,
         outputTargets: Set<SharedCommonizerTarget>,
         logLevel: CommonizerLogLevel,
         additionalSettings: List<AdditionalCommonizerSetting<*>>,
-    ) {
+    ): Unit = lock.withLock(outputDirectory) {
-        if (!project.isNativeDistributionCommonizationCacheEnabled) {
+        val todoOutputTargets = todoTargets(konanHome, outputDirectory, outputTargets)
+        if (todoOutputTargets.isEmpty()) return@withLock
+
+        /* Invoke commonizer with only 'to do' targets */
+        commonizer.commonizeNativeDistribution(
+            konanHome, outputDirectory, todoOutputTargets, logLevel, additionalSettings
+        )
+
+        /* Mark targets as successfully commonized */
+        todoOutputTargets
+            .map { outputTarget -> resolveCommonizedDirectory(outputDirectory, outputTarget) }
+            .filter { commonizedDirectory -> commonizedDirectory.isDirectory }
+            .forEach { commonizedDirectory -> commonizedDirectory.resolve(".success").createNewFile() }
+    }
+
+    private fun todoTargets(
+        konanHome: File, outputDirectory: File, outputTargets: Set<SharedCommonizerTarget>
+    ): Set<SharedCommonizerTarget> {
+        lock.checkLocked(outputDirectory)
+        logInfo("Calculating cache state for $outputTargets")
+
+        if (!isCachingEnabled) {
             logInfo("Cache disabled")
+            return if (isMissingPlatformLibraries(konanHome, outputTargets)) return emptySet()
+            else outputTargets
         }
 
-        withLock(outputDirectory) {
+        val cachedOutputTargets = outputTargets
-            val cachedOutputTargets = outputTargets
+            .filter { outputTarget -> isCached(resolveCommonizedDirectory(outputDirectory, outputTarget)) }
-                .filter { outputTarget -> isCached(resolveCommonizedDirectory(outputDirectory, outputTarget)) }
+            .onEach { outputTarget -> logInfo("Cache hit: $outputTarget already commonized") }
-                .onEach { outputTarget -> logInfo("Cache hit: $outputTarget already commonized") }
+            .toSet()
-                .toSet()
 
-            val enqueuedOutputTargets = if (project.isNativeDistributionCommonizationCacheEnabled) outputTargets - cachedOutputTargets
+        val todoOutputTargets = outputTargets - cachedOutputTargets
-            else outputTargets
 
-            if (canReturnFast(konanHome, enqueuedOutputTargets)) {
+        if (todoOutputTargets.isEmpty() || isMissingPlatformLibraries(konanHome, todoOutputTargets)) {
-                logInfo("All available targets are commonized already - Nothing to do")
+            logInfo("All available targets are commonized already - Nothing to do")
-                return
+            if (todoOutputTargets.isNotEmpty()) {
+                logInfo("Platforms cannot be commonized, because of missing platform libraries: $todoOutputTargets")
             }
 
-            enqueuedOutputTargets
+            return emptySet()
-                .map { outputTarget -> resolveCommonizedDirectory(outputDirectory, outputTarget) }
-                .forEach { commonizedDirectory -> if (commonizedDirectory.exists()) commonizedDirectory.deleteRecursively() }
-
-            commonizer.commonizeNativeDistribution(
-                konanHome, outputDirectory, enqueuedOutputTargets, logLevel, additionalSettings
-            )
-
-            enqueuedOutputTargets
-                .map { outputTarget -> resolveCommonizedDirectory(outputDirectory, outputTarget) }
-                .filter { commonizedDirectory -> commonizedDirectory.isDirectory }
-                .forEach { commonizedDirectory -> commonizedDirectory.resolve(".success").createNewFile() }
         }
+
+        return todoOutputTargets
     }
 
     private fun isCached(directory: File): Boolean {
@@ -67,11 +85,9 @@ internal class NativeDistributionCommonizationCache(
         return successMarkerFile.isFile
     }
 
-    private fun canReturnFast(
+    private fun isMissingPlatformLibraries(
         konanHome: File, missingOutputTargets: Set<CommonizerTarget>
     ): Boolean {
-        if (missingOutputTargets.isEmpty()) return true
-
         // If all platform lib dirs are missing, we can also return fast from the cache without invoking
         //  the commonizer
         return missingOutputTargets.allLeaves()
@@ -80,27 +96,53 @@ internal class NativeDistributionCommonizationCache(
             .none { platformLibsDir -> platformLibsDir.exists() }
     }
 
-    private inline fun <T> withLock(outputDirectory: File, action: () -> T): T {
+    /**
-        outputDirectory.mkdirs()
+     * Re-entrant lock implementation capable of locking a given output directory
-        val lockfile = outputDirectory.resolve(".lock")
+     * even between multiple process (Gradle Daemons)
-        logInfo("Acquire lock: ${lockfile.path} ...")
+     */
-        FileOutputStream(outputDirectory.resolve(".lock")).use { stream ->
+    private val lock = object {
-            val lock = stream.channel.lock()
+        private val reentrantLock = ReentrantLock()
-            assert(lock.isValid)
+        private val lockedOutputDirectories = mutableSetOf<File>()
-            return try {
+
-                logInfo("Lock acquired: ${lockfile.path}")
+        fun <T> withLock(outputDirectory: File, action: () -> T): T {
-                action()
+            /* Enter intra-process wide lock */
-            } finally {
+            reentrantLock.withLock {
-                lock.release()
+                if (outputDirectory in lockedOutputDirectories) {
-                logInfo("Lock released: ${lockfile.path}")
+                    /* Already acquired this directory and re-entered: We can just execute the action */
+                    return action()
+                }
+
+                /* Lock output directory inter-process wide */
+                outputDirectory.mkdirs()
+                val lockfile = outputDirectory.resolve(".lock")
+                logInfo("Acquire lock: ${lockfile.path} ...")
+                FileOutputStream(outputDirectory.resolve(".lock")).use { stream ->
+                    val lock = stream.channel.lock()
+                    assert(lock.isValid)
+                    return try {
+                        logInfo("Lock acquired: ${lockfile.path}")
+                        lockedOutputDirectories.add(outputDirectory)
+                        action()
+                    } finally {
+                        lockedOutputDirectories.remove(outputDirectory)
+                        lock.release()
+                        logInfo("Lock released: ${lockfile.path}")
+                    }
+                }
+            }
+        }
+
+        fun checkLocked(outputDirectory: File) {
+            check(reentrantLock.isHeldByCurrentThread) {
+                "Expected lock to be held by current thread ${Thread.currentThread().name}"
+            }
+
+            check(outputDirectory in lockedOutputDirectories) {
+                "Expected $outputDirectory to be locked. Locked directories: $lockedOutputDirectories"
             }
         }
     }
 
-    private fun logInfo(message: String) = project.logger.info("${Logging.prefix}: $message")
+    private fun logInfo(message: String) =
-
+        logger.info("Native Distribution Commonization: $message")
-    private object Logging {
-        const val prefix = "Native Distribution Commonization"
-    }
 }
-

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt

@@ -7,7 +7,6 @@ package org.jetbrains.kotlin.gradle.targets.native.internal
 
 import org.gradle.api.DefaultTask
 import org.gradle.api.Project
-import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.TaskAction
 import org.jetbrains.kotlin.commonizer.SharedCommonizerTarget
@@ -16,6 +15,7 @@ import org.jetbrains.kotlin.compilerRunner.KotlinNativeCommonizerToolRunner
 import org.jetbrains.kotlin.compilerRunner.konanHome
 import org.jetbrains.kotlin.compilerRunner.registerCommonizerClasspathConfigurationIfNecessary
 import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull
+import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider
 import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion
 import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMONIZED_LIBS_DIR
 import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_KLIB_DIR
@@ -26,34 +26,32 @@ internal open class NativeDistributionCommonizerTask : DefaultTask() {
 
     private val konanHome = project.file(project.konanHome)
 
-    @get:Input
+    private val commonizerTargets: Set<SharedCommonizerTarget> by lazy {
-    internal val commonizerTargets: Set<SharedCommonizerTarget>
+        project.collectAllSharedCommonizerTargetsFromBuild()
-        get() = project.collectAllSharedCommonizerTargetsFromBuild()
+    }
+
+    private val commonizer by lazy {
+        NativeDistributionCommonizationCache(
+            logger = project.logger,
+            isCachingEnabled = project.kotlinPropertiesProvider.enableNativeDistributionCommonizationCache,
+            commonizer = GradleCliCommonizer(KotlinNativeCommonizerToolRunner(project))
+        )
+    }
 
     @get:Internal
-    internal val commonizerRunner = KotlinNativeCommonizerToolRunner(project)
+    internal val rootOutputDirectory: File by lazy {
-
+        project.file(konanHome)
-    @get:Input
-    @Suppress("unused") // Only for up-to-date checker.
-    internal val commonizerJvmArgs: List<String>
-        get() = commonizerRunner.getCustomJvmArgs()
-
-    @Internal
-    internal fun getRootOutputDirectory(): File {
-        val kotlinVersion = project.getKotlinPluginVersion()
-
-        return project.file(konanHome)
             .resolve(KONAN_DISTRIBUTION_KLIB_DIR)
             .resolve(KONAN_DISTRIBUTION_COMMONIZED_LIBS_DIR)
-            .resolve(urlEncode(kotlinVersion))
+            .resolve(URLEncoder.encode(project.getKotlinPluginVersion(), Charsets.UTF_8.name()))
     }
 
     @TaskAction
     protected fun run() {
-        NativeDistributionCommonizationCache(project, GradleCliCommonizer(commonizerRunner)).commonizeNativeDistribution(
+        commonizer.commonizeNativeDistribution(
             konanHome = konanHome,
-            outputDirectory = getRootOutputDirectory(),
+            outputDirectory = rootOutputDirectory,
-            outputTargets = project.collectAllSharedCommonizerTargetsFromBuild(),
+            outputTargets = commonizerTargets,
             logLevel = project.commonizerLogLevel,
             additionalSettings = project.additionalCommonizerSettings,
         )
@@ -61,6 +59,9 @@ internal open class NativeDistributionCommonizerTask : DefaultTask() {
 
     init {
         project.registerCommonizerClasspathConfigurationIfNecessary()
+        outputs.upToDateWhen {
+            commonizer.isUpToDate(konanHome, rootOutputDirectory, commonizerTargets)
+        }
     }
 }
 
@@ -74,5 +75,3 @@ private fun Project.collectAllSharedCommonizerTargetsFromProject(): Set<SharedCo
         .filterIsInstance<SharedCommonizerTarget>()
         .toSet()
 }
-
-private fun urlEncode(value: String): String = URLEncoder.encode(value, Charsets.UTF_8.name())