[Gradle][MPP] Mark 'commonizeNativeDistribution' as UP-TO-DATE when cache hits

^KT-52632 Verification Pending
2022-06-08 15:50:39 +02:00
parent 02601745a9
commit 3c8a4c5896
5 changed files with 119 additions and 77 deletions
@@ -39,7 +39,7 @@ class CommonizerIT : BaseGradleIT() {
            }

            build("commonize", "-Pkotlin.mpp.enableNativeDistributionCommonizationCache=true") {
-                assertTasksExecuted(":commonizeNativeDistribution")
+                assertTasksUpToDate(":commonizeNativeDistribution")
                assertNativeDistributionCommonizationCacheHit()
                assertContains("Native Distribution Commonization: All available targets are commonized already")
                assertContains("Native Distribution Commonization: Lock acquired")
@@ -552,6 +552,7 @@ class CommonizerIT : BaseGradleIT() {
            build(":assemble", options = testSourceSetsDependingOnMainParameterOption) {
                assertTestSourceSetsDependingOnMainParameter()
                assertSuccessful()
+                assertTasksUpToDate(":commonizeNativeDistribution")
                assertContains("Native Distribution Commonization: Cache hit")
                assertTasksUpToDate(":commonizeCInterop")
            }
@@ -129,7 +129,7 @@ internal val Project.cleanNativeDistributionCommonizerTask: TaskProvider<Delete>
            configureTask = {
                group = "interop"
                description = "Deletes all previously commonized klib's from the Kotlin/Native distribution"
-                delete(commonizeNativeDistributionTask.map { it.getRootOutputDirectory() })
+                delete(commonizeNativeDistributionTask.map { it.rootOutputDirectory })
            }
        )
    }
@@ -81,7 +81,7 @@ private fun Project.getOriginalPlatformLibrariesFor(target: LeafCommonizerTarget
 }

 private fun NativeDistributionCommonizerTask.getCommonizedPlatformLibrariesFor(target: SharedCommonizerTarget): FileCollection {
-    val targetOutputDirectory = CommonizerOutputFileLayout.resolveCommonizedDirectory(getRootOutputDirectory(), target)
+    val targetOutputDirectory = CommonizerOutputFileLayout.resolveCommonizedDirectory(rootOutputDirectory, target)
    return project.filesProvider { targetOutputDirectory.listLibraryFiles() }.builtBy(this)
 }

@@ -7,59 +7,77 @@

 package org.jetbrains.kotlin.gradle.targets.native.internal

-import org.gradle.api.Project
+import org.gradle.api.logging.Logger
 import org.jetbrains.kotlin.commonizer.*
 import org.jetbrains.kotlin.commonizer.CommonizerOutputFileLayout.resolveCommonizedDirectory
-import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider
 import java.io.File
 import java.io.FileOutputStream
-
-internal val Project.isNativeDistributionCommonizationCacheEnabled: Boolean
-    get() = PropertiesProvider(this).enableNativeDistributionCommonizationCache
+import java.util.concurrent.locks.ReentrantLock
+import kotlin.concurrent.withLock

 internal class NativeDistributionCommonizationCache(
-    private val project: Project,
+    private val logger: Logger,
+    private val isCachingEnabled: Boolean,
    private val commonizer: NativeDistributionCommonizer
 ) : NativeDistributionCommonizer {

+    fun isUpToDate(
+        konanHome: File, outputDirectory: File, outputTargets: Set<SharedCommonizerTarget>
+    ): Boolean = lock.withLock(outputDirectory) {
+        todoTargets(konanHome, outputDirectory, outputTargets)
+    }.isEmpty()
+
    override fun commonizeNativeDistribution(
        konanHome: File,
        outputDirectory: File,
        outputTargets: Set<SharedCommonizerTarget>,
        logLevel: CommonizerLogLevel,
        additionalSettings: List<AdditionalCommonizerSetting<*>>,
-    ) {
-        if (!project.isNativeDistributionCommonizationCacheEnabled) {
+    ): Unit = lock.withLock(outputDirectory) {
+        val todoOutputTargets = todoTargets(konanHome, outputDirectory, outputTargets)
+        if (todoOutputTargets.isEmpty()) return@withLock
+
+        /* Invoke commonizer with only 'to do' targets */
+        commonizer.commonizeNativeDistribution(
+            konanHome, outputDirectory, todoOutputTargets, logLevel, additionalSettings
+        )
+
+        /* Mark targets as successfully commonized */
+        todoOutputTargets
+            .map { outputTarget -> resolveCommonizedDirectory(outputDirectory, outputTarget) }
+            .filter { commonizedDirectory -> commonizedDirectory.isDirectory }
+            .forEach { commonizedDirectory -> commonizedDirectory.resolve(".success").createNewFile() }
+    }
+
+    private fun todoTargets(
+        konanHome: File, outputDirectory: File, outputTargets: Set<SharedCommonizerTarget>
+    ): Set<SharedCommonizerTarget> {
+        lock.checkLocked(outputDirectory)
+        logInfo("Calculating cache state for $outputTargets")
+
+        if (!isCachingEnabled) {
            logInfo("Cache disabled")
+            return if (isMissingPlatformLibraries(konanHome, outputTargets)) return emptySet()
+            else outputTargets
        }

-        withLock(outputDirectory) {
-            val cachedOutputTargets = outputTargets
-                .filter { outputTarget -> isCached(resolveCommonizedDirectory(outputDirectory, outputTarget)) }
-                .onEach { outputTarget -> logInfo("Cache hit: $outputTarget already commonized") }
-                .toSet()
+        val cachedOutputTargets = outputTargets
+            .filter { outputTarget -> isCached(resolveCommonizedDirectory(outputDirectory, outputTarget)) }
+            .onEach { outputTarget -> logInfo("Cache hit: $outputTarget already commonized") }
+            .toSet()

-            val enqueuedOutputTargets = if (project.isNativeDistributionCommonizationCacheEnabled) outputTargets - cachedOutputTargets
-            else outputTargets
+        val todoOutputTargets = outputTargets - cachedOutputTargets

-            if (canReturnFast(konanHome, enqueuedOutputTargets)) {
-                logInfo("All available targets are commonized already - Nothing to do")
-                return
+        if (todoOutputTargets.isEmpty() || isMissingPlatformLibraries(konanHome, todoOutputTargets)) {
+            logInfo("All available targets are commonized already - Nothing to do")
+            if (todoOutputTargets.isNotEmpty()) {
+                logInfo("Platforms cannot be commonized, because of missing platform libraries: $todoOutputTargets")
            }

-            enqueuedOutputTargets
-                .map { outputTarget -> resolveCommonizedDirectory(outputDirectory, outputTarget) }
-                .forEach { commonizedDirectory -> if (commonizedDirectory.exists()) commonizedDirectory.deleteRecursively() }
-
-            commonizer.commonizeNativeDistribution(
-                konanHome, outputDirectory, enqueuedOutputTargets, logLevel, additionalSettings
-            )
-
-            enqueuedOutputTargets
-                .map { outputTarget -> resolveCommonizedDirectory(outputDirectory, outputTarget) }
-                .filter { commonizedDirectory -> commonizedDirectory.isDirectory }
-                .forEach { commonizedDirectory -> commonizedDirectory.resolve(".success").createNewFile() }
+            return emptySet()
        }
+
+        return todoOutputTargets
    }

    private fun isCached(directory: File): Boolean {
@@ -67,11 +85,9 @@ internal class NativeDistributionCommonizationCache(
        return successMarkerFile.isFile
    }

-    private fun canReturnFast(
+    private fun isMissingPlatformLibraries(
        konanHome: File, missingOutputTargets: Set<CommonizerTarget>
    ): Boolean {
-        if (missingOutputTargets.isEmpty()) return true
-
        // If all platform lib dirs are missing, we can also return fast from the cache without invoking
        //  the commonizer
        return missingOutputTargets.allLeaves()
@@ -80,27 +96,53 @@ internal class NativeDistributionCommonizationCache(
            .none { platformLibsDir -> platformLibsDir.exists() }
    }

-    private inline fun <T> withLock(outputDirectory: File, action: () -> T): T {
-        outputDirectory.mkdirs()
-        val lockfile = outputDirectory.resolve(".lock")
-        logInfo("Acquire lock: ${lockfile.path} ...")
-        FileOutputStream(outputDirectory.resolve(".lock")).use { stream ->
-            val lock = stream.channel.lock()
-            assert(lock.isValid)
-            return try {
-                logInfo("Lock acquired: ${lockfile.path}")
-                action()
-            } finally {
-                lock.release()
-                logInfo("Lock released: ${lockfile.path}")
+    /**
+     * Re-entrant lock implementation capable of locking a given output directory
+     * even between multiple process (Gradle Daemons)
+     */
+    private val lock = object {
+        private val reentrantLock = ReentrantLock()
+        private val lockedOutputDirectories = mutableSetOf<File>()
+
+        fun <T> withLock(outputDirectory: File, action: () -> T): T {
+            /* Enter intra-process wide lock */
+            reentrantLock.withLock {
+                if (outputDirectory in lockedOutputDirectories) {
+                    /* Already acquired this directory and re-entered: We can just execute the action */
+                    return action()
+                }
+
+                /* Lock output directory inter-process wide */
+                outputDirectory.mkdirs()
+                val lockfile = outputDirectory.resolve(".lock")
+                logInfo("Acquire lock: ${lockfile.path} ...")
+                FileOutputStream(outputDirectory.resolve(".lock")).use { stream ->
+                    val lock = stream.channel.lock()
+                    assert(lock.isValid)
+                    return try {
+                        logInfo("Lock acquired: ${lockfile.path}")
+                        lockedOutputDirectories.add(outputDirectory)
+                        action()
+                    } finally {
+                        lockedOutputDirectories.remove(outputDirectory)
+                        lock.release()
+                        logInfo("Lock released: ${lockfile.path}")
+                    }
+                }
+            }
+        }
+
+        fun checkLocked(outputDirectory: File) {
+            check(reentrantLock.isHeldByCurrentThread) {
+                "Expected lock to be held by current thread ${Thread.currentThread().name}"
+            }
+
+            check(outputDirectory in lockedOutputDirectories) {
+                "Expected $outputDirectory to be locked. Locked directories: $lockedOutputDirectories"
            }
        }
    }

-    private fun logInfo(message: String) = project.logger.info("${Logging.prefix}: $message")
-
-    private object Logging {
-        const val prefix = "Native Distribution Commonization"
-    }
+    private fun logInfo(message: String) =
+        logger.info("Native Distribution Commonization: $message")
 }
-
@@ -7,7 +7,6 @@ package org.jetbrains.kotlin.gradle.targets.native.internal

 import org.gradle.api.DefaultTask
 import org.gradle.api.Project
-import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.TaskAction
 import org.jetbrains.kotlin.commonizer.SharedCommonizerTarget
@@ -16,6 +15,7 @@ import org.jetbrains.kotlin.compilerRunner.KotlinNativeCommonizerToolRunner
 import org.jetbrains.kotlin.compilerRunner.konanHome
 import org.jetbrains.kotlin.compilerRunner.registerCommonizerClasspathConfigurationIfNecessary
 import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull
+import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider
 import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion
 import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMONIZED_LIBS_DIR
 import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_KLIB_DIR
@@ -26,34 +26,32 @@ internal open class NativeDistributionCommonizerTask : DefaultTask() {

    private val konanHome = project.file(project.konanHome)

-    @get:Input
-    internal val commonizerTargets: Set<SharedCommonizerTarget>
-        get() = project.collectAllSharedCommonizerTargetsFromBuild()
+    private val commonizerTargets: Set<SharedCommonizerTarget> by lazy {
+        project.collectAllSharedCommonizerTargetsFromBuild()
+    }
+
+    private val commonizer by lazy {
+        NativeDistributionCommonizationCache(
+            logger = project.logger,
+            isCachingEnabled = project.kotlinPropertiesProvider.enableNativeDistributionCommonizationCache,
+            commonizer = GradleCliCommonizer(KotlinNativeCommonizerToolRunner(project))
+        )
+    }

    @get:Internal
-    internal val commonizerRunner = KotlinNativeCommonizerToolRunner(project)
-
-    @get:Input
-    @Suppress("unused") // Only for up-to-date checker.
-    internal val commonizerJvmArgs: List<String>
-        get() = commonizerRunner.getCustomJvmArgs()
-
-    @Internal
-    internal fun getRootOutputDirectory(): File {
-        val kotlinVersion = project.getKotlinPluginVersion()
-
-        return project.file(konanHome)
+    internal val rootOutputDirectory: File by lazy {
+        project.file(konanHome)
            .resolve(KONAN_DISTRIBUTION_KLIB_DIR)
            .resolve(KONAN_DISTRIBUTION_COMMONIZED_LIBS_DIR)
-            .resolve(urlEncode(kotlinVersion))
+            .resolve(URLEncoder.encode(project.getKotlinPluginVersion(), Charsets.UTF_8.name()))
    }

    @TaskAction
    protected fun run() {
-        NativeDistributionCommonizationCache(project, GradleCliCommonizer(commonizerRunner)).commonizeNativeDistribution(
+        commonizer.commonizeNativeDistribution(
            konanHome = konanHome,
-            outputDirectory = getRootOutputDirectory(),
-            outputTargets = project.collectAllSharedCommonizerTargetsFromBuild(),
+            outputDirectory = rootOutputDirectory,
+            outputTargets = commonizerTargets,
            logLevel = project.commonizerLogLevel,
            additionalSettings = project.additionalCommonizerSettings,
        )
@@ -61,6 +59,9 @@ internal open class NativeDistributionCommonizerTask : DefaultTask() {

    init {
        project.registerCommonizerClasspathConfigurationIfNecessary()
+        outputs.upToDateWhen {
+            commonizer.isUpToDate(konanHome, rootOutputDirectory, commonizerTargets)
+        }
    }
 }

@@ -74,5 +75,3 @@ private fun Project.collectAllSharedCommonizerTargetsFromProject(): Set<SharedCo
        .filterIsInstance<SharedCommonizerTarget>()
        .toSet()
 }
-
-private fun urlEncode(value: String): String = URLEncoder.encode(value, Charsets.UTF_8.name())