From 3c8a4c5896a71e4095acb207941189a625070c48 Mon Sep 17 00:00:00 2001 From: "sebastian.sellmair" Date: Wed, 8 Jun 2022 15:50:39 +0200 Subject: [PATCH] [Gradle][MPP] Mark 'commonizeNativeDistribution' as UP-TO-DATE when cache hits ^KT-52632 Verification Pending --- .../jetbrains/kotlin/gradle/CommonizerIT.kt | 3 +- .../native/internal/CommonizerTasks.kt | 2 +- .../KotlinNativePlatformDependencies.kt | 2 +- .../NativeDistributionCommonizationCache.kt | 146 +++++++++++------- .../NativeDistributionCommonizerTask.kt | 43 +++--- 5 files changed, 119 insertions(+), 77 deletions(-) diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/CommonizerIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/CommonizerIT.kt index 0ab99b539e0..b0c8fe192e1 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/CommonizerIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/CommonizerIT.kt @@ -39,7 +39,7 @@ class CommonizerIT : BaseGradleIT() { } build("commonize", "-Pkotlin.mpp.enableNativeDistributionCommonizationCache=true") { - assertTasksExecuted(":commonizeNativeDistribution") + assertTasksUpToDate(":commonizeNativeDistribution") assertNativeDistributionCommonizationCacheHit() assertContains("Native Distribution Commonization: All available targets are commonized already") assertContains("Native Distribution Commonization: Lock acquired") @@ -552,6 +552,7 @@ class CommonizerIT : BaseGradleIT() { build(":assemble", options = testSourceSetsDependingOnMainParameterOption) { assertTestSourceSetsDependingOnMainParameter() assertSuccessful() + assertTasksUpToDate(":commonizeNativeDistribution") assertContains("Native Distribution Commonization: Cache hit") assertTasksUpToDate(":commonizeCInterop") } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt index 3ee60561b51..dbc286e7206 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTasks.kt @@ -129,7 +129,7 @@ internal val Project.cleanNativeDistributionCommonizerTask: TaskProvider configureTask = { group = "interop" description = "Deletes all previously commonized klib's from the Kotlin/Native distribution" - delete(commonizeNativeDistributionTask.map { it.getRootOutputDirectory() }) + delete(commonizeNativeDistributionTask.map { it.rootOutputDirectory }) } ) } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt index c32a762d814..2cd1bfdfb8b 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt @@ -81,7 +81,7 @@ private fun Project.getOriginalPlatformLibrariesFor(target: LeafCommonizerTarget } private fun NativeDistributionCommonizerTask.getCommonizedPlatformLibrariesFor(target: SharedCommonizerTarget): FileCollection { - val targetOutputDirectory = CommonizerOutputFileLayout.resolveCommonizedDirectory(getRootOutputDirectory(), target) + val targetOutputDirectory = CommonizerOutputFileLayout.resolveCommonizedDirectory(rootOutputDirectory, target) return project.filesProvider { targetOutputDirectory.listLibraryFiles() }.builtBy(this) } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizationCache.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizationCache.kt index a55d4cf1ce6..9bc5f7a75d1 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizationCache.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizationCache.kt @@ -7,59 +7,77 @@ package org.jetbrains.kotlin.gradle.targets.native.internal -import org.gradle.api.Project +import org.gradle.api.logging.Logger import org.jetbrains.kotlin.commonizer.* import org.jetbrains.kotlin.commonizer.CommonizerOutputFileLayout.resolveCommonizedDirectory -import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider import java.io.File import java.io.FileOutputStream - -internal val Project.isNativeDistributionCommonizationCacheEnabled: Boolean - get() = PropertiesProvider(this).enableNativeDistributionCommonizationCache +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock internal class NativeDistributionCommonizationCache( - private val project: Project, + private val logger: Logger, + private val isCachingEnabled: Boolean, private val commonizer: NativeDistributionCommonizer ) : NativeDistributionCommonizer { + fun isUpToDate( + konanHome: File, outputDirectory: File, outputTargets: Set + ): Boolean = lock.withLock(outputDirectory) { + todoTargets(konanHome, outputDirectory, outputTargets) + }.isEmpty() + override fun commonizeNativeDistribution( konanHome: File, outputDirectory: File, outputTargets: Set, logLevel: CommonizerLogLevel, additionalSettings: List>, - ) { - if (!project.isNativeDistributionCommonizationCacheEnabled) { + ): Unit = lock.withLock(outputDirectory) { + val todoOutputTargets = todoTargets(konanHome, outputDirectory, outputTargets) + if (todoOutputTargets.isEmpty()) return@withLock + + /* Invoke commonizer with only 'to do' targets */ + commonizer.commonizeNativeDistribution( + konanHome, outputDirectory, todoOutputTargets, logLevel, additionalSettings + ) + + /* Mark targets as successfully commonized */ + todoOutputTargets + .map { outputTarget -> resolveCommonizedDirectory(outputDirectory, outputTarget) } + .filter { commonizedDirectory -> commonizedDirectory.isDirectory } + .forEach { commonizedDirectory -> commonizedDirectory.resolve(".success").createNewFile() } + } + + private fun todoTargets( + konanHome: File, outputDirectory: File, outputTargets: Set + ): Set { + lock.checkLocked(outputDirectory) + logInfo("Calculating cache state for $outputTargets") + + if (!isCachingEnabled) { logInfo("Cache disabled") + return if (isMissingPlatformLibraries(konanHome, outputTargets)) return emptySet() + else outputTargets } - withLock(outputDirectory) { - val cachedOutputTargets = outputTargets - .filter { outputTarget -> isCached(resolveCommonizedDirectory(outputDirectory, outputTarget)) } - .onEach { outputTarget -> logInfo("Cache hit: $outputTarget already commonized") } - .toSet() + val cachedOutputTargets = outputTargets + .filter { outputTarget -> isCached(resolveCommonizedDirectory(outputDirectory, outputTarget)) } + .onEach { outputTarget -> logInfo("Cache hit: $outputTarget already commonized") } + .toSet() - val enqueuedOutputTargets = if (project.isNativeDistributionCommonizationCacheEnabled) outputTargets - cachedOutputTargets - else outputTargets + val todoOutputTargets = outputTargets - cachedOutputTargets - if (canReturnFast(konanHome, enqueuedOutputTargets)) { - logInfo("All available targets are commonized already - Nothing to do") - return + if (todoOutputTargets.isEmpty() || isMissingPlatformLibraries(konanHome, todoOutputTargets)) { + logInfo("All available targets are commonized already - Nothing to do") + if (todoOutputTargets.isNotEmpty()) { + logInfo("Platforms cannot be commonized, because of missing platform libraries: $todoOutputTargets") } - enqueuedOutputTargets - .map { outputTarget -> resolveCommonizedDirectory(outputDirectory, outputTarget) } - .forEach { commonizedDirectory -> if (commonizedDirectory.exists()) commonizedDirectory.deleteRecursively() } - - commonizer.commonizeNativeDistribution( - konanHome, outputDirectory, enqueuedOutputTargets, logLevel, additionalSettings - ) - - enqueuedOutputTargets - .map { outputTarget -> resolveCommonizedDirectory(outputDirectory, outputTarget) } - .filter { commonizedDirectory -> commonizedDirectory.isDirectory } - .forEach { commonizedDirectory -> commonizedDirectory.resolve(".success").createNewFile() } + return emptySet() } + + return todoOutputTargets } private fun isCached(directory: File): Boolean { @@ -67,11 +85,9 @@ internal class NativeDistributionCommonizationCache( return successMarkerFile.isFile } - private fun canReturnFast( + private fun isMissingPlatformLibraries( konanHome: File, missingOutputTargets: Set ): Boolean { - if (missingOutputTargets.isEmpty()) return true - // If all platform lib dirs are missing, we can also return fast from the cache without invoking // the commonizer return missingOutputTargets.allLeaves() @@ -80,27 +96,53 @@ internal class NativeDistributionCommonizationCache( .none { platformLibsDir -> platformLibsDir.exists() } } - private inline fun withLock(outputDirectory: File, action: () -> T): T { - outputDirectory.mkdirs() - val lockfile = outputDirectory.resolve(".lock") - logInfo("Acquire lock: ${lockfile.path} ...") - FileOutputStream(outputDirectory.resolve(".lock")).use { stream -> - val lock = stream.channel.lock() - assert(lock.isValid) - return try { - logInfo("Lock acquired: ${lockfile.path}") - action() - } finally { - lock.release() - logInfo("Lock released: ${lockfile.path}") + /** + * Re-entrant lock implementation capable of locking a given output directory + * even between multiple process (Gradle Daemons) + */ + private val lock = object { + private val reentrantLock = ReentrantLock() + private val lockedOutputDirectories = mutableSetOf() + + fun withLock(outputDirectory: File, action: () -> T): T { + /* Enter intra-process wide lock */ + reentrantLock.withLock { + if (outputDirectory in lockedOutputDirectories) { + /* Already acquired this directory and re-entered: We can just execute the action */ + return action() + } + + /* Lock output directory inter-process wide */ + outputDirectory.mkdirs() + val lockfile = outputDirectory.resolve(".lock") + logInfo("Acquire lock: ${lockfile.path} ...") + FileOutputStream(outputDirectory.resolve(".lock")).use { stream -> + val lock = stream.channel.lock() + assert(lock.isValid) + return try { + logInfo("Lock acquired: ${lockfile.path}") + lockedOutputDirectories.add(outputDirectory) + action() + } finally { + lockedOutputDirectories.remove(outputDirectory) + lock.release() + logInfo("Lock released: ${lockfile.path}") + } + } + } + } + + fun checkLocked(outputDirectory: File) { + check(reentrantLock.isHeldByCurrentThread) { + "Expected lock to be held by current thread ${Thread.currentThread().name}" + } + + check(outputDirectory in lockedOutputDirectories) { + "Expected $outputDirectory to be locked. Locked directories: $lockedOutputDirectories" } } } - private fun logInfo(message: String) = project.logger.info("${Logging.prefix}: $message") - - private object Logging { - const val prefix = "Native Distribution Commonization" - } + private fun logInfo(message: String) = + logger.info("Native Distribution Commonization: $message") } - diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt index 416b346377f..b434cf3c464 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/NativeDistributionCommonizerTask.kt @@ -7,7 +7,6 @@ package org.jetbrains.kotlin.gradle.targets.native.internal import org.gradle.api.DefaultTask import org.gradle.api.Project -import org.gradle.api.tasks.Input import org.gradle.api.tasks.Internal import org.gradle.api.tasks.TaskAction import org.jetbrains.kotlin.commonizer.SharedCommonizerTarget @@ -16,6 +15,7 @@ import org.jetbrains.kotlin.compilerRunner.KotlinNativeCommonizerToolRunner import org.jetbrains.kotlin.compilerRunner.konanHome import org.jetbrains.kotlin.compilerRunner.registerCommonizerClasspathConfigurationIfNecessary import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull +import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMONIZED_LIBS_DIR import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_KLIB_DIR @@ -26,34 +26,32 @@ internal open class NativeDistributionCommonizerTask : DefaultTask() { private val konanHome = project.file(project.konanHome) - @get:Input - internal val commonizerTargets: Set - get() = project.collectAllSharedCommonizerTargetsFromBuild() + private val commonizerTargets: Set by lazy { + project.collectAllSharedCommonizerTargetsFromBuild() + } + + private val commonizer by lazy { + NativeDistributionCommonizationCache( + logger = project.logger, + isCachingEnabled = project.kotlinPropertiesProvider.enableNativeDistributionCommonizationCache, + commonizer = GradleCliCommonizer(KotlinNativeCommonizerToolRunner(project)) + ) + } @get:Internal - internal val commonizerRunner = KotlinNativeCommonizerToolRunner(project) - - @get:Input - @Suppress("unused") // Only for up-to-date checker. - internal val commonizerJvmArgs: List - get() = commonizerRunner.getCustomJvmArgs() - - @Internal - internal fun getRootOutputDirectory(): File { - val kotlinVersion = project.getKotlinPluginVersion() - - return project.file(konanHome) + internal val rootOutputDirectory: File by lazy { + project.file(konanHome) .resolve(KONAN_DISTRIBUTION_KLIB_DIR) .resolve(KONAN_DISTRIBUTION_COMMONIZED_LIBS_DIR) - .resolve(urlEncode(kotlinVersion)) + .resolve(URLEncoder.encode(project.getKotlinPluginVersion(), Charsets.UTF_8.name())) } @TaskAction protected fun run() { - NativeDistributionCommonizationCache(project, GradleCliCommonizer(commonizerRunner)).commonizeNativeDistribution( + commonizer.commonizeNativeDistribution( konanHome = konanHome, - outputDirectory = getRootOutputDirectory(), - outputTargets = project.collectAllSharedCommonizerTargetsFromBuild(), + outputDirectory = rootOutputDirectory, + outputTargets = commonizerTargets, logLevel = project.commonizerLogLevel, additionalSettings = project.additionalCommonizerSettings, ) @@ -61,6 +59,9 @@ internal open class NativeDistributionCommonizerTask : DefaultTask() { init { project.registerCommonizerClasspathConfigurationIfNecessary() + outputs.upToDateWhen { + commonizer.isUpToDate(konanHome, rootOutputDirectory, commonizerTargets) + } } } @@ -74,5 +75,3 @@ private fun Project.collectAllSharedCommonizerTargetsFromProject(): Set() .toSet() } - -private fun urlEncode(value: String): String = URLEncoder.encode(value, Charsets.UTF_8.name())