From 178b5db8dee954171a0964aaac21de3ef00e45c5 Mon Sep 17 00:00:00 2001 From: Dmitriy Dolovov Date: Fri, 10 Jul 2020 22:13:47 +0700 Subject: [PATCH] [Commonizer] More precise up-to-date checks in Gradle task Don't annotate the whole 'distributionDir' with @InputDirectory. The contents of the distribution directory may grow over time due to compilation cache and new commonized libraries. Don't annotate 'baseDestinationDir' with @InputDirectory. The base destination directory may contain numerous subdirectories with the libraries produced for different combinations of commonized targets. This means that the contents of the base directory will grow over time. Use more precise directory up-to-dateness checks: * Annotate the directory with the original common libraries (i.e. 'stdlib') as @InputDirectory. * Annotate the directory with the original platform libraries (all targets altogether) as @InputDirectory. * For each combination of commonized targets mark the list of the directories with resulting libraries as @OutputDirectories. ^KT-40120 --- .../targets/native/internal/CommonizerTask.kt | 175 ++++++++++++------ .../KotlinNativePlatformDependencies.kt | 20 +- 2 files changed, 130 insertions(+), 65 deletions(-) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTask.kt index b9b661912ab..872f441f63f 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CommonizerTask.kt @@ -9,8 +9,9 @@ import org.gradle.api.DefaultTask import org.gradle.api.Project import org.gradle.api.tasks.* import org.jetbrains.kotlin.compilerRunner.KotlinNativeKlibCommonizerToolRunner -import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMONIZED_LIBS_DIR +import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_COMMON_LIBS_DIR import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_KLIB_DIR +import org.jetbrains.kotlin.konan.library.KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR import org.jetbrains.kotlin.konan.target.KonanTarget import java.io.File import java.io.IOException @@ -21,84 +22,152 @@ import java.time.* import java.util.* import javax.inject.Inject +/** + * Note: Using [resultingLibsDirs] isn't the safest option for up-to-date checker, as in multi-project build + * this may cause re-running the commonizer for the same groups several times. Hopefully, the commonizer has + * inner up-to-date check that prevents doing extra work. + */ +internal data class CommonizerSubtaskParams( + // The ordered list of unique targets. + @get:Input val orderedTargetNames: List, + + // Only for up-to-date checker. The directories with the resulting libs + // (common first, then platforms in the same order as in 'orderedTargetNames'). + @get:OutputDirectories val resultingLibsDirs: List, + + @get:Internal val destinationDir: File +) + internal data class CommonizerTaskParams( - @get:InputDirectory val distributionDir: File, - @get:InputDirectory val baseDestinationDir: File, - @Internal val targetGroups: List>, - @get:Input val kotlinVersion: String + @get:Input val kotlinVersion: String, + + // Only for up-to-date checker. The directory with the original common libs. + @get:InputDirectory val originalCommonLibsDir: File, + + // Only for up-to-date checker. The directory with the original platform libs. + @get:InputDirectory val originalPlatformLibsDir: File, + + @get:Internal val baseDestinationDir: File, + + @get:Nested val subtasks: List ) { - @Internal - val commandLineArguments = mutableListOf() + @get:Internal + lateinit var commandLineArguments: List - @Internal - val successPostActions = mutableListOf<() -> Unit>() + @get:Internal + lateinit var successPostActions: List<() -> Unit> - @Internal - val failurePostActions = mutableListOf<() -> Unit>() + @get:Internal + lateinit var failurePostActions: List<() -> Unit> - // It isn't best option for "up-to-date" checker (for multi project build it will be started few times) - // but commonizer has inner check - @get:OutputDirectories - val destinationDirs: List + companion object { + fun build( + kotlinVersion: String, + targetGroups: List>, + distributionDir: File, + baseDestinationDir: File + ): CommonizerTaskParams { + val distributionLibsDir = distributionDir.resolve(KONAN_DISTRIBUTION_KLIB_DIR) - // need stable order of targets for consistency - private val orderedTargetGroups = targetGroups.map { it.sortedBy { target -> target.name } } + val commandLineArguments = mutableListOf() + val successPostActions = mutableListOf<() -> Unit>() + val failurePostActions = mutableListOf<() -> Unit>() - @Input - fun getTargetNames() = orderedTargetGroups.map { it.map { target -> target.name } } - - init { - destinationDirs = orderedTargetGroups.map { orderedTargets -> - if (orderedTargets.size == 1) { - // no need to commonize, just use the libraries from the distribution - distributionDir.resolve(KONAN_DISTRIBUTION_KLIB_DIR) - } else { - val discriminator = buildString { - orderedTargets.joinTo(this, separator = "-") - append("-") - append(kotlinVersion.toLowerCase().base64) - } - - val destinationDir = baseDestinationDir.resolve(discriminator) - if (!destinationDir.isDirectory) { - val parentDir = destinationDir.parentFile - parentDir.mkdirs() - - val destinationTmpDir = createTempDir( - prefix = "tmp-" + destinationDir.name, - suffix = ".new", - directory = parentDir + val subtasks = targetGroups.map { targets -> + val orderedTargetNames = targets.map { it.name }.sorted() + if (orderedTargetNames.size == 1) { + // no need to commonize, just use the libraries from the distribution + buildSubtask( + destinationDir = distributionLibsDir, + orderedTargetNames = orderedTargetNames ) + } else { + val discriminator = buildString { + orderedTargetNames.joinTo(this, separator = "-") + append("-") + append(kotlinVersion.toLowerCase().base64) + } - commandLineArguments += "native-dist-commonize" - commandLineArguments += "-distribution-path" - commandLineArguments += distributionDir.toString() - commandLineArguments += "-output-path" - commandLineArguments += destinationTmpDir.toString() - commandLineArguments += "-targets" - commandLineArguments += orderedTargets.joinToString(separator = ",") + val destinationDir = baseDestinationDir.resolve(discriminator) + if (!destinationDir.isDirectory) { + val parentDir = destinationDir.parentFile + parentDir.mkdirs() - successPostActions.add { renameDirectory(destinationTmpDir, destinationDir) } - failurePostActions.add { renameToTempAndDelete(destinationTmpDir) } + val destinationTmpDir = createTempDir( + prefix = "tmp-" + destinationDir.name, + suffix = ".new", + directory = parentDir + ) + + commandLineArguments += "native-dist-commonize" + commandLineArguments += "-distribution-path" + commandLineArguments += distributionDir.toString() + commandLineArguments += "-output-path" + commandLineArguments += destinationTmpDir.toString() + commandLineArguments += "-targets" + commandLineArguments += orderedTargetNames.joinToString(separator = ",") + + successPostActions.add { renameDirectory(destinationTmpDir, destinationDir) } + failurePostActions.add { renameToTempAndDelete(destinationTmpDir) } + } + + buildSubtask( + destinationDir = destinationDir, + orderedTargetNames = orderedTargetNames + ) } + } - destinationDir + return CommonizerTaskParams( + kotlinVersion = kotlinVersion, + originalCommonLibsDir = commonLibsDir(distributionLibsDir), + originalPlatformLibsDir = platformLibsDir(distributionLibsDir), + baseDestinationDir = baseDestinationDir, + subtasks = subtasks + ).also { + it.commandLineArguments = commandLineArguments + it.successPostActions = successPostActions + it.failurePostActions = failurePostActions } } + + private fun commonLibsDir(baseDir: File): File = baseDir.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR) + private fun platformLibsDir(baseDir: File): File = baseDir.resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR) + + private fun platformLibsDirs(baseDir: File, orderedTargetNames: List): List { + val platformLibsDir = platformLibsDir(baseDir) + return orderedTargetNames.map(platformLibsDir::resolve) + } + + private fun resultingLibsDirs(baseDir: File, orderedTargetNames: List): List { + return mutableListOf().apply { + this += commonLibsDir(baseDir) + this += platformLibsDirs(baseDir, orderedTargetNames) + } + } + + private fun buildSubtask(destinationDir: File, orderedTargetNames: List) = CommonizerSubtaskParams( + orderedTargetNames = orderedTargetNames, + resultingLibsDirs = resultingLibsDirs(destinationDir, orderedTargetNames), + destinationDir = destinationDir + ) } } internal const val COMMONIZER_TASK_NAME = "runCommonizer" internal open class CommonizerTask @Inject constructor( - @Nested val params: CommonizerTaskParams + @get:Nested val params: CommonizerTaskParams ) : DefaultTask() { @TaskAction fun run() { // first of all remove directories with unused commonized libraries plus temporary directories with commonized libraries // that accidentally were not cleaned up before - cleanUp(params.baseDestinationDir, excludedDirectories = params.destinationDirs) + cleanUp( + baseDirectory = params.baseDestinationDir, + excludedDirectories = params.subtasks.map { it.destinationDir } + ) try { callCommonizerCLI(project, params.commandLineArguments) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt index 52eb564552a..3040e9c184b 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/KotlinNativePlatformDependencies.kt @@ -104,16 +104,13 @@ private class NativePlatformDependencyResolver(val project: Project, val kotlinV check(!alreadyResolved) alreadyResolved = true - val targetGroups: List>> = - dependencies.keys - .filterIsInstance() - .map { it to it.targets } + val targetGroups: List = dependencies.keys.filterIsInstance() - val commonizerTaskParams = CommonizerTaskParams( + val commonizerTaskParams = CommonizerTaskParams.build( + kotlinVersion, + targetGroups.map { it.targets }, distributionDir, - distributionDir.resolve(KONAN_DISTRIBUTION_KLIB_DIR).resolve(KONAN_DISTRIBUTION_COMMONIZED_LIBS_DIR), - targetGroups.map { it.second }, - kotlinVersion + distributionDir.resolve(KONAN_DISTRIBUTION_KLIB_DIR).resolve(KONAN_DISTRIBUTION_COMMONIZED_LIBS_DIR) ) val commonizerTaskProvider = project.registerTask( @@ -122,10 +119,9 @@ private class NativePlatformDependencyResolver(val project: Project, val kotlinV listOf(commonizerTaskParams) ) {} - val commonizedLibsDirs = - commonizerTaskParams.destinationDirs - .mapIndexed { index: Int, commonizedLibsDir: File -> targetGroups[index].first to commonizedLibsDir } - .toMap() + val commonizedLibsDirs: Map = commonizerTaskParams.subtasks.mapIndexed { index, subtask -> + targetGroups[index] to subtask.destinationDir + }.toMap() // then, resolve dependencies one by one dependencies.forEach { (dependency, actions) ->