From 357e14dc04b13719fe04471b46c6254924d498cd Mon Sep 17 00:00:00 2001 From: Anton Lakotka Date: Thu, 12 Jan 2023 16:37:41 +0100 Subject: [PATCH] [Gradle] Add support Configuration Cache for MetadataDependencyTransformationTask * Write transformedLibraries to a file after Task execution * Write visible source sets to a file after Task execution * Rearrange task properties, inputs outputs in the way that task state is serializable to configuration cache. * Introduce transformDependenciesMetadata task to be an umbrella for other transform tasks. This will be convinient for tests and other development purposes ^KT-49933 --- .../mpp/GranularMetadataTransformation.kt | 9 +- .../mpp/KotlinMetadataLibraryDirectories.kt | 5 +- .../MetadataDependencyTransformationTask.kt | 181 +++++++++--------- ...KpmMetadataDependencyTransformationTask.kt | 6 +- .../plugin/mpp/transformMetadataLibraries.kt | 5 +- .../KotlinMetadataTargetConfigurator.kt | 6 +- .../targets/native/tasks/KotlinNativeTasks.kt | 8 +- 7 files changed, 116 insertions(+), 104 deletions(-) diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/GranularMetadataTransformation.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/GranularMetadataTransformation.kt index 18853912f5b..e1e85156e03 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/GranularMetadataTransformation.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/GranularMetadataTransformation.kt @@ -136,9 +136,12 @@ internal class GranularMetadataTransformation( val metadataDependencyResolutions: Iterable by lazy { doTransform() } - val visibleSourceSetsByComponentId: Map> = metadataDependencyResolutions - .filterIsInstance() - .associate { it.dependency.id.uniqueKey to it.allVisibleSourceSetNames } + val visibleSourceSetsByComponentId: Map> by lazy { + metadataDependencyResolutions + .filterIsInstance() + .groupBy { it.dependency.id.uniqueKey } + .mapValues { (_, visibleSourceSets) -> visibleSourceSets.flatMapTo(mutableSetOf()) { it.allVisibleSourceSetNames } } + } private fun doTransform(): Iterable { val result = mutableListOf() diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/KotlinMetadataLibraryDirectories.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/KotlinMetadataLibraryDirectories.kt index 66455291a83..483f800afec 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/KotlinMetadataLibraryDirectories.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/KotlinMetadataLibraryDirectories.kt @@ -6,13 +6,14 @@ package org.jetbrains.kotlin.gradle.plugin.mpp import org.gradle.api.Project +import org.gradle.api.file.ProjectLayout import java.io.File private const val kotlinTransformedMetadataLibraries = "kotlinTransformedMetadataLibraries" private const val kotlinTransformedCInteropMetadataLibraries = "kotlinTransformedCInteropMetadataLibraries" -internal fun Project.kotlinTransformedMetadataLibraryDirectoryForBuild(sourceSetName: String): File = - buildDir.resolve(kotlinTransformedMetadataLibraries).resolve(sourceSetName) +internal fun ProjectLayout.kotlinTransformedMetadataLibraryDirectoryForBuild(sourceSetName: String): File = + buildDirectory.get().asFile.resolve(kotlinTransformedMetadataLibraries).resolve(sourceSetName) internal val Project.kotlinTransformedMetadataLibraryDirectoryForIde: File get() = rootDir.resolve(".gradle").resolve("kotlin").resolve(kotlinTransformedMetadataLibraries) diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/MetadataDependencyTransformationTask.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/MetadataDependencyTransformationTask.kt index 8313bb7d9c3..c7f53c18f66 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/MetadataDependencyTransformationTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/MetadataDependencyTransformationTask.kt @@ -6,24 +6,22 @@ package org.jetbrains.kotlin.gradle.plugin.mpp import org.gradle.api.DefaultTask +import org.gradle.api.file.ConfigurableFileCollection import org.gradle.api.file.FileCollection +import org.gradle.api.file.ProjectLayout +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.model.ObjectFactory import org.gradle.api.provider.Provider import org.gradle.api.tasks.* import org.gradle.work.NormalizeLineEndings import org.jetbrains.kotlin.gradle.dsl.kotlinExtension -import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet -import org.jetbrains.kotlin.gradle.plugin.sources.KotlinDependencyScope import org.jetbrains.kotlin.gradle.plugin.sources.KotlinDependencyScope.* import org.jetbrains.kotlin.gradle.plugin.sources.internal -import org.jetbrains.kotlin.gradle.targets.metadata.ALL_COMPILE_METADATA_CONFIGURATION_NAME import org.jetbrains.kotlin.gradle.targets.metadata.KotlinMetadataTargetConfigurator -import org.jetbrains.kotlin.gradle.targets.metadata.ResolvedMetadataFilesProvider import org.jetbrains.kotlin.gradle.targets.metadata.dependsOnClosureWithInterCompilationDependencies -import org.jetbrains.kotlin.gradle.utils.filesProvider -import org.jetbrains.kotlin.gradle.utils.getValue -import org.jetbrains.kotlin.gradle.utils.notCompatibleWithConfigurationCacheCompat -import org.jetbrains.kotlin.gradle.utils.outputFilesProvider +import org.jetbrains.kotlin.gradle.tasks.locateTask +import org.jetbrains.kotlin.gradle.utils.* import java.io.File import javax.inject.Inject @@ -34,78 +32,103 @@ typealias TransformKotlinGranularMetadata = MetadataDependencyTransformationTask open class MetadataDependencyTransformationTask @Inject constructor( - @get:Internal - @field:Transient - val kotlinSourceSet: KotlinSourceSet + kotlinSourceSet: KotlinSourceSet, + private val objectFactory: ObjectFactory, + private val projectLayout: ProjectLayout ) : DefaultTask() { - init { - notCompatibleWithConfigurationCacheCompat( - "Task $name does not support Gradle Configuration Cache. Check KT-49933 for more info" - ) - } + //region Task Configuration State & Inputs + private val transformationParameters = GranularMetadataTransformation.Params(project, kotlinSourceSet) @get:OutputDirectory - val outputsDir: File by project.provider { - project.kotlinTransformedMetadataLibraryDirectoryForBuild(kotlinSourceSet.name) - } + val outputsDir: File get() = projectLayout.kotlinTransformedMetadataLibraryDirectoryForBuild(transformationParameters.sourceSetName) @Suppress("unused") // Gradle input @get:InputFiles @get:PathSensitive(PathSensitivity.RELATIVE) @get:IgnoreEmptyDirectories @get:NormalizeLineEndings - internal val configurationToResolve: FileCollection get() = kotlinSourceSet.internal.resolvableMetadataConfiguration + internal val configurationToResolve: FileCollection = kotlinSourceSet.internal.resolvableMetadataConfiguration - private val participatingSourceSets: Set - get() = kotlinSourceSet.internal.withDependsOnClosure.toMutableSet().apply { + @delegate:Transient // Only needed for configuring task inputs + private val participatingSourceSets: Set by lazy { + kotlinSourceSet.internal.withDependsOnClosure.toMutableSet().apply { if (any { it.name == KotlinSourceSet.COMMON_MAIN_SOURCE_SET_NAME }) add(project.kotlinExtension.sourceSets.getByName(KotlinSourceSet.COMMON_MAIN_SOURCE_SET_NAME)) } + } @Suppress("unused") // Gradle input @get:Input - internal val inputSourceSetsAndCompilations: Map> by project.provider { + internal val inputSourceSetsAndCompilations: Map> by lazy { participatingSourceSets.associate { sourceSet -> sourceSet.name to sourceSet.internal.compilations.map { it.name }.sorted() } } - private val participatingCompilations: Iterable> - get() = participatingSourceSets.flatMap { it.internal.compilations }.toSet() - @Suppress("unused") // Gradle input @get:Input - internal val inputCompilationDependencies: Map>> by project.provider { - participatingCompilations.associate { + internal val inputCompilationDependencies: Map>> by lazy { + participatingSourceSets.flatMap { it.internal.compilations }.associate { it.name to project.configurations.getByName(it.compileDependencyConfigurationName) .allDependencies.map { listOf(it.group, it.name, it.version) }.toSet() } } - private val parentVisibleSourceSets: List>> by lazy { - dependsOnClosureWithInterCompilationDependencies(kotlinSourceSet).map { - project.tasks.withType(MetadataDependencyTransformationTask::class.java) - .getByName(KotlinMetadataTargetConfigurator.transformGranularMetadataTaskName(it.name)) - .transformation - .visibleSourceSetsByComponentId + @get:OutputFile + val transformedLibrariesFileIndex: RegularFileProperty = objectFactory + .fileProperty() + .apply { set(outputsDir.resolve("${kotlinSourceSet.name}.transformedLibraries")) } + + @get:OutputFile + val visibleSourceSetsFile: RegularFileProperty = objectFactory + .fileProperty() + .apply { set(outputsDir.resolve("${kotlinSourceSet.name}.visibleSourceSets")) } + + @get:InputFiles + val parentVisibleSourceSetFiles: ConfigurableFileCollection = objectFactory + .fileCollection() + .from( + { + val parentSourceSets: List> = dependsOnClosureWithInterCompilationDependencies(kotlinSourceSet).mapNotNull { + project + .tasks + .locateTask(KotlinMetadataTargetConfigurator.transformGranularMetadataTaskName(it.name)) + ?.flatMap { it.visibleSourceSetsFile.map { it.asFile } } + } + + parentSourceSets + } + ) + + //endregion Task Configuration State & Inputs + + @TaskAction + fun transformMetadata() { + val transformation = GranularMetadataTransformation( + params = transformationParameters, + parentVisibleSourceSetsProvider = { parentVisibleSourceSetFiles.map(::readVisibleSourceSetsFile) } + ) + + if (outputsDir.isDirectory) { + outputsDir.deleteRecursively() } + outputsDir.mkdirs() + + val metadataDependencyResolutions = transformation.metadataDependencyResolutions + + metadataDependencyResolutions + .filterIsInstance() + .forEach { chooseVisibleSourceSets -> + objectFactory.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, true) + } + + writeTransformedLibraries(metadataDependencyResolutions) + writeVisibleSourceSets(transformation.visibleSourceSetsByComponentId) } - private val transformation = GranularMetadataTransformation( - params = GranularMetadataTransformation.Params(project, kotlinSourceSet), - parentVisibleSourceSetsProvider = { parentVisibleSourceSets } - ) - - @get:Internal - @delegate:Transient // exclude from Gradle instant execution state - internal val metadataDependencyResolutions: Iterable by project.provider { - transformation.metadataDependencyResolutions - } - - @get:Internal - internal val transformedLibraries: Provider> = project.provider { - metadataDependencyResolutions.flatMap { resolution -> + private fun writeTransformedLibraries(resolutions: Iterable) { + val fileList = resolutions.flatMap { resolution -> val files: Iterable = when (resolution) { is MetadataDependencyResolution.ChooseVisibleSourceSets -> when (val metadataProvider = resolution.metadataProvider) { is MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ArtifactMetadataProvider -> metadataProvider.read { content -> @@ -123,8 +146,7 @@ open class MetadataDependencyTransformationTask } } is MetadataDependencyResolution.Exclude -> emptySet() - is MetadataDependencyResolution.KeepOriginalDependency -> transformation - .params + is MetadataDependencyResolution.KeepOriginalDependency -> transformationParameters .resolvedMetadataConfiguration .componentArtifacts(resolution.dependency) .map { it.file } @@ -132,47 +154,28 @@ open class MetadataDependencyTransformationTask files } + + val content = fileList.joinToString("\n") + + transformedLibrariesFileIndex.get().asFile.writeText(content) } - /** - * Contains "actually transformed" metadata libraries (extracted from any [CompositeMetadataArtifact]) as well as - * [FileCollection]s pointing to project dependency klibs. - * - * Project dependencies are still required to be listed here (despite not being produced by this task), since - * this [filesByResolution] will be used to build the compile-classpath for metadata compilations. - */ - @get:Internal - internal val filesByResolution: Map - get() = metadataDependencyResolutions - .filterIsInstance() - .associateWith { chooseVisibleSourceSets -> - outputFilesProvider { - project.transformMetadataLibrariesForBuild( - chooseVisibleSourceSets, outputsDir, materializeFiles = false - ) - } - } - - @TaskAction - fun transformMetadata() { - if (outputsDir.isDirectory) { - outputsDir.deleteRecursively() + private fun writeVisibleSourceSets(visibleSourceSetsByComponentId: Map>) { + val content = visibleSourceSetsByComponentId.entries.joinToString("\n") { (id, visibleSourceSets) -> + "$id => ${visibleSourceSets.joinToString(",")}" } - outputsDir.mkdirs() - - metadataDependencyResolutions - .filterIsInstance() - .forEach { chooseVisibleSourceSets -> - project.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, true) - } + visibleSourceSetsFile.get().asFile.writeText(content) } -} -internal class SourceSetResolvedMetadataProvider( - taskProvider: TaskProvider -) : ResolvedMetadataFilesProvider { - override val buildDependencies: Iterable> = listOf(taskProvider) - override val metadataResolutions: Iterable by taskProvider.map { it.metadataDependencyResolutions } - override val metadataFilesByResolution: Map - by taskProvider.map { it.filesByResolution } -} + private fun readVisibleSourceSetsFile(file: File): Map> = file + .readLines() + .associate { string -> + val (id, visibleSourceSetsString) = string.split(" => ") + id to visibleSourceSetsString.split(",").toSet() + } + + @get:Internal // Warning! transformedLibraries is available only after Task Execution + val transformedLibraries: Provider> get() = transformedLibrariesFileIndex.map { regularFile -> + regularFile.asFile.readLines().map { File(it) } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/GradleKpmMetadataDependencyTransformationTask.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/GradleKpmMetadataDependencyTransformationTask.kt index ca5b9d75051..a5e5d7d719a 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/GradleKpmMetadataDependencyTransformationTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/GradleKpmMetadataDependencyTransformationTask.kt @@ -7,6 +7,7 @@ package org.jetbrains.kotlin.gradle.plugin.mpp.pm20 import org.gradle.api.DefaultTask import org.gradle.api.file.FileCollection +import org.gradle.api.model.ObjectFactory import org.gradle.api.tasks.* import org.gradle.work.NormalizeLineEndings import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution @@ -22,6 +23,7 @@ internal open class GradleKpmMetadataDependencyTransformationTask @get:Internal @field:Transient val fragment: GradleKpmFragment, + private val objectFactory: ObjectFactory, //FIXME annotations private val transformation: GradleKpmFragmentGranularMetadataResolver ) : DefaultTask() { @@ -72,7 +74,7 @@ internal open class GradleKpmMetadataDependencyTransformationTask get() = metadataDependencyResolutions .filterIsInstance() .associateWith { chooseVisibleSourceSets -> - project.files(project.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, materializeFiles = false)) + project.files(objectFactory.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, materializeFiles = false)) .builtBy(this) } @@ -86,7 +88,7 @@ internal open class GradleKpmMetadataDependencyTransformationTask metadataDependencyResolutions .filterIsInstance() .forEach { chooseVisibleSourceSets -> - project.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, materializeFiles = true) + objectFactory.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, materializeFiles = true) } } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/transformMetadataLibraries.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/transformMetadataLibraries.kt index fdfd31d32f0..4d77ca2070a 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/transformMetadataLibraries.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/transformMetadataLibraries.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.gradle.plugin.mpp import org.gradle.api.Project +import org.gradle.api.model.ObjectFactory import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ArtifactMetadataProvider import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ProjectMetadataProvider import java.io.File @@ -37,11 +38,11 @@ internal fun Project.transformMetadataLibrariesForIde( * * In case the [resolution] points to a project dependency, then the output file collections will be returned. */ -internal fun Project.transformMetadataLibrariesForBuild( +internal fun ObjectFactory.transformMetadataLibrariesForBuild( resolution: MetadataDependencyResolution.ChooseVisibleSourceSets, outputDirectory: File, materializeFiles: Boolean ): Iterable { return when (resolution.metadataProvider) { - is ProjectMetadataProvider -> project.files( + is ProjectMetadataProvider -> fileCollection().from( resolution.visibleSourceSetNamesExcludingDependsOn.map { visibleSourceSetName -> resolution.metadataProvider.getSourceSetCompiledMetadata(visibleSourceSetName) } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/metadata/KotlinMetadataTargetConfigurator.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/metadata/KotlinMetadataTargetConfigurator.kt index 5cbca6cf380..43c7f5801ee 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/metadata/KotlinMetadataTargetConfigurator.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/metadata/KotlinMetadataTargetConfigurator.kt @@ -6,8 +6,7 @@ package org.jetbrains.kotlin.gradle.targets.metadata import org.gradle.api.Project -import org.gradle.api.artifacts.Configuration -import org.gradle.api.artifacts.component.ComponentIdentifier +import org.gradle.api.Task import org.gradle.api.artifacts.result.ResolvedArtifactResult import org.gradle.api.attributes.AttributeContainer import org.gradle.api.attributes.Category @@ -57,6 +56,7 @@ class KotlinMetadataTargetConfigurator : KotlinOnlyTargetConfigurator, KotlinMetadataTarget>(createTestCompilation = false) { companion object { internal const val ALL_METADATA_JAR_NAME = "allMetadataJar" + internal const val TRANSFORM_ALL_SOURCESETS_DEPENDENCIES_METADATA = "transformDependenciesMetadata" internal fun transformGranularMetadataTaskName(sourceSetName: String) = lowerCamelCaseName("transform", sourceSetName, "DependenciesMetadata") @@ -344,6 +344,8 @@ class KotlinMetadataTargetConfigurator : "Generates serialized dependencies metadata for compilation '${compilation.name}' of target '${compilation.target.name}' (for tooling)" } + project.locateOrRegisterTask(TRANSFORM_ALL_SOURCESETS_DEPENDENCIES_METADATA).dependsOn(transformationTask) + val artifacts = sourceSet.internal.resolvableMetadataConfiguration.incoming.artifacts // Metadata from visible source sets within dependsOn closure diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt index 6715b43ae1f..42c477cf4db 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt @@ -171,9 +171,7 @@ abstract class AbstractKotlinNativeCompile< { // Avoid resolving these dependencies during task graph construction when we can't build the target: if (konanTarget.enabledOnCurrentHost) - objectFactory.fileCollection().from( - compilation.compileDependencyFiles.filterOutPublishableInteropLibs(project) - ) + objectFactory.fileCollection().from({ compilation.compileDependencyFiles }) else objectFactory.fileCollection() } ) @@ -467,6 +465,8 @@ internal constructor( return SharedCompilationData(manifestFile, isAllowCommonizer, refinesModule) } + private val libDirectories = project.buildLibDirectories() + @TaskAction fun compile() { val output = outputFile.get() @@ -482,7 +482,7 @@ internal constructor( optimized, debuggable, konanTarget, - libraries.files.filterKlibsPassedToCompiler(), + libraries.filterOutPublishableInteropLibs(libDirectories).files.filterKlibsPassedToCompiler(), languageSettings, enableEndorsedLibs, compilerOptions,