[Gradle] Add support Configuration Cache for MetadataDependencyTransformationTask

* Write transformedLibraries to a file after Task execution
* Write visible source sets to a file after Task execution
* Rearrange task properties, inputs outputs in the way that task state
  is serializable to configuration cache.
* Introduce transformDependenciesMetadata task to be an umbrella for
  other transform tasks. This will be convinient for tests and other
  development purposes

^KT-49933

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/GranularMetadataTransformation.kt

@@ -136,9 +136,12 @@ internal class GranularMetadataTransformation(
 
     val metadataDependencyResolutions: Iterable<MetadataDependencyResolution> by lazy { doTransform() }
 
-    val visibleSourceSetsByComponentId: Map<ComponentIdentifierKey, Set<String>> = metadataDependencyResolutions
+    val visibleSourceSetsByComponentId: Map<ComponentIdentifierKey, Set<String>> by lazy {
-        .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
+        metadataDependencyResolutions
-        .associate { it.dependency.id.uniqueKey to it.allVisibleSourceSetNames }
+            .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
+            .groupBy { it.dependency.id.uniqueKey }
+            .mapValues { (_, visibleSourceSets) -> visibleSourceSets.flatMapTo(mutableSetOf()) { it.allVisibleSourceSetNames } }
+    }
 
     private fun doTransform(): Iterable<MetadataDependencyResolution> {
         val result = mutableListOf<MetadataDependencyResolution>()

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/KotlinMetadataLibraryDirectories.kt

@@ -6,13 +6,14 @@
 package org.jetbrains.kotlin.gradle.plugin.mpp
 
 import org.gradle.api.Project
+import org.gradle.api.file.ProjectLayout
 import java.io.File
 
 private const val kotlinTransformedMetadataLibraries = "kotlinTransformedMetadataLibraries"
 private const val kotlinTransformedCInteropMetadataLibraries = "kotlinTransformedCInteropMetadataLibraries"
 
-internal fun Project.kotlinTransformedMetadataLibraryDirectoryForBuild(sourceSetName: String): File =
+internal fun ProjectLayout.kotlinTransformedMetadataLibraryDirectoryForBuild(sourceSetName: String): File =
-    buildDir.resolve(kotlinTransformedMetadataLibraries).resolve(sourceSetName)
+    buildDirectory.get().asFile.resolve(kotlinTransformedMetadataLibraries).resolve(sourceSetName)
 
 internal val Project.kotlinTransformedMetadataLibraryDirectoryForIde: File
     get() = rootDir.resolve(".gradle").resolve("kotlin").resolve(kotlinTransformedMetadataLibraries)

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/MetadataDependencyTransformationTask.kt

@@ -6,24 +6,22 @@
 package org.jetbrains.kotlin.gradle.plugin.mpp
 
 import org.gradle.api.DefaultTask
+import org.gradle.api.file.ConfigurableFileCollection
 import org.gradle.api.file.FileCollection
+import org.gradle.api.file.ProjectLayout
+import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.model.ObjectFactory
 import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.*
 import org.gradle.work.NormalizeLineEndings
 import org.jetbrains.kotlin.gradle.dsl.kotlinExtension
-import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation
 import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
-import org.jetbrains.kotlin.gradle.plugin.sources.KotlinDependencyScope
 import org.jetbrains.kotlin.gradle.plugin.sources.KotlinDependencyScope.*
 import org.jetbrains.kotlin.gradle.plugin.sources.internal
-import org.jetbrains.kotlin.gradle.targets.metadata.ALL_COMPILE_METADATA_CONFIGURATION_NAME
 import org.jetbrains.kotlin.gradle.targets.metadata.KotlinMetadataTargetConfigurator
-import org.jetbrains.kotlin.gradle.targets.metadata.ResolvedMetadataFilesProvider
 import org.jetbrains.kotlin.gradle.targets.metadata.dependsOnClosureWithInterCompilationDependencies
-import org.jetbrains.kotlin.gradle.utils.filesProvider
+import org.jetbrains.kotlin.gradle.tasks.locateTask
-import org.jetbrains.kotlin.gradle.utils.getValue
+import org.jetbrains.kotlin.gradle.utils.*
-import org.jetbrains.kotlin.gradle.utils.notCompatibleWithConfigurationCacheCompat
-import org.jetbrains.kotlin.gradle.utils.outputFilesProvider
 import java.io.File
 import javax.inject.Inject
 
@@ -34,78 +32,103 @@ typealias TransformKotlinGranularMetadata = MetadataDependencyTransformationTask
 
 open class MetadataDependencyTransformationTask
 @Inject constructor(
-    @get:Internal
+    kotlinSourceSet: KotlinSourceSet,
-    @field:Transient
+    private val objectFactory: ObjectFactory,
-    val kotlinSourceSet: KotlinSourceSet
+    private val projectLayout: ProjectLayout
 ) : DefaultTask() {
 
-    init {
+    //region Task Configuration State & Inputs
-        notCompatibleWithConfigurationCacheCompat(
+    private val transformationParameters = GranularMetadataTransformation.Params(project, kotlinSourceSet)
-            "Task $name does not support Gradle Configuration Cache. Check KT-49933 for more info"
-        )
-    }
 
     @get:OutputDirectory
-    val outputsDir: File by project.provider {
+    val outputsDir: File get() = projectLayout.kotlinTransformedMetadataLibraryDirectoryForBuild(transformationParameters.sourceSetName)
-        project.kotlinTransformedMetadataLibraryDirectoryForBuild(kotlinSourceSet.name)
-    }
 
     @Suppress("unused") // Gradle input
     @get:InputFiles
     @get:PathSensitive(PathSensitivity.RELATIVE)
     @get:IgnoreEmptyDirectories
     @get:NormalizeLineEndings
-    internal val configurationToResolve: FileCollection get() = kotlinSourceSet.internal.resolvableMetadataConfiguration
+    internal val configurationToResolve: FileCollection = kotlinSourceSet.internal.resolvableMetadataConfiguration
 
-    private val participatingSourceSets: Set<KotlinSourceSet>
+    @delegate:Transient // Only needed for configuring task inputs
-        get() = kotlinSourceSet.internal.withDependsOnClosure.toMutableSet().apply {
+    private val participatingSourceSets: Set<KotlinSourceSet> by lazy {
+        kotlinSourceSet.internal.withDependsOnClosure.toMutableSet().apply {
             if (any { it.name == KotlinSourceSet.COMMON_MAIN_SOURCE_SET_NAME })
                 add(project.kotlinExtension.sourceSets.getByName(KotlinSourceSet.COMMON_MAIN_SOURCE_SET_NAME))
         }
+    }
 
     @Suppress("unused") // Gradle input
     @get:Input
-    internal val inputSourceSetsAndCompilations: Map<String, Iterable<String>> by project.provider {
+    internal val inputSourceSetsAndCompilations: Map<String, Iterable<String>> by lazy {
         participatingSourceSets.associate { sourceSet ->
             sourceSet.name to sourceSet.internal.compilations.map { it.name }.sorted()
         }
     }
 
-    private val participatingCompilations: Iterable<KotlinCompilation<*>>
-        get() = participatingSourceSets.flatMap { it.internal.compilations }.toSet()
-
     @Suppress("unused") // Gradle input
     @get:Input
-    internal val inputCompilationDependencies: Map<String, Set<List<String?>>> by project.provider {
+    internal val inputCompilationDependencies: Map<String, Set<List<String?>>> by lazy {
-        participatingCompilations.associate {
+        participatingSourceSets.flatMap { it.internal.compilations }.associate {
             it.name to project.configurations.getByName(it.compileDependencyConfigurationName)
                 .allDependencies.map { listOf(it.group, it.name, it.version) }.toSet()
         }
     }
 
-    private val parentVisibleSourceSets: List<Map<String, Set<String>>> by lazy {
+    @get:OutputFile
-        dependsOnClosureWithInterCompilationDependencies(kotlinSourceSet).map {
+    val transformedLibrariesFileIndex: RegularFileProperty = objectFactory
-            project.tasks.withType(MetadataDependencyTransformationTask::class.java)
+        .fileProperty()
-                .getByName(KotlinMetadataTargetConfigurator.transformGranularMetadataTaskName(it.name))
+        .apply { set(outputsDir.resolve("${kotlinSourceSet.name}.transformedLibraries")) }
-                .transformation
+
-                .visibleSourceSetsByComponentId
+    @get:OutputFile
+    val visibleSourceSetsFile: RegularFileProperty = objectFactory
+        .fileProperty()
+        .apply { set(outputsDir.resolve("${kotlinSourceSet.name}.visibleSourceSets")) }
+
+    @get:InputFiles
+    val parentVisibleSourceSetFiles: ConfigurableFileCollection = objectFactory
+        .fileCollection()
+        .from(
+            {
+                val parentSourceSets: List<Provider<File>> = dependsOnClosureWithInterCompilationDependencies(kotlinSourceSet).mapNotNull {
+                    project
+                        .tasks
+                        .locateTask<MetadataDependencyTransformationTask>(KotlinMetadataTargetConfigurator.transformGranularMetadataTaskName(it.name))
+                        ?.flatMap { it.visibleSourceSetsFile.map { it.asFile } }
+                }
+
+                parentSourceSets
+            }
+        )
+
+    //endregion Task Configuration State & Inputs
+
+    @TaskAction
+    fun transformMetadata() {
+        val transformation = GranularMetadataTransformation(
+            params = transformationParameters,
+            parentVisibleSourceSetsProvider = { parentVisibleSourceSetFiles.map(::readVisibleSourceSetsFile) }
+        )
+
+        if (outputsDir.isDirectory) {
+            outputsDir.deleteRecursively()
         }
+        outputsDir.mkdirs()
+
+        val metadataDependencyResolutions = transformation.metadataDependencyResolutions
+
+        metadataDependencyResolutions
+            .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
+            .forEach { chooseVisibleSourceSets ->
+                objectFactory.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, true)
+            }
+
+        writeTransformedLibraries(metadataDependencyResolutions)
+        writeVisibleSourceSets(transformation.visibleSourceSetsByComponentId)
     }
 
-    private val transformation = GranularMetadataTransformation(
+    private fun writeTransformedLibraries(resolutions: Iterable<MetadataDependencyResolution>) {
-        params = GranularMetadataTransformation.Params(project, kotlinSourceSet),
+        val fileList = resolutions.flatMap { resolution ->
-        parentVisibleSourceSetsProvider = { parentVisibleSourceSets }
-    )
-
-    @get:Internal
-    @delegate:Transient // exclude from Gradle instant execution state
-    internal val metadataDependencyResolutions: Iterable<MetadataDependencyResolution> by project.provider {
-        transformation.metadataDependencyResolutions
-    }
-
-    @get:Internal
-    internal val transformedLibraries: Provider<Iterable<File>> = project.provider {
-        metadataDependencyResolutions.flatMap { resolution ->
             val files: Iterable<File> = when (resolution) {
                 is MetadataDependencyResolution.ChooseVisibleSourceSets -> when (val metadataProvider = resolution.metadataProvider) {
                     is MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ArtifactMetadataProvider -> metadataProvider.read { content ->
@@ -123,8 +146,7 @@ open class MetadataDependencyTransformationTask
                     }
                 }
                 is MetadataDependencyResolution.Exclude -> emptySet()
-                is MetadataDependencyResolution.KeepOriginalDependency -> transformation
+                is MetadataDependencyResolution.KeepOriginalDependency -> transformationParameters
-                    .params
                     .resolvedMetadataConfiguration
                     .componentArtifacts(resolution.dependency)
                     .map { it.file }
@@ -132,47 +154,28 @@ open class MetadataDependencyTransformationTask
 
             files
         }
+
+        val content = fileList.joinToString("\n")
+
+        transformedLibrariesFileIndex.get().asFile.writeText(content)
     }
 
-    /**
+    private fun writeVisibleSourceSets(visibleSourceSetsByComponentId: Map<String, Set<String>>) {
-     * Contains "actually transformed" metadata libraries (extracted from any [CompositeMetadataArtifact]) as well as
+        val content = visibleSourceSetsByComponentId.entries.joinToString("\n") { (id, visibleSourceSets) ->
-     * [FileCollection]s pointing to project dependency klibs.
+            "$id => ${visibleSourceSets.joinToString(",")}"
-     *
-     * Project dependencies are still required to be listed here (despite not being produced by this task), since
-     * this [filesByResolution] will be used to build the compile-classpath for metadata compilations.
-     */
-    @get:Internal
-    internal val filesByResolution: Map<out MetadataDependencyResolution, FileCollection>
-        get() = metadataDependencyResolutions
-            .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
-            .associateWith { chooseVisibleSourceSets ->
-                outputFilesProvider {
-                    project.transformMetadataLibrariesForBuild(
-                        chooseVisibleSourceSets, outputsDir, materializeFiles = false
-                    )
-                }
-            }
-
-    @TaskAction
-    fun transformMetadata() {
-        if (outputsDir.isDirectory) {
-            outputsDir.deleteRecursively()
         }
-        outputsDir.mkdirs()
+        visibleSourceSetsFile.get().asFile.writeText(content)
+    }
 
-        metadataDependencyResolutions
+    private fun readVisibleSourceSetsFile(file: File): Map<String, Set<String>> = file
-            .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
+        .readLines()
-            .forEach { chooseVisibleSourceSets ->
+        .associate { string ->
-                project.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, true)
+            val (id, visibleSourceSetsString) = string.split(" => ")
-            }
+            id to visibleSourceSetsString.split(",").toSet()
+        }
+
+    @get:Internal // Warning! transformedLibraries is available only after Task Execution
+    val transformedLibraries: Provider<List<File>> get() = transformedLibrariesFileIndex.map { regularFile ->
+        regularFile.asFile.readLines().map { File(it) }
     }
 }
-
-internal class SourceSetResolvedMetadataProvider(
-    taskProvider: TaskProvider<out MetadataDependencyTransformationTask>
-) : ResolvedMetadataFilesProvider {
-    override val buildDependencies: Iterable<TaskProvider<*>> = listOf(taskProvider)
-    override val metadataResolutions: Iterable<MetadataDependencyResolution> by taskProvider.map { it.metadataDependencyResolutions }
-    override val metadataFilesByResolution: Map<out MetadataDependencyResolution, FileCollection>
-            by taskProvider.map { it.filesByResolution }
-}

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/GradleKpmMetadataDependencyTransformationTask.kt

@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.gradle.plugin.mpp.pm20
 
 import org.gradle.api.DefaultTask
 import org.gradle.api.file.FileCollection
+import org.gradle.api.model.ObjectFactory
 import org.gradle.api.tasks.*
 import org.gradle.work.NormalizeLineEndings
 import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution
@@ -22,6 +23,7 @@ internal open class GradleKpmMetadataDependencyTransformationTask
     @get:Internal
     @field:Transient
     val fragment: GradleKpmFragment,
+    private val objectFactory: ObjectFactory,
     //FIXME annotations
     private val transformation: GradleKpmFragmentGranularMetadataResolver
 ) : DefaultTask() {
@@ -72,7 +74,7 @@ internal open class GradleKpmMetadataDependencyTransformationTask
         get() = metadataDependencyResolutions
             .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
             .associateWith { chooseVisibleSourceSets ->
-                project.files(project.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, materializeFiles = false))
+                project.files(objectFactory.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, materializeFiles = false))
                     .builtBy(this)
             }
 
@@ -86,7 +88,7 @@ internal open class GradleKpmMetadataDependencyTransformationTask
         metadataDependencyResolutions
             .filterIsInstance<MetadataDependencyResolution.ChooseVisibleSourceSets>()
             .forEach { chooseVisibleSourceSets ->
-                project.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, materializeFiles = true)
+                objectFactory.transformMetadataLibrariesForBuild(chooseVisibleSourceSets, outputsDir, materializeFiles = true)
             }
     }
 }

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/transformMetadataLibraries.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.gradle.plugin.mpp
 
 import org.gradle.api.Project
+import org.gradle.api.model.ObjectFactory
 import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ArtifactMetadataProvider
 import org.jetbrains.kotlin.gradle.plugin.mpp.MetadataDependencyResolution.ChooseVisibleSourceSets.MetadataProvider.ProjectMetadataProvider
 import java.io.File
@@ -37,11 +38,11 @@ internal fun Project.transformMetadataLibrariesForIde(
  *
  * In case the [resolution] points to a project dependency, then the output file collections will be returned.
  */
-internal fun Project.transformMetadataLibrariesForBuild(
+internal fun ObjectFactory.transformMetadataLibrariesForBuild(
     resolution: MetadataDependencyResolution.ChooseVisibleSourceSets, outputDirectory: File, materializeFiles: Boolean
 ): Iterable<File> {
     return when (resolution.metadataProvider) {
-        is ProjectMetadataProvider -> project.files(
+        is ProjectMetadataProvider -> fileCollection().from(
             resolution.visibleSourceSetNamesExcludingDependsOn.map { visibleSourceSetName ->
                 resolution.metadataProvider.getSourceSetCompiledMetadata(visibleSourceSetName)
             }

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/metadata/KotlinMetadataTargetConfigurator.kt

@@ -6,8 +6,7 @@
 package org.jetbrains.kotlin.gradle.targets.metadata
 
 import org.gradle.api.Project
-import org.gradle.api.artifacts.Configuration
+import org.gradle.api.Task
-import org.gradle.api.artifacts.component.ComponentIdentifier
 import org.gradle.api.artifacts.result.ResolvedArtifactResult
 import org.gradle.api.attributes.AttributeContainer
 import org.gradle.api.attributes.Category
@@ -57,6 +56,7 @@ class KotlinMetadataTargetConfigurator :
     KotlinOnlyTargetConfigurator<KotlinCompilation<*>, KotlinMetadataTarget>(createTestCompilation = false) {
     companion object {
         internal const val ALL_METADATA_JAR_NAME = "allMetadataJar"
+        internal const val TRANSFORM_ALL_SOURCESETS_DEPENDENCIES_METADATA = "transformDependenciesMetadata"
 
         internal fun transformGranularMetadataTaskName(sourceSetName: String) =
             lowerCamelCaseName("transform", sourceSetName, "DependenciesMetadata")
@@ -344,6 +344,8 @@ class KotlinMetadataTargetConfigurator :
                 "Generates serialized dependencies metadata for compilation '${compilation.name}' of target '${compilation.target.name}' (for tooling)"
         }
 
+        project.locateOrRegisterTask<Task>(TRANSFORM_ALL_SOURCESETS_DEPENDENCIES_METADATA).dependsOn(transformationTask)
+
         val artifacts = sourceSet.internal.resolvableMetadataConfiguration.incoming.artifacts
 
         // Metadata from visible source sets within dependsOn closure

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt

@@ -171,9 +171,7 @@ abstract class AbstractKotlinNativeCompile<
         {
             // Avoid resolving these dependencies during task graph construction when we can't build the target:
             if (konanTarget.enabledOnCurrentHost)
-                objectFactory.fileCollection().from(
+                objectFactory.fileCollection().from({ compilation.compileDependencyFiles })
-                    compilation.compileDependencyFiles.filterOutPublishableInteropLibs(project)
-                )
             else objectFactory.fileCollection()
         }
     )
@@ -467,6 +465,8 @@ internal constructor(
         return SharedCompilationData(manifestFile, isAllowCommonizer, refinesModule)
     }
 
+    private val libDirectories = project.buildLibDirectories()
+
     @TaskAction
     fun compile() {
         val output = outputFile.get()
@@ -482,7 +482,7 @@ internal constructor(
             optimized,
             debuggable,
             konanTarget,
-            libraries.files.filterKlibsPassedToCompiler(),
+            libraries.filterOutPublishableInteropLibs(libDirectories).files.filterKlibsPassedToCompiler(),
             languageSettings,
             enableEndorsedLibs,
             compilerOptions,