Rework adding sources to FilteringSourceRootsContainer in JVM tasks

Support adding lazily-evaluated file collections (i.e. tracking their
source roots lazily, as opposed to eagerly evaluating the source roots.

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptGenerateStubsTask.kt

@@ -38,7 +38,9 @@ import javax.inject.Inject
 open class KaptGenerateStubsTask @Inject constructor(
     objectFactory: ObjectFactory
 ) : KotlinCompile() {
-    override val sourceRootsContainer = FilteringSourceRootsContainer(emptyList(), { isSourceRootAllowed(it) })
+
+    @field:Transient
+    override val sourceRootsContainer = FilteringSourceRootsContainer(objectFactory, { isSourceRootAllowed(it) })
 
     override val kotlinOptions: KotlinJvmOptions = KotlinJvmOptionsImpl()
 
@@ -109,10 +111,11 @@ open class KaptGenerateStubsTask @Inject constructor(
     }
 
     private val sourceRoots by project.provider {
-        kotlinCompileTask.getSourceRoots().let {
+        kotlinCompileTask.getSourceRoots().let { compileTaskSourceRoots ->
-            val javaSourceRoots = it.javaSourceRoots.filterTo(HashSet()) { isSourceRootAllowed(it) }
+            SourceRoots.ForJvm(
-            val kotlinSourceFiles = it.kotlinSourceFiles.filterTo(ArrayList()) { isSourceRootAllowed(it) }
+                compileTaskSourceRoots.kotlinSourceFiles.filterTo(mutableListOf()) { isSourceRootAllowed(it) },
-            SourceRoots.ForJvm(kotlinSourceFiles, javaSourceRoots)
+                javaSourceRootsProvider = { compileTaskSourceRoots.javaSourceRoots.filterTo(mutableSetOf()) { isSourceRootAllowed(it) } }
+            )
         }
     }
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptTask.kt

@@ -132,12 +132,16 @@ abstract class KaptTask @Inject constructor(
     @get:Internal
     var useBuildCache: Boolean = false
 
+    private val sourceRootsFromKotlinTask by project.provider {
+        kotlinCompileTask.sourceRootsContainer.sourceRoots
+    }
+
     @get:InputFiles
     @get:PathSensitive(PathSensitivity.RELATIVE)
     val source: FileCollection = objectFactory
         .fileCollection()
         .from(
-            { kotlinCompileTask.sourceRootsContainer.sourceRoots },
+            { sourceRootsFromKotlinTask },
             stubsDir
         )
         .asFileTree

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPlugin.kt

@@ -15,6 +15,7 @@ import org.gradle.api.artifacts.maven.MavenResolver
 import org.gradle.api.attributes.Usage
 import org.gradle.api.file.ConfigurableFileTree
 import org.gradle.api.file.FileCollection
+import org.gradle.api.file.SourceDirectorySet
 import org.gradle.api.logging.Logging
 import org.gradle.api.plugins.InvalidPluginException
 import org.gradle.api.plugins.JavaPlugin
@@ -146,11 +147,11 @@ internal abstract class KotlinSourceSetProcessor<T : AbstractKotlinCompile<*>>(
         java.allSource.srcDirs(kotlinSrcDirsToAdd)
     }
 
-    private fun filterOutJavaSrcDirsIfPossible(sourceDirectories: FileCollection): FileCollection {
+    private fun filterOutJavaSrcDirsIfPossible(sourceDirectories: SourceDirectorySet): FileCollection {
         val java = javaSourceSet ?: return sourceDirectories
 
         // Build a lazily-resolved file collection that filters out Java sources from sources of this sourceDirectorySet
-        return sourceDirectories.minus(java.java.sourceDirectories)
+        return sourceDirectories.sourceDirectories.minus(java.java.sourceDirectories)
     }
 
     private fun createAdditionalClassesTaskForIdeRunner() {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/KotlinMultiplatformPlugin.kt

@@ -43,6 +43,7 @@ import org.jetbrains.kotlin.konan.target.HostManager
 import org.jetbrains.kotlin.konan.target.KonanTarget.*
 import org.jetbrains.kotlin.konan.target.presetName
 import org.jetbrains.kotlin.statistics.metrics.StringMetrics
+import java.io.File
 
 class KotlinMultiplatformPlugin : Plugin<Project> {
 
@@ -294,7 +295,7 @@ internal fun sourcesJarTask(compilation: KotlinCompilation<*>, componentName: St
 
 internal fun sourcesJarTask(
     project: Project,
-    sourceSets: Lazy<Map<String, FileCollection>>,
+    sourceSets: Lazy<Map<String, Iterable<File>>>,
     componentName: String?,
     artifactNameAppendix: String
 ): TaskProvider<Jar> = sourcesJarTaskNamed(lowerCamelCaseName(componentName, "sourcesJar"), project, sourceSets, artifactNameAppendix)
@@ -302,7 +303,7 @@ internal fun sourcesJarTask(
 internal fun sourcesJarTaskNamed(
     taskName: String,
     project: Project,
-    sourceSets: Lazy<Map<String, FileCollection>>,
+    sourceSets: Lazy<Map<String, Iterable<File>>>,
     artifactNameAppendix: String
 ): TaskProvider<Jar> {
     project.locateTask<Jar>(taskName)?.let {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/kotlinCompilations.kt

@@ -307,7 +307,7 @@ internal fun addCommonSourcesToKotlinCompileTask(
     project: Project,
     taskName: String,
     sourceFileExtensions: Iterable<String>,
-    sources: () -> Iterable<File>
+    sources: () -> Any
 ) = addSourcesToKotlinCompileTask(project, taskName, sourceFileExtensions, lazyOf(true), sources)
 
 // FIXME this function dangerously ignores an incorrect type of the task (e.g. if the actual task is a K/N one); consider reporting a failure
@@ -316,13 +316,18 @@ internal fun addSourcesToKotlinCompileTask(
     taskName: String,
     sourceFileExtensions: Iterable<String>,
     addAsCommonSources: Lazy<Boolean> = lazyOf(false),
-    sources: () -> Iterable<File>
+    /** Evaluated as project.files(...) */
+    sources: () -> Any
 ) {
     fun AbstractKotlinCompile<*>.configureAction() {
-        source(project.files(Callable { sources() }))
+        // In this call, the super-implementation of `source` adds the directories files to the roots of the union file tree,
+        // so it's OK to pass just the source roots.
+        source(Callable(sources))
         sourceFilesExtensions(sourceFileExtensions)
-        commonSourceSet += project.files(Callable {
+
-            if (addAsCommonSources.value) sources() else emptyList<Any>()
+        // The `commonSourceSet` is passed to the compiler as-is, converted with toList
+        commonSourceSet += project.files(Callable<Any> {
+            if (addAsCommonSources.value) sources else emptyList<Any>()
         })
     }
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/FragmentSourcesProvider.kt

@@ -7,37 +7,47 @@ package org.jetbrains.kotlin.gradle.plugin.mpp.pm20
 
 import org.gradle.api.Project
 import org.gradle.api.file.FileCollection
+import org.gradle.api.provider.Provider
 import org.gradle.language.base.plugins.LifecycleBasePlugin
 import org.jetbrains.kotlin.project.model.utils.variantsContainingFragment
+import java.io.File
 
+typealias SourceRoots = Iterable<File>
+typealias SourceRootsProvider = Provider<out SourceRoots>
+typealias MultipleSourceRootsProvider = Provider<out Iterable<SourceRootsProvider>>
+typealias SourceRootsProvidersByFragment = Map<KotlinGradleFragment, SourceRootsProvider>
+
+/** Note: the API is [Provider]-based rather than FileCollection-based because FileCollection API erases the internal structure of the
+ * file sets, and this internal structure is currently needed for correctly inferring Java source roots from the sources added to the
+ * JVM compilations (it is important to pass sources in SourceDirectorySets) */
 open class FragmentSourcesProvider {
     protected open fun getSourcesFromFragmentsAsMap(
         fragments: Iterable<KotlinGradleFragment>
-    ): Map<KotlinGradleFragment, FileCollection> =
+    ): SourceRootsProvidersByFragment =
-        fragments.associateWith { it.project.files(it.project.provider { it.kotlinSourceRoots }) }
+        fragments.associateWith { it.project.provider { it.kotlinSourceRoots } }
 
-    open fun getFragmentOwnSources(fragment: KotlinGradleFragment): FileCollection =
+    open fun getFragmentOwnSources(fragment: KotlinGradleFragment): SourceRootsProvider =
         getSourcesFromFragmentsAsMap(listOf(fragment)).values.single()
 
-    open fun getAllFragmentSourcesAsMap(module: KotlinGradleModule): Map<KotlinGradleFragment, FileCollection> =
+    open fun getAllFragmentSourcesAsMap(module: KotlinGradleModule): SourceRootsProvidersByFragment =
         getSourcesFromFragmentsAsMap(module.fragments)
 
-    open fun getSourcesFromRefinesClosureAsMap(fragment: KotlinGradleFragment): Map<KotlinGradleFragment, FileCollection> =
+    open fun getSourcesFromRefinesClosureAsMap(fragment: KotlinGradleFragment): SourceRootsProvidersByFragment =
         getSourcesFromFragmentsAsMap(fragment.refinesClosure)
 
-    open fun getSourcesFromRefinesClosure(fragment: KotlinGradleFragment): FileCollection =
+    open fun getSourcesFromRefinesClosure(fragment: KotlinGradleFragment): MultipleSourceRootsProvider =
-        fragment.project.files(fragment.project.provider { getSourcesFromRefinesClosureAsMap(fragment).values })
+        fragment.project.provider { getSourcesFromRefinesClosureAsMap(fragment).values }
 
-    open fun getCommonSourcesFromRefinesClosure(fragment: KotlinGradleFragment): FileCollection {
+    open fun getCommonSourcesFromRefinesClosure(fragment: KotlinGradleFragment): MultipleSourceRootsProvider {
         val containingModule = fragment.containingModule
         val project = containingModule.project
         getSourcesFromRefinesClosureAsMap(fragment)
-        return project.files(project.provider {
+        return project.provider {
             fragment.refinesClosure.filter {
                 // Every fragment refined by some other fragment should be considered common, even if it is included in just one variant
                 containingModule.variantsContainingFragment(it).toSet() != setOf(it)
-            }.map { it.kotlinSourceRoots }
+            }.map { project.provider { it.kotlinSourceRoots } }
-        })
+        }
     }
 }
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/KotlinCompilationTaskConfigurator.kt

@@ -22,10 +22,10 @@ open class KotlinCompilationTaskConfigurator(
 ) {
     open val fragmentSourcesProvider: FragmentSourcesProvider = FragmentSourcesProvider()
 
-    open fun getSourcesForFragmentCompilation(fragment: KotlinGradleFragment) =
+    open fun getSourcesForFragmentCompilation(fragment: KotlinGradleFragment): MultipleSourceRootsProvider =
         fragmentSourcesProvider.getSourcesFromRefinesClosure(fragment)
 
-    open fun getCommonSourcesForFragmentCompilation(fragment: KotlinGradleFragment) =
+    open fun getCommonSourcesForFragmentCompilation(fragment: KotlinGradleFragment): MultipleSourceRootsProvider =
         fragmentSourcesProvider.getCommonSourcesFromRefinesClosure(fragment)
 
     fun createKotlinJvmCompilationTask(

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/abstractVariantFactories.kt

@@ -51,7 +51,7 @@ abstract class AbstractKotlinGradleVariantFactory<T : KotlinGradleVariant>(
         sourcesJarTaskNamed(
             fragment.sourceArchiveTaskName,
             project,
-            lazy { FragmentSourcesProvider().getSourcesFromRefinesClosureAsMap(fragment).mapKeys { it.key.fragmentName } },
+            lazy { FragmentSourcesProvider().getSourcesFromRefinesClosureAsMap(fragment).entries.associate { it.key.fragmentName to it.value.get() } },
             fragment.name
         )
     }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/pm20/fragmentsMetadataCompilation.kt

@@ -9,6 +9,7 @@ import org.gradle.api.DefaultTask
 import org.gradle.api.Project
 import org.gradle.api.attributes.Usage
 import org.gradle.api.file.FileCollection
+import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.TaskProvider
 import org.gradle.jvm.tasks.Jar
 import org.jetbrains.kotlin.gradle.dsl.pm20Extension
@@ -27,6 +28,7 @@ import org.jetbrains.kotlin.gradle.utils.dashSeparatedName
 import org.jetbrains.kotlin.gradle.utils.lowerCamelCaseName
 import org.jetbrains.kotlin.library.KLIB_FILE_EXTENSION
 import org.jetbrains.kotlin.project.model.KotlinModuleFragment
+import java.io.File
 import java.util.concurrent.Callable
 
 internal fun configureMetadataResolutionAndBuild(module: KotlinGradleModule) {
@@ -65,7 +67,7 @@ internal fun configureMetadataExposure(module: KotlinGradleModule) {
     val sourcesArtifact = sourcesJarTaskNamed(
         module.disambiguateName("allSourcesJar"),
         project,
-        lazy { FragmentSourcesProvider().getAllFragmentSourcesAsMap(module).mapKeys { it.key.fragmentName } },
+        lazy { FragmentSourcesProvider().getAllFragmentSourcesAsMap(module).entries.associate { it.key.fragmentName to it.value.get() } },
         sourcesArtifactAppendix
     )
     DocumentationVariantConfigurator().createSourcesElementsConfiguration(
@@ -222,12 +224,12 @@ private class MetadataCompilationTasksConfigurator(project: Project) : KotlinCom
         addCommonSourcesToKotlinCompileTask(project, compilationData.compileKotlinTaskName, emptyList()) { commonSources }
     }
 
-    override fun getSourcesForFragmentCompilation(fragment: KotlinGradleFragment): FileCollection {
+    override fun getSourcesForFragmentCompilation(fragment: KotlinGradleFragment): MultipleSourceRootsProvider {
-        return fragmentSourcesProvider.getFragmentOwnSources(fragment)
+        return project.provider { listOf(fragmentSourcesProvider.getFragmentOwnSources(fragment)) }
     }
 
-    override fun getCommonSourcesForFragmentCompilation(fragment: KotlinGradleFragment): FileCollection {
+    override fun getCommonSourcesForFragmentCompilation(fragment: KotlinGradleFragment): MultipleSourceRootsProvider {
-        return fragmentSourcesProvider.getFragmentOwnSources(fragment)
+        return project.provider { listOf(fragmentSourcesProvider.getFragmentOwnSources(fragment)) }
     }
 }
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/SourceRoots.kt

@@ -1,14 +1,18 @@
 package org.jetbrains.kotlin.gradle.tasks
 
+import org.gradle.api.file.FileCollection
 import org.gradle.api.file.FileTree
 import org.gradle.api.file.SourceDirectorySet
 import org.gradle.api.logging.Logger
+import org.gradle.api.model.ObjectFactory
+import org.gradle.api.provider.Provider
 import org.jetbrains.kotlin.gradle.logging.kotlinDebug
 import org.jetbrains.kotlin.gradle.utils.isJavaFile
 import org.jetbrains.kotlin.gradle.utils.isKotlinFile
 import org.jetbrains.kotlin.gradle.utils.isParentOf
 import java.io.File
 import java.util.*
+import java.util.concurrent.Callable
 
 internal sealed class SourceRoots(val kotlinSourceFiles: List<File>) {
     private companion object {
@@ -20,14 +24,21 @@ internal sealed class SourceRoots(val kotlinSourceFiles: List<File>) {
         logger.kotlinDebug { "$taskName source roots: ${dumpPaths(kotlinSourceFiles)}" }
     }
 
-    class ForJvm(kotlinSourceFiles: List<File>, val javaSourceRoots: Set<File>) : SourceRoots(kotlinSourceFiles) {
+    class ForJvm constructor(
+        kotlinSourceFiles: List<File>,
+        private val javaSourceRootsProvider: () -> Set<File>
+    ) : SourceRoots(kotlinSourceFiles) {
+        val javaSourceRoots: Set<File>
+            get() = javaSourceRootsProvider()
+
+        constructor(kotlinSourceFiles: List<File>, allSourceRoots: FileCollection, taskSource: FileCollection) : this(kotlinSourceFiles, {
+            findRootsForSources(allSourceRoots, taskSource.filter(File::isJavaFile)).toSet()
+        })
+
         companion object {
             fun create(taskSource: FileTree, sourceRoots: FilteringSourceRootsContainer, sourceFilesExtensions: List<String>): ForJvm {
                 val kotlinSourceFiles = (taskSource as Iterable<File>).filter { it.isKotlinFile(sourceFilesExtensions) }
-                val javaSourceRoots = findRootsForSources(
+                return ForJvm(kotlinSourceFiles, sourceRoots.sourceRoots, taskSource)
-                    sourceRoots.sourceRoots, taskSource.filter(File::isJavaFile)
-                )
-                return ForJvm(kotlinSourceFiles, javaSourceRoots)
             }
 
             private fun findRootsForSources(allSourceRoots: Iterable<File>, sources: Iterable<File>): Set<File> {
@@ -60,33 +71,43 @@ internal sealed class SourceRoots(val kotlinSourceFiles: List<File>) {
     }
 }
 
-internal class FilteringSourceRootsContainer(roots: List<File> = emptyList(), val filter: (File) -> Boolean = { true }) {
+internal class FilteringSourceRootsContainer(
-    private val mutableSourceRoots = roots.filterTo(mutableListOf(), filter)
+    private val objectFactory: ObjectFactory,
+    val filter: (File) -> Boolean = { true }
+) {
+    private val sourceContainers: MutableList<Any> = mutableListOf()
 
-    val sourceRoots: List<File>
+    private fun getFilteredSourceRootsFrom(any: Any) = objectFactory.fileCollection().from(Callable {
-        get() = mutableSourceRoots
+        val resultItems = mutableListOf<Any>()
+        fun getRootsFrom(item: Any?) {
+            when (item) {
+                is SourceDirectorySet -> resultItems.add(item.sourceDirectories)
+                is Callable<*> -> getRootsFrom(item.call())
+                is Provider<*> -> if (item.isPresent) getRootsFrom(item.get())
+                is FileCollection -> resultItems.add(item)
+                is Iterable<*> -> item.forEach { getRootsFrom(it) }
+                is Array<*> -> item.forEach { getRootsFrom(it) }
+                is Any /* not null */ -> resultItems.add(item)
+            }
+        }
+        getRootsFrom(any)
+        resultItems
+    }).filter(filter)
+
+    val sourceRoots: FileCollection
+        get() = getFilteredSourceRootsFrom(sourceContainers)
 
     fun clear() {
-        mutableSourceRoots.clear()
+        sourceContainers.clear()
     }
 
-    fun set(source: Any?): List<File> {
+    fun set(source: Any): FileCollection {
         clear()
         return add(source)
     }
 
-    fun add(vararg sources: Any?): List<File> {
+    fun add(vararg sources: Any): FileCollection {
-        val filteredDirs = mutableListOf<File>()
+        sourceContainers.addAll(sources.toList())
-        for (source in sources) {
+        return getFilteredSourceRootsFrom(sources)
-            when (source) {
-                is SourceDirectorySet -> filteredDirs += source.srcDirs.filter { filter(it) }
-                is File -> if (filter(source)) filteredDirs.add(source)
-                is Collection<*> -> source.forEach { filteredDirs += add(it) }
-                is Array<*> -> source.forEach { filteredDirs += add(it) }
-            }
-        }
-
-        mutableSourceRoots += filteredDirs
-        return filteredDirs
     }
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt

@@ -471,7 +471,13 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
         get() = taskData.compilation.kotlinOptions as KotlinJvmOptions
 
     @get:Internal
-    internal open val sourceRootsContainer = FilteringSourceRootsContainer()
+    @field:Transient
+    internal open val sourceRootsContainer = FilteringSourceRootsContainer(project.objects)
+
+    private val jvmSourceRoots by project.provider {
+        // serialize in the task state for configuration caching; avoid building anew in task execution, as it may access the project model
+        SourceRoots.ForJvm.create(source, sourceRootsContainer, sourceFilesExtensions)
+    }
 
     /** A package prefix that is used for locating Java sources in a directory structure with non-full-depth packages.
      *
@@ -512,8 +518,7 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
         KotlinJvmCompilerArgumentsContributor(KotlinJvmCompilerArgumentsProvider(this))
     }
 
-    @Internal
+    override fun getSourceRoots(): SourceRoots.ForJvm = jvmSourceRoots
-    override fun getSourceRoots() = SourceRoots.ForJvm.create(getSource(), sourceRootsContainer, sourceFilesExtensions)
 
     override fun callCompilerAsync(args: K2JVMCompilerArguments, sourceRoots: SourceRoots, changedFiles: ChangedFiles) {
         sourceRoots as SourceRoots.ForJvm