[Gradle][MPP] Implement InternalKotlinSourceSet with observable dependsOn

This is used twofold:
1) The dependsOnClosure and withDependsOnClosure can be accessed
w/o any additional overhead

2) This can later be used to maintain caches for a
replacement of the slow compilationsBySourceSet util

KT-52726

libraries/tools/kotlin-gradle-plugin/build.gradle.kts

@@ -99,6 +99,7 @@ dependencies {
         functionalTestImplementation("com.github.gundy:semver4j:0.16.4:nodeps") {
             exclude(group = "*")
         }
+        functionalTestImplementation("org.reflections:reflections:0.10.2")
     }
 
     testCompileOnly(project(":compiler"))

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/sources/AbstractKotlinSourceSet.kt

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+@file:Suppress("LeakingThis")
+
+package org.jetbrains.kotlin.gradle.plugin.sources
+
+import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
+import org.jetbrains.kotlin.gradle.utils.MutableObservableSet
+import org.jetbrains.kotlin.gradle.utils.ObservableSet
+
+abstract class AbstractKotlinSourceSet : InternalKotlinSourceSet {
+    private val dependsOnImpl = MutableObservableSet<KotlinSourceSet>()
+    private val dependsOnClosureImpl = MutableObservableSet<KotlinSourceSet>()
+    private val withDependsOnClosureImpl = MutableObservableSet<KotlinSourceSet>(this)
+
+    final override val dependsOn: ObservableSet<KotlinSourceSet>
+        get() = dependsOnImpl
+
+    final override val dependsOnClosure: ObservableSet<KotlinSourceSet>
+        get() = dependsOnClosureImpl
+
+    final override val withDependsOnClosure: ObservableSet<KotlinSourceSet>
+        get() = withDependsOnClosureImpl
+
+    final override fun dependsOn(other: KotlinSourceSet) {
+        if (other == this) return
+        /*
+        Circular dependsOn hierarchies are not allowed:
+        Throw if this SourceSet is already present in the dependsOnClosure of 'other'
+         */
+        checkForCircularDependsOnEdges(other)
+
+        /* Nothing to-do, if already added as dependency */
+        if (!dependsOnImpl.add(other)) return
+
+        /* Maintain dependsOn closure sets */
+        other.internal.withDependsOnClosure.forAll { inDependsOnClosure ->
+            this.dependsOnClosureImpl.add(inDependsOnClosure)
+            this.withDependsOnClosureImpl.add(inDependsOnClosure)
+        }
+
+        afterDependsOnAdded(other)
+    }
+
+    protected open fun afterDependsOnAdded(other: KotlinSourceSet) = Unit
+}
+

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/sources/CircularDependsOnChecker.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.plugin.sources
+
+import org.gradle.api.InvalidUserCodeException
+import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
+
+internal fun KotlinSourceSet.checkForCircularDependsOnEdges(other: KotlinSourceSet): Nothing? {
+    val stack = mutableListOf(this)
+    val visited = hashSetOf<KotlinSourceSet>()
+
+    fun checkReachableRecursively(from: KotlinSourceSet) {
+        if (!visited.add(from)) return
+        stack += from
+        if (this == from) throw InvalidUserCodeException(
+            "Circular dependsOn hierarchy found in the Kotlin source sets: ${(stack.toList()).joinToString(" -> ") { it.name }}"
+        )
+        from.dependsOn.forEach { next -> checkReachableRecursively(next) }
+        stack -= from
+    }
+
+    checkReachableRecursively(other)
+    return null
+}
+

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/sources/DefaultKotlinSourceSet.kt

@@ -3,6 +3,8 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
+@file:Suppress("DeprecatedCallableAddReplaceWith")
+
 package org.jetbrains.kotlin.gradle.plugin.sources
 
 import org.gradle.api.Action
@@ -27,7 +29,7 @@ const val METADATA_CONFIGURATION_NAME_SUFFIX = "DependenciesMetadata"
 abstract class DefaultKotlinSourceSet @Inject constructor(
     private val project: Project,
     val displayName: String
-) : KotlinSourceSet {
+) : AbstractKotlinSourceSet() {
 
     override val apiConfigurationName: String
         get() = disambiguateName(API)
@@ -87,22 +89,12 @@ abstract class DefaultKotlinSourceSet @Inject constructor(
     override fun dependencies(configure: Action<KotlinDependencyHandler>) =
         dependencies { configure.execute(this) }
 
-    override fun dependsOn(other: KotlinSourceSet) {
-        dependsOnSourceSetsImpl.add(other)
-
-        // Fail-fast approach: check on each new added edge and report a circular dependency at once when the edge is added.
-        checkForCircularDependencies()
-
+    override fun afterDependsOnAdded(other: KotlinSourceSet) {
         project.runProjectConfigurationHealthCheckWhenEvaluated {
             defaultSourceSetLanguageSettingsChecker.runAllChecks(this@DefaultKotlinSourceSet, other)
         }
     }
 
-    private val dependsOnSourceSetsImpl = mutableSetOf<KotlinSourceSet>()
-
-    override val dependsOn: Set<KotlinSourceSet>
-        get() = dependsOnSourceSetsImpl
-
     override fun toString(): String = "source set $name"
 
     private val explicitlyAddedCustomSourceFilesExtensions = ArrayList<String>()
@@ -221,32 +213,6 @@ internal val defaultSourceSetLanguageSettingsChecker =
         ).allChecks
     )
 
-private fun KotlinSourceSet.checkForCircularDependencies() {
-    // If adding an edge creates a cycle, than the source node of the edge belongs to the cycle, so run DFS from that node
-    // to check whether it became reachable from itself
-    val visited = hashSetOf<KotlinSourceSet>()
-    val stack = LinkedHashSet<KotlinSourceSet>() // Store the stack explicitly to pretty-print the cycle
-
-    fun checkReachableRecursively(from: KotlinSourceSet) {
-        stack += from
-        visited += from
-
-        for (to in from.dependsOn) {
-            if (to == this@checkForCircularDependencies)
-                throw InvalidUserCodeException(
-                    "Circular dependsOn hierarchy found in the Kotlin source sets: " +
-                            (stack.toList() + to).joinToString(" -> ") { it.name }
-                )
-
-            if (to !in visited) {
-                checkReachableRecursively(to)
-            }
-        }
-        stack -= from
-    }
-
-    checkReachableRecursively(this@checkForCircularDependencies)
-}
 
 internal fun KotlinSourceSet.disambiguateName(simpleName: String): String {
     val nameParts = listOfNotNull(this.name.takeIf { it != "main" }, simpleName)
@@ -256,15 +222,18 @@ internal fun KotlinSourceSet.disambiguateName(simpleName: String): String {
 internal fun createDefaultSourceDirectorySet(project: Project, name: String?): SourceDirectorySet =
     project.objects.sourceDirectorySet(name!!, name)
 
+@Deprecated("Use InternalKotlinSourceSet.dependsOnClosure instead")
+val KotlinSourceSet.dependsOnClosure: Set<KotlinSourceSet> get() = this.internal.dependsOnClosure
 
-val KotlinSourceSet.dependsOnClosure get() = closure<KotlinSourceSet> { it.dependsOn }
+@Deprecated("Use InternalKotlinSourceSet.withDependsOnClosure instead")
+val KotlinSourceSet.withDependsOnClosure: Set<KotlinSourceSet> get() = this.internal.withDependsOnClosure
 
-val KotlinSourceSet.withDependsOnClosure get() = withClosure { it.dependsOn }
+val Iterable<KotlinSourceSet>.dependsOnClosure: Set<KotlinSourceSet>
+    get() = flatMap { it.internal.dependsOnClosure }.toSet() - this.toSet()
 
-val Iterable<KotlinSourceSet>.dependsOnClosure get() = closure<KotlinSourceSet> { it.dependsOn }
+val Iterable<KotlinSourceSet>.withDependsOnClosure: Set<KotlinSourceSet>
+    get() = flatMap { it.internal.withDependsOnClosure }.toSet()
 
-val Iterable<KotlinSourceSet>.withDependsOnClosure get() = withClosure<KotlinSourceSet> { it.dependsOn }
-
-internal fun KotlinMultiplatformExtension.findSourceSetsDependingOn(sourceSet: KotlinSourceSet): Set<KotlinSourceSet> {
+fun KotlinMultiplatformExtension.findSourceSetsDependingOn(sourceSet: KotlinSourceSet): Set<KotlinSourceSet> {
     return sourceSet.closure { seedSourceSet -> sourceSets.filter { otherSourceSet -> seedSourceSet in otherSourceSet.dependsOn } }
 }

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/sources/InternalKotlinSourceSet.kt

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.plugin.sources
+
+import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
+import org.jetbrains.kotlin.gradle.utils.ObservableSet
+
+internal val KotlinSourceSet.internal: InternalKotlinSourceSet
+    get() = (this as? InternalKotlinSourceSet) ?: throw IllegalArgumentException(
+        "KotlinSourceSet $name does not implement ${InternalKotlinSourceSet::class.simpleName}"
+    )
+
+internal interface InternalKotlinSourceSet : KotlinSourceSet {
+    override val dependsOn: ObservableSet<KotlinSourceSet>
+    val dependsOnClosure: ObservableSet<KotlinSourceSet>
+    val withDependsOnClosure: ObservableSet<KotlinSourceSet>
+}

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/sources/kpm/FragmentMappedKotlinSourceSet.kt

@@ -15,6 +15,7 @@ import org.jetbrains.kotlin.gradle.plugin.KotlinDependencyHandler
 import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
 import org.jetbrains.kotlin.gradle.plugin.LanguageSettingsBuilder
 import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.*
+import org.jetbrains.kotlin.gradle.plugin.sources.AbstractKotlinSourceSet
 import org.jetbrains.kotlin.gradle.plugin.sources.createDefaultSourceDirectorySet
 import javax.inject.Inject
 
@@ -22,7 +23,7 @@ abstract class FragmentMappedKotlinSourceSet @Inject constructor(
     private val sourceSetName: String,
     private val project: Project,
     internal val underlyingFragment: GradleKpmFragment
-) : KotlinSourceSet {
+) : AbstractKotlinSourceSet() {
     val displayName: String
         get() = sourceSetName
 
@@ -69,7 +70,7 @@ abstract class FragmentMappedKotlinSourceSet @Inject constructor(
     override fun dependencies(configure: Action<KotlinDependencyHandler>) =
         underlyingFragment.dependencies(configure)
 
-    override fun dependsOn(other: KotlinSourceSet) {
+    override fun afterDependsOnAdded(other: KotlinSourceSet) {
         if (other !is FragmentMappedKotlinSourceSet) {
             throw InvalidUserDataException("Could set up dependsOn relationship with an unknown source set $other")
         }
@@ -77,11 +78,6 @@ abstract class FragmentMappedKotlinSourceSet @Inject constructor(
         underlyingFragment.refines(otherFragment)
     }
 
-    override val dependsOn: Set<KotlinSourceSet>
-        get() = project.kotlinExtension.sourceSets.filter {
-            it is FragmentMappedKotlinSourceSet && it.underlyingFragment in underlyingFragment.declaredRefinesDependencies
-        }.toSet()
-
     override fun toString(): String = "source set $name"
 
     override val requiresVisibilityOf: Set<KotlinSourceSet>

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/utils/ObservableSet.kt

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.utils
+
+interface ObservableSet<T> : Set<T> {
+    fun whenObjectAdded(action: (T) -> Unit)
+    fun forAll(action: (T) -> Unit)
+}
+
+internal class MutableObservableSet<T>(vararg elements: T) : ObservableSet<T>, MutableSet<T> {
+    private val underlying = mutableSetOf(*elements)
+    private val whenObjectAddedActions = mutableListOf<(T) -> Unit>()
+    private val forAllActions = mutableListOf<(T) -> Unit>()
+
+    override fun whenObjectAdded(action: (T) -> Unit) {
+        whenObjectAddedActions.add(action)
+    }
+
+    override fun forAll(action: (T) -> Unit) {
+        forAllActions.add(action)
+        underlying.toList().forEach(action)
+    }
+
+    override val size: Int
+        get() = underlying.size
+
+    override fun clear() {
+        underlying.clear()
+    }
+
+    override fun addAll(elements: Collection<T>): Boolean {
+        val toAdd = elements.toSet() - underlying
+        return underlying.addAll(toAdd).also {
+            toAdd.forEach { added ->
+                whenObjectAddedActions.toList().forEach { action -> action(added) }
+                forAllActions.toList().forEach { action -> action(added) }
+            }
+        }
+    }
+
+    override fun add(element: T): Boolean {
+        return underlying.add(element).also {
+            whenObjectAddedActions.toList().forEach { action -> action(element) }
+            forAllActions.toList().forEach { action -> action(element) }
+        }
+    }
+
+    override fun isEmpty(): Boolean {
+        return underlying.isEmpty()
+    }
+
+    override fun iterator(): MutableIterator<T> {
+        return underlying.iterator()
+    }
+
+    override fun retainAll(elements: Collection<T>): Boolean {
+        return underlying.retainAll(elements)
+    }
+
+    override fun removeAll(elements: Collection<T>): Boolean {
+        return underlying.removeAll(elements)
+    }
+
+    override fun remove(element: T): Boolean {
+        return underlying.remove(element)
+    }
+
+    override fun containsAll(elements: Collection<T>): Boolean {
+        return underlying.containsAll(elements)
+    }
+
+    override fun contains(element: T): Boolean {
+        return underlying.contains(element)
+    }
+
+    init {
+        underlying.addAll(elements)
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/functionalTest/kotlin/org/jetbrains/kotlin/gradle/ObservableSetTest.kt

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+@file:Suppress("FunctionName")
+
+package org.jetbrains.kotlin.gradle
+
+import org.jetbrains.kotlin.gradle.utils.MutableObservableSet
+import org.junit.Test
+import kotlin.test.assertEquals
+
+class ObservableSetTest {
+
+    class TestListener : (Int) -> Unit {
+        val invocations = mutableListOf<Int>()
+        override fun invoke(p1: Int) {
+            invocations.add(p1)
+        }
+    }
+
+    @Test
+    fun `test - forAll`() {
+        val set = MutableObservableSet(1)
+        val testListener1 = TestListener()
+        val testListener2 = TestListener()
+
+        set.forAll(testListener1)
+        assertEquals(listOf(1), testListener1.invocations)
+
+        set.add(2)
+        assertEquals(listOf(1, 2), testListener1.invocations)
+
+        set.addAll(listOf(3, 4))
+        assertEquals(listOf(1, 2, 3, 4), testListener1.invocations)
+
+        set.forAll(testListener2)
+        assertEquals(listOf(1, 2, 3, 4), testListener2.invocations)
+    }
+
+    @Test
+    fun `test - whenObjectAdded`() {
+        val set = MutableObservableSet(1)
+        val testListener1 = TestListener()
+        val testListener2 = TestListener()
+
+        set.whenObjectAdded(testListener1)
+        assertEquals(emptyList(), testListener1.invocations)
+
+        set.add(2)
+        assertEquals(listOf(2), testListener1.invocations)
+
+        set.addAll(listOf(3, 4))
+        assertEquals(listOf(2, 3, 4), testListener1.invocations)
+
+        set.whenObjectAdded(testListener2)
+        assertEquals(emptyList(), testListener2.invocations)
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/functionalTest/kotlin/org/jetbrains/kotlin/gradle/reflectionUtils.kt

@@ -0,0 +1,10 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle
+
+import org.reflections.Reflections
+
+val reflections = Reflections("org.jetbrains")

libraries/tools/kotlin-gradle-plugin/src/functionalTest/kotlin/org/jetbrains/kotlin/gradle/sources/InternalKotlinSourceSetTest.kt

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+@file:Suppress("FunctionName")
+
+package org.jetbrains.kotlin.gradle.sources
+
+import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
+import org.jetbrains.kotlin.gradle.plugin.sources.InternalKotlinSourceSet
+import org.jetbrains.kotlin.gradle.reflections
+import kotlin.reflect.full.isSubclassOf
+import kotlin.test.Test
+import kotlin.test.fail
+
+class InternalKotlinSourceSetTest {
+    @Test
+    fun `test - all implementations of KotlinSourceSet - implement InternalKotlinSourceSet`() {
+        val subtypesOfKotlinSourceSet = reflections.getSubTypesOf(KotlinSourceSet::class.java)
+        subtypesOfKotlinSourceSet
+            .filter { subtype -> !subtype.isInterface }
+            .forEach { implementation ->
+                if (!implementation.kotlin.isSubclassOf(InternalKotlinSourceSet::class)) {
+                    fail("$implementation does not implement ${InternalKotlinSourceSet::class}")
+                }
+            }
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/plugin/sources/SourceSetVisibilityInferenceTest.kt

@@ -16,6 +16,8 @@ import org.jetbrains.kotlin.gradle.dsl.KotlinCompile
 import org.jetbrains.kotlin.gradle.plugin.*
 import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.KotlinCompilationData
 import org.jetbrains.kotlin.project.model.LanguageSettings
+import org.jetbrains.kotlin.tooling.core.closure
+import org.jetbrains.kotlin.tooling.core.withClosure
 import kotlin.test.Test
 import kotlin.test.assertEquals
 import kotlin.test.assertFailsWith
@@ -234,17 +236,11 @@ class SourceSetCompilationDsl {
     }
 }
 
-class MockKotlinSourceSet(private val name: String) : KotlinSourceSet {
+class MockKotlinSourceSet(private val name: String) : AbstractKotlinSourceSet() {
     override fun getName(): String = name
 
-    override val dependsOn: MutableSet<KotlinSourceSet> = mutableSetOf()
-
     override val requiresVisibilityOf: MutableSet<KotlinSourceSet> = mutableSetOf()
 
-    override fun dependsOn(other: KotlinSourceSet) {
-        dependsOn += other
-    }
-
     override fun requiresVisibilityOf(other: KotlinSourceSet) {
         requiresVisibilityOf += other
     }