Build: Group KGP Integration Tests by Gradle version

#KTI-1596

To avoid Gradle daemons pile up during test execution, run each Gradle
version in a separate task. Gradle daemon used for testing will be
terminated at the end of each execution preventing memory exhausting
in a low-memory environment (e.g., build agents).

libraries/tools/kotlin-gradle-plugin-integration-tests/build.gradle.kts

@@ -274,6 +274,56 @@ val memoryPerGradleTestWorkerMb = 6000
 val maxParallelTestForks =
     (totalMaxMemoryForTestsMb / memoryPerGradleTestWorkerMb).coerceIn(1, Runtime.getRuntime().availableProcessors())
 
+// Must be in sync with TestVersions.kt KTI-1612
+val gradleVersions = listOf(
+    "6.8.3",
+    "6.9.4",
+    "7.0.2",
+    "7.1.1",
+    "7.2",
+    "7.3.3",
+    "7.4.2",
+    "7.5.1",
+    "7.6.3",
+    "8.0.2",
+    "8.1.1",
+    "8.2.1",
+    "8.3",
+    "8.4",
+    "8.5",
+    "8.6"
+)
+val junitTags = listOf("JvmKGP", "DaemonsKGP", "JsKGP", "NativeKGP", "MppKGP", "AndroidKGP", "OtherKGP")
+val requiresKotlinNative = listOf("NativeKGP", "MppKGP", "OtherKGP")
+val gradleVersionTaskGroup = "Kotlin Gradle Plugin Verification grouped by Gradle version"
+
+junitTags.forEach { junitTag ->
+    val taskPrefix = "kgp${junitTag.substringBefore("KGP")}"
+    val tasksByGradleVersion = gradleVersions.map { gradleVersion ->
+        tasks.register<Test>("${taskPrefix}TestsForGradle_${gradleVersion.replace(".", "_")}") {
+            group = gradleVersionTaskGroup
+            description = "Runs all tests for Kotlin Gradle plugins against Gradle $gradleVersion"
+            maxParallelForks = maxParallelTestForks
+
+            systemProperty("gradle.integration.tests.gradle.version.filter", gradleVersion)
+            if (junitTag in requiresKotlinNative) {
+                applyKotlinNativeFromCurrentBranchIfNeeded()
+            }
+
+            useJUnitPlatform {
+                includeTags(junitTag)
+                excludeTags(*(junitTags - junitTag).toTypedArray())
+                includeEngines("junit-jupiter")
+            }
+        }
+    }
+
+    tasks.register("${taskPrefix}TestsGroupedByGradleVersion") {
+        group = gradleVersionTaskGroup
+        dependsOn(tasksByGradleVersion)
+    }
+}
+
 val allParallelTestsTask = tasks.register<Test>("kgpAllParallelTests") {
     group = KGP_TEST_TASKS_GROUP
     description = "Runs all tests for Kotlin Gradle plugins except daemon ones"

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/java/org/jetbrains/kotlin/gradle/testbase/GradleAndroidTest.java

@@ -21,5 +21,6 @@ import java.lang.annotation.Target;
 @AndroidTestVersions
 @ParameterizedTest(name = "AGP {1} with {0}: {displayName}")
 @ArgumentsSource(GradleAndAgpArgumentsProvider.class)
+@ExtendWith(DisabledIfNoArgumentsProvided.class)
 public @interface GradleAndroidTest {
 }

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/java/org/jetbrains/kotlin/gradle/testbase/GradleTest.java

@@ -20,5 +20,6 @@ import java.lang.annotation.Target;
 @GradleTestVersions
 @ParameterizedTest(name = "{0}: {displayName}")
 @ArgumentsSource(GradleArgumentsProvider.class)
+@ExtendWith(DisabledIfNoArgumentsProvided.class)
 public @interface GradleTest {
 }

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/java/org/jetbrains/kotlin/gradle/testbase/GradleWithJdkTest.java

@@ -21,5 +21,6 @@ import java.lang.annotation.Target;
 @JdkVersions
 @ParameterizedTest(name = "{1} with {0}: {displayName}")
 @ArgumentsSource(GradleAndJdkArgumentsProvider.class)
+@ExtendWith(DisabledIfNoArgumentsProvided.class)
 public @interface GradleWithJdkTest {
 }

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/HierarchicalStructureOptOutIT.kt

@@ -104,5 +104,6 @@ internal class HierarchicalStructureOptInMigrationArtifactContentMppIT : KGPBase
     @GradleTestVersions
     @ParameterizedTest(name = "{0} mode {1}: {displayName}")
     @ArgumentsSource(GradleAndHmppModeProvider::class)
+    @ExtendWith(DisabledIfNoArgumentsProvided::class)
     annotation class GradleWithHmppModeTest
 }

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinSpecificDependenciesIT.kt

@@ -513,6 +513,7 @@ class KotlinSpecificDependenciesIT : KGPBaseTest() {
     @GradleTestVersions
     @ParameterizedTest(name = "{1} with {0}: {displayName}")
     @ArgumentsSource(GradleAndTestFrameworksArgumentsProvider::class)
+    @ExtendWith(DisabledIfNoArgumentsProvided::class)
     fun testFrameworkSelectionJvm(
         gradleVersion: GradleVersion,
         testFramework: Pair<String, String>
@@ -536,6 +537,7 @@ class KotlinSpecificDependenciesIT : KGPBaseTest() {
     @GradleTestVersions
     @ParameterizedTest(name = "{1} with {0}: {displayName}")
     @ArgumentsSource(GradleAndTestFrameworksArgumentsProvider::class)
+    @ExtendWith(DisabledIfNoArgumentsProvided::class)
     fun testFrameworkSelectionMppJvm(
         gradleVersion: GradleVersion,
         testFramework: Pair<String, String>
@@ -560,6 +562,7 @@ class KotlinSpecificDependenciesIT : KGPBaseTest() {
     @GradleTestVersions
     @ParameterizedTest(name = "{1} with {0}: {displayName}")
     @ArgumentsSource(GradleAndTestFrameworksArgumentsProvider::class)
+    @ExtendWith(DisabledIfNoArgumentsProvided::class)
     fun testFrameworkSelectionMppCommon(
         gradleVersion: GradleVersion,
         testFramework: Pair<String, String>

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/MppHighlightingTestDataWithGradleIT.kt

@@ -238,5 +238,6 @@ internal class MppHighlightingTestDataWithGradleIT : KGPBaseTest() {
     @GradleTestVersions
     @ParameterizedTest(name = "{0} - {1} - {2} - {3}: {displayName}")
     @ArgumentsSource(GradleAndMppHighlightingProvider::class)
+    @ExtendWith(DisabledIfNoArgumentsProvided::class)
     annotation class GradleWithMppHighlightingTest
 }

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/apple/XcodeDirectIntegrationIT.kt

@@ -24,6 +24,7 @@ class XcodeDirectIntegrationIT : KGPBaseTest() {
     @DisplayName("Xcode direct integration")
     @ParameterizedTest(name = "{displayName} with {1}, {0} and isStatic={2}")
     @ArgumentsSource(XcodeArgumentsProvider::class)
+    @ExtendWith(DisabledIfNoArgumentsProvided::class)
     fun test(
         gradleVersion: GradleVersion,
         iosApp: String,

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/TestVersions.kt

@@ -9,6 +9,7 @@ import org.gradle.api.JavaVersion
 import org.gradle.util.GradleVersion
 import org.jetbrains.kotlin.gradle.utils.minSupportedGradleVersion
 
+// Must be in sync with gradleVersions in libraries/tools/kotlin-gradle-plugin-integration-tests/build.gradle.kts KTI-1612
 interface TestVersions {
 
     // https://gradle.org/nightly/

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/argumentProviders.kt

@@ -7,20 +7,26 @@ package org.jetbrains.kotlin.gradle.testbase
 
 import org.gradle.api.JavaVersion
 import org.gradle.util.GradleVersion
+import org.junit.jupiter.api.extension.ConditionEvaluationResult
+import org.junit.jupiter.api.extension.ExecutionCondition
 import org.junit.jupiter.api.extension.ExtensionContext
 import org.junit.jupiter.params.provider.Arguments
 import org.junit.jupiter.params.provider.ArgumentsProvider
+import org.junit.jupiter.params.provider.ArgumentsSource
+import org.junit.jupiter.params.support.AnnotationConsumerInitializer
+import org.junit.platform.commons.JUnitException
+import org.junit.platform.commons.util.AnnotationUtils
+import org.junit.platform.commons.util.ReflectionUtils
 import java.io.File
 import java.util.stream.Stream
 import kotlin.streams.asStream
-import kotlin.streams.toList
 
 @Target(AnnotationTarget.FUNCTION, AnnotationTarget.ANNOTATION_CLASS, AnnotationTarget.CLASS)
 @Retention(AnnotationRetention.RUNTIME)
 annotation class GradleTestVersions(
     val minVersion: String = TestVersions.Gradle.MIN_SUPPORTED,
     val maxVersion: String = TestVersions.Gradle.MAX_SUPPORTED,
-    val additionalVersions: Array<String> = []
+    val additionalVersions: Array<String> = [],
 )
 
 inline fun <reified T : Annotation> findAnnotation(context: ExtensionContext): T {
@@ -54,8 +60,20 @@ inline fun <reified T : Annotation> findAnnotation(context: ExtensionContext): T
 
 open class GradleArgumentsProvider : ArgumentsProvider {
     override fun provideArguments(
-        context: ExtensionContext
+        context: ExtensionContext,
     ): Stream<out Arguments> {
+        val gradleVersions = gradleVersions(context)
+        val versionFilter = context.getConfigurationParameter("gradle.integration.tests.gradle.version.filter")
+            .map { GradleVersion.version(it) }
+
+        return gradleVersions
+            .asSequence()
+            .filter { gradleVersion -> versionFilter.map { gradleVersion == it }.orElse(true) }
+            .map { Arguments.of(it) }
+            .asStream()
+    }
+
+    protected fun gradleVersions(context: ExtensionContext): Set<GradleVersion> {
         val versionsAnnotation = findAnnotation<GradleTestVersions>(context)
 
         fun max(a: GradleVersion, b: GradleVersion) = if (a >= b) a else b
@@ -73,9 +91,6 @@ open class GradleArgumentsProvider : ArgumentsProvider {
         }
 
         return setOf(minGradleVersion, *additionalGradleVersions.toTypedArray(), maxGradleVersion)
-            .asSequence()
-            .map { Arguments.of(it) }
-            .asStream()
     }
 }
 
@@ -83,11 +98,11 @@ open class GradleArgumentsProvider : ArgumentsProvider {
 @Retention(AnnotationRetention.RUNTIME)
 annotation class JdkVersions(
     val versions: Array<JavaVersion> = [JavaVersion.VERSION_1_8, JavaVersion.VERSION_21],
-    val compatibleWithGradle: Boolean = true
+    val compatibleWithGradle: Boolean = true,
 ) {
     class ProvidedJdk(
         val version: JavaVersion,
-        val location: File
+        val location: File,
     ) {
         override fun toString(): String {
             return "JDK $version"
@@ -97,7 +112,7 @@ annotation class JdkVersions(
 
 class GradleAndJdkArgumentsProvider : GradleArgumentsProvider() {
     override fun provideArguments(
-        context: ExtensionContext
+        context: ExtensionContext,
     ): Stream<out Arguments> {
         val jdkAnnotation = findAnnotation<JdkVersions>(context)
         val providedJdks = jdkAnnotation
@@ -109,7 +124,9 @@ class GradleAndJdkArgumentsProvider : GradleArgumentsProvider() {
                 )
             }
 
-        val gradleVersions = super.provideArguments(context).map { it.get().first() as GradleVersion }.toList()
+        val gradleVersions = gradleVersions(context)
+        val versionFilter = context.getConfigurationParameter("gradle.integration.tests.gradle.version.filter")
+            .map { GradleVersion.version(it) }
 
         return providedJdks
             .flatMap { providedJdk ->
@@ -129,6 +146,7 @@ class GradleAndJdkArgumentsProvider : GradleArgumentsProvider() {
                     .map { it to providedJdk }
             }
             .asSequence()
+            .filter { (gradleVersion, _) -> versionFilter.map { gradleVersion == it }.orElse(true) }
             .map {
                 Arguments.of(it.first, it.second)
             }
@@ -149,12 +167,12 @@ class GradleAndJdkArgumentsProvider : GradleArgumentsProvider() {
 annotation class AndroidTestVersions(
     val minVersion: String = TestVersions.AGP.MIN_SUPPORTED,
     val maxVersion: String = TestVersions.AGP.MAX_SUPPORTED,
-    val additionalVersions: Array<String> = []
+    val additionalVersions: Array<String> = [],
 )
 
 class GradleAndAgpArgumentsProvider : GradleArgumentsProvider() {
     override fun provideArguments(
-        context: ExtensionContext
+        context: ExtensionContext,
     ): Stream<out Arguments> {
         val agpVersionsAnnotation = findAnnotation<AndroidTestVersions>(context)
         val agpVersions = setOfNotNull(
@@ -163,11 +181,13 @@ class GradleAndAgpArgumentsProvider : GradleArgumentsProvider() {
             if (agpVersionsAnnotation.minVersion < agpVersionsAnnotation.maxVersion) agpVersionsAnnotation.maxVersion else null
         )
 
-        val gradleVersions = super.provideArguments(context).map { it.get().first() as GradleVersion }.toList()
+        val gradleVersions = gradleVersions(context)
+        val versionFilter = context.getConfigurationParameter("gradle.integration.tests.gradle.version.filter")
+            .map { GradleVersion.version(it) }
 
         return agpVersions
             .flatMap { version ->
-                val agpVersion = TestVersions.AgpCompatibilityMatrix.values().find { it.version == version }
+                val agpVersion = TestVersions.AgpCompatibilityMatrix.entries.find { it.version == version }
                     ?: throw IllegalArgumentException("AGP version $version is not defined in TestVersions.AGP!")
 
                 val providedJdk = JdkVersions.ProvidedJdk(
@@ -186,6 +206,7 @@ class GradleAndAgpArgumentsProvider : GradleArgumentsProvider() {
                     }
             }
             .asSequence()
+            .filter { agpTestArguments -> versionFilter.map { agpTestArguments.gradleVersion == it }.orElse(true) }
             .map {
                 Arguments.of(it.gradleVersion, it.agpVersion, it.jdkVersion)
             }
@@ -195,6 +216,41 @@ class GradleAndAgpArgumentsProvider : GradleArgumentsProvider() {
     data class AgpTestArguments(
         val gradleVersion: GradleVersion,
         val agpVersion: String,
-        val jdkVersion: JdkVersions.ProvidedJdk
+        val jdkVersion: JdkVersions.ProvidedJdk,
     )
 }
+
+class DisabledIfNoArgumentsProvided : ExecutionCondition {
+    override fun evaluateExecutionCondition(context: ExtensionContext): ConditionEvaluationResult {
+        val gradleVersionFilterParameter = context.getConfigurationParameter("gradle.integration.tests.gradle.version.filter")
+        if (!gradleVersionFilterParameter.isPresent)
+            return ConditionEvaluationResult.enabled("No Gradle version filter provided")
+
+        val argumentProviders = AnnotationUtils.findRepeatableAnnotations(context.requiredTestMethod, ArgumentsSource::class.java)
+            .map(ArgumentsSource::value)
+            .map { instantiateArgumentsProvider(it.java) }
+            .map { provider -> AnnotationConsumerInitializer.initialize(context.requiredTestMethod, provider) }
+
+        return if (argumentProviders.any { it.provideArguments(context).count() == 0L })
+            ConditionEvaluationResult.disabled("No arguments provided")
+        else
+            ConditionEvaluationResult.enabled("Arguments provided")
+    }
+
+    private fun instantiateArgumentsProvider(clazz: Class<out ArgumentsProvider>): ArgumentsProvider {
+        try {
+            return ReflectionUtils.newInstance(clazz)
+        } catch (ex: Exception) {
+            if (ex is NoSuchMethodException) {
+                val message = String.format(
+                    "Failed to find a no-argument constructor for ArgumentsProvider [%s]. "
+                            + "Please ensure that a no-argument constructor exists and "
+                            + "that the class is either a top-level class or a static nested class",
+                    clazz.getName()
+                )
+                throw JUnitException(message, ex)
+            }
+            throw ex
+        }
+    }
+}