[BTA tests] Split compatibility tests and business logic test suits

^KT-61860 In Progress

compiler/build-tools/kotlin-build-tools-api-tests/README.md

@@ -23,7 +23,7 @@ the Build Tools API's and its implementations' versions.
 Few rules you should follow while writing tests:
 - All tests should be written using [JUnit 5 platform](https://junit.org/junit5/docs/current/user-guide/#overview).
 - All the compilation test classes should extend [BaseCompilationTest](./src/testCommon/kotlin/compilation/model/BaseCompilationTest.kt)
-- Consider using the scenario DSL for the incremental compilation tests, an usage example is located [here](./src/testCommon/kotlin/compilation/ExampleIncrementalScenarioTest.kt)
+- Consider using the scenario DSL for the incremental compilation tests, an usage example is located [here](src/testExample/kotlin/ExampleIncrementalScenarioTest.kt)
 - Add `@DisplayName(...)` with meaningful description both for test class and methods inside. This will allow developers easier
   to understand what test is about.
 - Don't create one big test suite (class). Consider splitting tests into smaller suites. All tests are running in parallel (except daemon tests)

compiler/build-tools/kotlin-build-tools-api-tests/build.gradle.kts

@@ -1,5 +1,3 @@
-import org.gradle.configurationcache.extensions.capitalized
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 import org.jetbrains.kotlin.tooling.core.KotlinToolingVersion
 
 plugins {
@@ -27,58 +25,15 @@ kotlin {
     }
 }
 
-class BuildToolsApiTestSuit(
+val compatibilityTestsVersions = listOf(
-    val testName: String,
+    BuildToolsVersion(KotlinToolingVersion(project.version.toString()), isCurrent = true),
-    val apiVersion: BuildToolsVersion,
+    BuildToolsVersion(KotlinToolingVersion(1, 9, 20, null)),
-    val implVersion: BuildToolsVersion,
-    /**
-     * Tells whether only tests marked with the `CompatibilityTest` JUnit tag must be run
-     */
-    val onlyCompatibilityTests: Boolean = true,
 )
 
-val testMatrix = listOf(
-    BuildToolsApiTestSuit(
-        "example",
-        BuildToolsVersion(KotlinToolingVersion(project.version.toString()), isCurrent = true),
-        BuildToolsVersion(KotlinToolingVersion(project.version.toString()), isCurrent = true),
-        onlyCompatibilityTests = false,
-    ),
-    BuildToolsApiTestSuit(
-        "testSnapshotToSnapshot",
-        BuildToolsVersion(KotlinToolingVersion(project.version.toString()), isCurrent = true),
-        BuildToolsVersion(KotlinToolingVersion(project.version.toString()), isCurrent = true),
-        onlyCompatibilityTests = false,
-    ),
-    BuildToolsApiTestSuit(
-        "test1.9.20ToSnapshot",
-        BuildToolsVersion(KotlinToolingVersion(1, 9, 20, null)),
-        BuildToolsVersion(KotlinToolingVersion(project.version.toString()), isCurrent = true),
-    ),
-    BuildToolsApiTestSuit(
-        "testSnapshotTo1.9.20",
-        BuildToolsVersion(KotlinToolingVersion(project.version.toString()), isCurrent = true),
-        BuildToolsVersion(KotlinToolingVersion(1, 9, 20, null)),
-    ),
-)
-
-val SourceSet.kotlinCompileTask
-    get() = tasks.named<KotlinCompile>("compile${name.capitalized()}Kotlin")
-
 class BuildToolsVersion(val version: KotlinToolingVersion, val isCurrent: Boolean = false) {
     override fun toString() = version.toString()
 }
 
-fun KotlinCompile.ensureCompiledAgainstExpectedBuildToolsApiVersion(version: BuildToolsVersion) {
-    if (version.isCurrent) return
-    // the check is required for the case when Gradle substitutes external dependencies with project ones
-    doFirst {
-        check(libraries.any { "kotlin-build-tools-api-${version}" in it.name }) {
-            "compilation classpath must contain kotlin-build-tools-api:$version"
-        }
-    }
-}
-
 fun Test.ensureExecutedAgainstExpectedBuildToolsImplVersion(version: BuildToolsVersion) {
     if (version.isCurrent) return
     // the check is required for the case when Gradle substitutes external dependencies with project ones
@@ -89,65 +44,81 @@ fun Test.ensureExecutedAgainstExpectedBuildToolsImplVersion(version: BuildToolsV
     }
 }
 
-fun SourceSet.configureApiVersionSourceDirectories() {
+fun SourceSet.configureCompatibilitySourceDirectories() {
     java.setSrcDirs(
         listOf(
-            layout.projectDirectory.dir("src/testCommon/java"),
+            layout.projectDirectory.dir("src/testCompatibility/java"),
         )
     )
     kotlin.setSrcDirs(
         listOf(
-            layout.projectDirectory.dir("src/testCommon/java"),
+            layout.projectDirectory.dir("src/testCompatibility/java"),
-            layout.projectDirectory.dir("src/testCommon/kotlin"),
+            layout.projectDirectory.dir("src/testCompatibility/kotlin"),
         )
     )
     resources.setSrcDirs(
         listOf(
-            layout.projectDirectory.dir("src/testCommon/resources"),
+            layout.projectDirectory.dir("src/testCompatibility/resources"),
         )
     )
 }
 
+val businessLogicTestSuits = setOf(
+    "testExample",
+)
+
 testing {
     suites {
-        for (suitConfig in testMatrix) {
+        for (suit in businessLogicTestSuits) {
-            register<JvmTestSuite>(suitConfig.testName) {
+            register<JvmTestSuite>(suit)
-                sources.configureApiVersionSourceDirectories()
+        }
+
+        for (implVersion in compatibilityTestsVersions) {
+            register<JvmTestSuite>("testCompatibility${implVersion}") {
+                sources.configureCompatibilitySourceDirectories()
                 dependencies {
-                    useJUnitJupiter(libs.versions.junit5.get())
+                    if (implVersion.isCurrent) {
-
-                    compileOnly(project()) // propagate stdlib from the main dependencies for compilation, the runtime dependency provides the actual required version
-                    implementation(project()) {
-                        isTransitive = false
-                    }
-                    implementation(project(":kotlin-tooling-core"))
-
-                    if (suitConfig.apiVersion.isCurrent) {
-                        compileOnly(project(":compiler:build-tools:kotlin-build-tools-api"))
-                    } else {
-                        compileOnly("org.jetbrains.kotlin:kotlin-build-tools-api:${suitConfig.apiVersion}")
-                    }
-                    sources.kotlinCompileTask.configure {
-                        ensureCompiledAgainstExpectedBuildToolsApiVersion(suitConfig.apiVersion)
-                    }
-
-                    if (suitConfig.implVersion.isCurrent) {
                         runtimeOnly(project(":compiler:build-tools:kotlin-build-tools-impl"))
                     } else {
-                        runtimeOnly("org.jetbrains.kotlin:kotlin-build-tools-impl:${suitConfig.implVersion}")
+                        runtimeOnly("org.jetbrains.kotlin:kotlin-build-tools-impl:${implVersion}")
                     }
                 }
-
                 targets.all {
                     projectTest(taskName = testTask.name, jUnitMode = JUnitMode.JUnit5) {
-                        ensureExecutedAgainstExpectedBuildToolsImplVersion(suitConfig.implVersion)
+                        ensureExecutedAgainstExpectedBuildToolsImplVersion(implVersion)
-                        useJUnitPlatform {
+                        systemProperty(
-                            if (suitConfig.onlyCompatibilityTests) {
+                            "kotlin.build-tools-api.impl-version",
-                                includeTags("CompatibilityTests")
+                            implVersion.toString()
-                            }
+                        ) // TODO: remove after KT-63862
-                        }
+                    }
-                        systemProperty("kotlin.build-tools-api.log.level", "DEBUG")
+                }
-                        systemProperty("kotlin.build-tools-api.impl-version", suitConfig.implVersion.toString()) // TODO: remove after KT-63862
+            }
+        }
+
+        withType<JvmTestSuite>().configureEach configureSuit@{
+            val isRegular = this@configureSuit.name in businessLogicTestSuits
+            dependencies {
+                useJUnitJupiter(libs.versions.junit5.get())
+
+                compileOnly(project()) // propagate stdlib from the main dependencies for compilation, the runtime dependency provides the actual required version
+                implementation(project()) {
+                    isTransitive = false
+                }
+                implementation(project(":kotlin-tooling-core"))
+                compileOnly(project(":compiler:build-tools:kotlin-build-tools-api"))
+                if (isRegular) {
+                    runtimeOnly(project(":compiler:build-tools:kotlin-build-tools-impl"))
+                }
+            }
+
+            targets.all {
+                projectTest(taskName = testTask.name, jUnitMode = JUnitMode.JUnit5) {
+                    systemProperty("kotlin.build-tools-api.log.level", "DEBUG")
+                    if (isRegular) {
+                        systemProperty(
+                            "kotlin.build-tools-api.impl-version",
+                            project.version.toString()
+                        ) // TODO: remove after KT-63862
                     }
                 }
             }
@@ -156,5 +127,5 @@ testing {
 }
 
 tasks.named("check") {
-    dependsOn(testing.suites)
+    dependsOn(testing.suites.matching { it.name != "testExample" }) // do not run example tests by default
 }

compiler/build-tools/kotlin-build-tools-api-tests/src/main/kotlin/CompatibilityTests.kt

@@ -1,13 +0,0 @@
-/*
- * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.buildtools.api.tests
-
-import org.junit.jupiter.api.Tag
-
-@Target(AnnotationTarget.CLASS, AnnotationTarget.ANNOTATION_CLASS, AnnotationTarget.FUNCTION)
-@Retention(AnnotationRetention.RUNTIME)
-@Tag("CompatibilityTest")
-annotation class CompatibilityTests

compiler/build-tools/kotlin-build-tools-api-tests/src/testCompatibility/kotlin/ExampleCompatibilityCompilationTest.kt

@@ -3,10 +3,9 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
-package org.jetbrains.kotlin.buildtools.api.tests.compilation
+package org.jetbrains.kotlin.buildtools.api.tests
 
 import org.jetbrains.kotlin.buildtools.api.CompilerExecutionStrategyConfiguration
-import org.jetbrains.kotlin.buildtools.api.tests.CompatibilityTests
 import org.jetbrains.kotlin.buildtools.api.tests.compilation.assertions.assertOutputs
 import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.BaseCompilationTest
 import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.DefaultStrategyAgnosticCompilationTest
@@ -16,7 +15,6 @@ import org.junit.jupiter.api.DisplayName
 
 @Disabled("Example tests for evaluation purposes of the DSL")
 class ExampleCompatibilityCompilationTest : BaseCompilationTest() {
-    @CompatibilityTests
     @DefaultStrategyAgnosticCompilationTest
     @DisplayName("Sample compatibility compilation test that is run as part of each test suit")
     fun myTest(strategyConfig: CompilerExecutionStrategyConfiguration) {

compiler/build-tools/kotlin-build-tools-api-tests/src/testCompatibility/kotlin/ExampleCompatibilityTest.kt

@@ -3,10 +3,9 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
-package org.jetbrains.kotlin.buildtools.api.tests.compilation
+package org.jetbrains.kotlin.buildtools.api.tests
 
 import org.jetbrains.kotlin.buildtools.api.CompilationService
-import org.jetbrains.kotlin.buildtools.api.tests.CompatibilityTests
 import org.junit.jupiter.api.Assertions
 import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.DisplayName
@@ -16,7 +15,6 @@ import org.junit.jupiter.api.Test
 class ExampleCompatibilityTest {
     private val compilationService = CompilationService.loadImplementation(ExampleCompatibilityTest::class.java.classLoader)
 
-    @CompatibilityTests
     @Test
     @DisplayName("Sample compatibility test that is run as part of each test suit")
     fun testDefaultNonIncrementalSettings() {

compiler/build-tools/kotlin-build-tools-api-tests/src/testExample/kotlin/ExampleIncrementalCompilationTest.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
@@ -13,12 +13,10 @@ import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.BaseCompilati
 import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.DefaultStrategyAgnosticCompilationTest
 import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.LogLevel
 import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.project
-import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.DisplayName
 import kotlin.io.path.readText
 import kotlin.io.path.writeText
 
-@Disabled("Example tests for evaluation purposes of the DSL")
 class ExampleIncrementalCompilationTest : BaseCompilationTest() {
     @DisplayName("Sample IC test with a single module")
     @DefaultStrategyAgnosticCompilationTest

compiler/build-tools/kotlin-build-tools-api-tests/src/testExample/kotlin/ExampleIncrementalScenarioTest.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
@@ -10,10 +10,8 @@ import org.jetbrains.kotlin.buildtools.api.tests.compilation.assertions.assertCo
 import org.jetbrains.kotlin.buildtools.api.tests.compilation.scenario.scenario
 import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.BaseCompilationTest
 import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.DefaultStrategyAgnosticCompilationTest
-import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.DisplayName
 
-@Disabled("Example tests for evaluation purposes of the DSL")
 class ExampleIncrementalScenarioTest : BaseCompilationTest() {
     @DefaultStrategyAgnosticCompilationTest
     @DisplayName("Sample scenario DSL IC test with a single module")

compiler/build-tools/kotlin-build-tools-api-tests/src/testExample/kotlin/ExampleNonIncrementalCompilationTest.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
@@ -12,11 +12,9 @@ import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.BaseCompilati
 import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.DefaultStrategyAgnosticCompilationTest
 import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.LogLevel
 import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.project
-import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.DisplayName
 import kotlin.io.path.writeText
 
-@Disabled("Example tests for evaluation purposes of the DSL")
 class ExampleNonIncrementalCompilationTest : BaseCompilationTest() {
     @DisplayName("Sample non-incremental compilation test with two modules")
     @DefaultStrategyAgnosticCompilationTest