diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt index 9b332b32433..56ed12ae9bc 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt @@ -354,7 +354,18 @@ internal object NativeTestSupport { val enclosingTestClass = enclosingTestClass val testProcessSettings = getOrCreateTestProcessSettings() - val computedTestConfiguration = computeTestConfiguration(enclosingTestClass) + val computedTestConfiguration = computeTestConfiguration(enclosingTestClass).run { + if (TestGroupCreation.getFromProperty() == TestGroupCreation.EAGER && + configuration.providerClass == ExtTestCaseGroupProvider::class + ) { + val annotation = UseEagerExtTestCaseGroupProvider() + val testConfiguration = annotation.annotationClass.findAnnotation() + ?: error("Unable to find annotation for Eager test group creation") + ComputedTestConfiguration(testConfiguration, annotation) + } else { + this + } + } val settings = buildList { // Put common settings: diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/TestCase.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/TestCase.kt index b4925efc863..d05b9690514 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/TestCase.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/TestCase.kt @@ -292,7 +292,7 @@ internal interface TestCaseGroupId { * [TestCase]s inside of the group with similar [TestCompilerArgs] can be compiled to the single * executable file to reduce the time spent for compiling and speed-up overall test execution. */ -internal interface TestCaseGroup { +internal sealed interface TestCaseGroup { fun isEnabled(testCaseId: TestCaseId): Boolean fun getByName(testCaseId: TestCaseId): TestCase? @@ -323,18 +323,28 @@ internal interface TestCaseGroup { } } - companion object { - val ALL_DISABLED = object : TestCaseGroup { - override fun isEnabled(testCaseId: TestCaseId) = false - override fun getByName(testCaseId: TestCaseId) = unsupported() + data class MetaGroup(val testCaseGroupId: TestCaseGroupId, val testGroups: Set) : TestCaseGroup { + override fun isEnabled(testCaseId: TestCaseId): Boolean = testGroups.all { it.isEnabled(testCaseId) } - override fun getRegularOnly( - freeCompilerArgs: TestCompilerArgs, - sharedModules: Set, - runnerType: TestRunnerType - ) = unsupported() + override fun getByName(testCaseId: TestCaseId): TestCase? = testGroups.firstNotNullOfOrNull { it.getByName(testCaseId) } - private fun unsupported(): Nothing = fail { "This function should not be called" } - } + override fun getRegularOnly( + freeCompilerArgs: TestCompilerArgs, + sharedModules: Set, + runnerType: TestRunnerType, + ): Collection = testGroups.flatMap { it.getRegularOnly(freeCompilerArgs, sharedModules, runnerType) } + } + + data object AllDisabled : TestCaseGroup { + override fun isEnabled(testCaseId: TestCaseId) = false + override fun getByName(testCaseId: TestCaseId) = unsupported() + + override fun getRegularOnly( + freeCompilerArgs: TestCompilerArgs, + sharedModules: Set, + runnerType: TestRunnerType + ) = unsupported() + + private fun unsupported(): Nothing = fail { "This function should not be called" } } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/EagerExtTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/EagerExtTestCaseGroupProvider.kt new file mode 100644 index 00000000000..d49bcc9fd1e --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/EagerExtTestCaseGroupProvider.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.test.blackbox.support.group + +import org.jetbrains.kotlin.konan.test.blackbox.support.TestCaseGroup +import org.jetbrains.kotlin.konan.test.blackbox.support.TestCaseGroupId +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.ExternalSourceTransformersProvider +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.Settings +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.TestRoots +import org.jetbrains.kotlin.konan.test.blackbox.support.util.ExternalSourceTransformers +import org.jetbrains.kotlin.konan.test.blackbox.support.util.ThreadSafeCache +import org.jetbrains.kotlin.test.utils.TransformersFunctions.removeOptionalJvmInlineAnnotation +import java.io.File + +/** + * TestCaseGroup provider that eagerly creates test groups. + * + * Provider creates test groups as big as possible by grouping them by [TestRoots]. + * Comparing to [ExtTestCaseGroupProvider] makes all groups eagerly on first request for the current id's test root. + * + * @see TestCaseGroupProvider + * @see TestCaseGroup.MetaGroup + */ +internal class EagerExtTestCaseGroupProvider : ExtTestCaseGroupProvider() { + private val metaGroup = ThreadSafeCache() + + override fun getTestCaseGroup(testCaseGroupId: TestCaseGroupId, settings: Settings): TestCaseGroup? { + check(testCaseGroupId is TestCaseGroupId.TestDataDir) + + val testRoot = settings.findTestRoot(testCaseGroupId.dir) + val testRootDataDir = TestCaseGroupId.TestDataDir(testRoot) + + return metaGroup.computeIfAbsent(testRootDataDir) { + val groups = testRootDataDir.dir.walkTopDown() + .filter { f -> f.isDirectory } + .mapNotNull { + val extendedSettings = object : Settings( + parent = settings, + settings = listOf(ExternalSourceTransformersProvider::class to JvmInlineAnnotationRemover) + ) {} + super.getTestCaseGroup(TestCaseGroupId.TestDataDir(it), extendedSettings) + }.toSet() + + TestCaseGroup.MetaGroup(testRootDataDir, groups) + } + } + + private fun Settings.findTestRoot(file: File) = + get().roots.single { + val fileComponents = file.absolutePath.split(File.separator) + val rootComponents = it.absolutePath.split(File.separator) + + fileComponents.forEachIndexed { index, s -> + if (index < rootComponents.size && s != rootComponents[index]) { + return@single false + } + } + true + } + + /* + * It is necessary to use this source processor as soon as it is being used in inline classes tests. + * This processor is registered in the class constructor, but during the test grouping it is not accessible. + * This happens because we eagerly iterate through the test data, while JUnit does not create actual test instances. + */ + private object JvmInlineAnnotationRemover : ExternalSourceTransformersProvider { + override fun getSourceTransformers(testDataFile: File): ExternalSourceTransformers = listOf(removeOptionalJvmInlineAnnotation) + } +} diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/ExtTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/ExtTestCaseGroupProvider.kt index 5651fc3b5b9..06e27baf174 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/ExtTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/ExtTestCaseGroupProvider.kt @@ -58,7 +58,7 @@ import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail import org.jetbrains.kotlin.utils.addIfNotNull import java.io.File -internal class ExtTestCaseGroupProvider : TestCaseGroupProvider, TestDisposable(parentDisposable = null) { +internal open class ExtTestCaseGroupProvider : TestCaseGroupProvider, TestDisposable(parentDisposable = null) { private val structureFactory = ExtTestDataFileStructureFactory(parentDisposable = this) private val sharedModules = ThreadSafeCache() @@ -73,7 +73,7 @@ internal class ExtTestCaseGroupProvider : TestCaseGroupProvider, TestDisposable( val excludes: Set = settings.get().filesAndDirectories if (testDataDir in excludes) - return@computeIfAbsent TestCaseGroup.ALL_DISABLED + return@computeIfAbsent TestCaseGroup.AllDisabled val (excludedTestDataFiles, testDataFiles) = testDataDir.listFiles() ?.filter { file -> file.isFile && file.extension == "kt" } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/StandardTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/StandardTestCaseGroupProvider.kt index 9c40386aa1b..e57d0f528bf 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/StandardTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/StandardTestCaseGroupProvider.kt @@ -37,7 +37,7 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { val excludes: Set = settings.get().filesAndDirectories if (testDataDir in excludes) - return@computeIfAbsent TestCaseGroup.ALL_DISABLED + return@computeIfAbsent TestCaseGroup.AllDisabled val (excludedTestDataFiles, includedTestDataFiles) = testDataFiles .filter { file -> file.isFile && file.extension == "kt" } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/UseEagerExtTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/UseEagerExtTestCaseGroupProvider.kt new file mode 100644 index 00000000000..a1061fb469a --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/UseEagerExtTestCaseGroupProvider.kt @@ -0,0 +1,18 @@ +/* + * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.test.blackbox.support.group + +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.DisabledTestDataFiles +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.GeneratedSources +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.TestConfiguration +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.TestRoots + +@Target(AnnotationTarget.CLASS) +@TestConfiguration( + providerClass = EagerExtTestCaseGroupProvider::class, + requiredSettings = [TestRoots::class, GeneratedSources::class, DisabledTestDataFiles::class] +) +annotation class UseEagerExtTestCaseGroupProvider \ No newline at end of file diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunProvider.kt index 21d9cbf20d0..18474856f27 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunProvider.kt @@ -38,9 +38,10 @@ internal class TestRunProvider( * Produces a single [TestRun] per [TestCase]. So-called "one test case/one test run" mode. * * If [TestCase] contains multiple functions annotated with [kotlin.test.Test], then all these functions will be executed - * in one shot. If either function will fail, the whole JUnit test will be considered as failed. + * in one shot. If either function fails, the whole JUnit test will be considered as failed. * * Example: + * ``` * //+++ testData file (foo.kt): +++// * @kotlin.test.Test * fun one() { /* ... */ } @@ -58,6 +59,7 @@ internal class TestRunProvider( * // If either of test functions fails, the whole "testFoo()" JUnit test is marked as failed. * } * } + * ``` */ fun getSingleTestRun( testCaseId: TestCaseId, @@ -72,10 +74,11 @@ internal class TestRunProvider( * If [TestCase] contains multiple functions annotated with [kotlin.test.Test], then a separate [TestRun] will be produced * for each such function. * - * This allows to have a better granularity in tests. So that every individual test method inside [TestCase] will be considered + * This allows having a better granularity in tests. So that every test method inside [TestCase] will be considered * as an individual JUnit test, and will be presented as a separate row in JUnit test report. * * Example: + * ``` * //+++ testData file (foo.kt): +++// * @kotlin.test.Test * fun one() { /* ... */ } @@ -95,6 +98,7 @@ internal class TestRunProvider( * // in the test report, and "testFoo.two" will be presented as passed. * } * } + * ``` */ fun getTestRuns( testCaseId: TestCaseId, @@ -129,6 +133,11 @@ internal class TestRunProvider( val testCase = testCaseGroup.getByName(testCaseId) ?: fail { "No test case for $testCaseId" } + val testCaseGroupId = if (testCaseGroup is TestCaseGroup.MetaGroup) + testCaseGroup.testCaseGroupId + else + testCaseId.testCaseGroupId + val testCompilation = when (testCase.kind) { TestKind.STANDALONE, TestKind.STANDALONE_NO_TR, TestKind.STANDALONE_LLDB -> { // Create a separate compilation for each standalone test case. @@ -143,7 +152,7 @@ internal class TestRunProvider( val testRunnerType = testCase.extras().runnerType cachedCompilations.computeIfAbsent( TestCompilationCacheKey.Grouped( - testCaseGroupId = testCaseId.testCaseGroupId, + testCaseGroupId = testCaseGroupId, freeCompilerArgs = testCase.freeCompilerArgs, sharedModules = testCase.sharedModules, runnerType = testRunnerType diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestProcessSettings.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestProcessSettings.kt index fa1eb824834..e902577a7fb 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestProcessSettings.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestProcessSettings.kt @@ -305,6 +305,21 @@ internal enum class CompilerOutputInterceptor { NONE } +internal enum class TestGroupCreation { + DEFAULT, + EAGER; + + companion object { + private const val PROPERTY = "kotlin.internal.native.test.eagerGroupCreation" + + fun getFromProperty(): TestGroupCreation = System.getProperty(PROPERTY) + ?.let { + if (it.toBoolean()) EAGER + else DEFAULT + } ?: DEFAULT + } +} + internal enum class BinaryLibraryKind { STATIC, DYNAMIC } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestRunSettings.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestRunSettings.kt index 7ed73f37e9d..b0d9998a075 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestRunSettings.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestRunSettings.kt @@ -12,7 +12,7 @@ import java.io.File /** * All instances of test classes. * - * [allInstances] - all test class instances ordered from innermost to outermost + * [allInstances] - all test class instances ordered from outermost to innermost * [enclosingTestInstance] - the outermost test instance */ internal class BlackBoxTestInstances(val allInstances: List) { diff --git a/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt b/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt index 454071115dd..e2e77455b9a 100644 --- a/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt +++ b/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt @@ -28,6 +28,7 @@ private enum class TestProperty(shortName: String) { EXECUTION_TIMEOUT("executionTimeout"), SANITIZER("sanitizer"), SHARED_TEST_EXECUTION("sharedTestExecution"), + EAGER_GROUP_CREATION("eagerGroupCreation"), TEAMCITY("teamcity"); val fullName = "kotlin.internal.native.test.$shortName" @@ -197,6 +198,7 @@ fun Project.nativeTest( compute(EXECUTION_TIMEOUT) compute(SANITIZER) compute(SHARED_TEST_EXECUTION) + compute(EAGER_GROUP_CREATION) // Pass whether tests are running at TeamCity. computePrivate(TEAMCITY) { kotlinBuildProperties.isTeamcityBuild.toString() }