azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N][test] Eager test group creation based on test roots

Browse Source 
This commit adds an option to make test grouping eagerly create larger
groups of tests. Each MetaGroup is created based on the test root the
test file is located in. Test compilation tries to compile all
compatible tests in the group into the final binary/executable.

This grouping strategy allows infrastructure to reduce the number of
produced artifacts, and along with running tests from the same binary,
reduce execution time.

This is a part of ^KT-58928 to implement running tests on iOS devices.
This commit is contained in:
Pavel Punegov
2024-02-05 21:05:13 +01:00
committed by Space Team
parent 4d723bfa85
commit 3cc117336a
10 changed files with 157 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt
+12 -1
View File
@@ -354,7 +354,18 @@ internal object NativeTestSupport {
val enclosingTestClass = enclosingTestClass
val testProcessSettings = getOrCreateTestProcessSettings()
val computedTestConfiguration = computeTestConfiguration(enclosingTestClass)
val computedTestConfiguration = computeTestConfiguration(enclosingTestClass).run {
if (TestGroupCreation.getFromProperty() == TestGroupCreation.EAGER &&
configuration.providerClass == ExtTestCaseGroupProvider::class
) {
val annotation = UseEagerExtTestCaseGroupProvider()
val testConfiguration = annotation.annotationClass.findAnnotation<TestConfiguration>()
?: error("Unable to find annotation for Eager test group creation")
ComputedTestConfiguration(testConfiguration, annotation)
} else {
this
}
}
val settings = buildList {
// Put common settings:
native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/TestCase.kt
+22 -12
View File
@@ -292,7 +292,7 @@ internal interface TestCaseGroupId {
* [TestCase]s inside of the group with similar [TestCompilerArgs] can be compiled to the single
* executable file to reduce the time spent for compiling and speed-up overall test execution.
*/
internal interface TestCaseGroup {
internal sealed interface TestCaseGroup {
fun isEnabled(testCaseId: TestCaseId): Boolean
fun getByName(testCaseId: TestCaseId): TestCase?
@@ -323,18 +323,28 @@ internal interface TestCaseGroup {
}
}
companion object {
val ALL_DISABLED = object : TestCaseGroup {
override fun isEnabled(testCaseId: TestCaseId) = false
override fun getByName(testCaseId: TestCaseId) = unsupported()
data class MetaGroup(val testCaseGroupId: TestCaseGroupId, val testGroups: Set<TestCaseGroup>) : TestCaseGroup {
override fun isEnabled(testCaseId: TestCaseId): Boolean = testGroups.all { it.isEnabled(testCaseId) }
override fun getRegularOnly(
freeCompilerArgs: TestCompilerArgs,
sharedModules: Set<TestModule.Shared>,
runnerType: TestRunnerType
) = unsupported()
override fun getByName(testCaseId: TestCaseId): TestCase? = testGroups.firstNotNullOfOrNull { it.getByName(testCaseId) }
private fun unsupported(): Nothing = fail { "This function should not be called" }
}
override fun getRegularOnly(
freeCompilerArgs: TestCompilerArgs,
sharedModules: Set<TestModule.Shared>,
runnerType: TestRunnerType,
): Collection<TestCase> = testGroups.flatMap { it.getRegularOnly(freeCompilerArgs, sharedModules, runnerType) }
}
data object AllDisabled : TestCaseGroup {
override fun isEnabled(testCaseId: TestCaseId) = false
override fun getByName(testCaseId: TestCaseId) = unsupported()
override fun getRegularOnly(
freeCompilerArgs: TestCompilerArgs,
sharedModules: Set<TestModule.Shared>,
runnerType: TestRunnerType
) = unsupported()
private fun unsupported(): Nothing = fail { "This function should not be called" }
}
}
native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/EagerExtTestCaseGroupProvider.kt
+72
View File
@@ -0,0 +1,72 @@
/*
* Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.konan.test.blackbox.support.group
import org.jetbrains.kotlin.konan.test.blackbox.support.TestCaseGroup
import org.jetbrains.kotlin.konan.test.blackbox.support.TestCaseGroupId
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.ExternalSourceTransformersProvider
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.Settings
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.TestRoots
import org.jetbrains.kotlin.konan.test.blackbox.support.util.ExternalSourceTransformers
import org.jetbrains.kotlin.konan.test.blackbox.support.util.ThreadSafeCache
import org.jetbrains.kotlin.test.utils.TransformersFunctions.removeOptionalJvmInlineAnnotation
import java.io.File
/**
* TestCaseGroup provider that eagerly creates test groups.
*
* Provider creates test groups as big as possible by grouping them by [TestRoots].
* Comparing to [ExtTestCaseGroupProvider] makes all groups eagerly on first request for the current id's test root.
*
* @see TestCaseGroupProvider
* @see TestCaseGroup.MetaGroup
*/
internal class EagerExtTestCaseGroupProvider : ExtTestCaseGroupProvider() {
private val metaGroup = ThreadSafeCache<TestCaseGroupId.TestDataDir, TestCaseGroup.MetaGroup?>()
override fun getTestCaseGroup(testCaseGroupId: TestCaseGroupId, settings: Settings): TestCaseGroup? {
check(testCaseGroupId is TestCaseGroupId.TestDataDir)
val testRoot = settings.findTestRoot(testCaseGroupId.dir)
val testRootDataDir = TestCaseGroupId.TestDataDir(testRoot)
return metaGroup.computeIfAbsent(testRootDataDir) {
val groups = testRootDataDir.dir.walkTopDown()
.filter { f -> f.isDirectory }
.mapNotNull {
val extendedSettings = object : Settings(
parent = settings,
settings = listOf(ExternalSourceTransformersProvider::class to JvmInlineAnnotationRemover)
) {}
super.getTestCaseGroup(TestCaseGroupId.TestDataDir(it), extendedSettings)
}.toSet()
TestCaseGroup.MetaGroup(testRootDataDir, groups)
}
}
private fun Settings.findTestRoot(file: File) =
get<TestRoots>().roots.single {
val fileComponents = file.absolutePath.split(File.separator)
val rootComponents = it.absolutePath.split(File.separator)
fileComponents.forEachIndexed { index, s ->
if (index < rootComponents.size && s != rootComponents[index]) {
return@single false
}
}
true
}
/*
* It is necessary to use this source processor as soon as it is being used in inline classes tests.
* This processor is registered in the class constructor, but during the test grouping it is not accessible.
* This happens because we eagerly iterate through the test data, while JUnit does not create actual test instances.
*/
private object JvmInlineAnnotationRemover : ExternalSourceTransformersProvider {
override fun getSourceTransformers(testDataFile: File): ExternalSourceTransformers = listOf(removeOptionalJvmInlineAnnotation)
}
}
native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/ExtTestCaseGroupProvider.kt
+2 -2
View File
@@ -58,7 +58,7 @@ import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
import org.jetbrains.kotlin.utils.addIfNotNull
import java.io.File
internal class ExtTestCaseGroupProvider : TestCaseGroupProvider, TestDisposable(parentDisposable = null) {
internal open class ExtTestCaseGroupProvider : TestCaseGroupProvider, TestDisposable(parentDisposable = null) {
private val structureFactory = ExtTestDataFileStructureFactory(parentDisposable = this)
private val sharedModules = ThreadSafeCache<String, TestModule.Shared?>()
@@ -73,7 +73,7 @@ internal class ExtTestCaseGroupProvider : TestCaseGroupProvider, TestDisposable(
val excludes: Set<File> = settings.get<DisabledTestDataFiles>().filesAndDirectories
if (testDataDir in excludes)
return@computeIfAbsent TestCaseGroup.ALL_DISABLED
return@computeIfAbsent TestCaseGroup.AllDisabled
val (excludedTestDataFiles, testDataFiles) = testDataDir.listFiles()
?.filter { file -> file.isFile && file.extension == "kt" }
native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/StandardTestCaseGroupProvider.kt
+1 -1
View File
@@ -37,7 +37,7 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
val excludes: Set<File> = settings.get<DisabledTestDataFiles>().filesAndDirectories
if (testDataDir in excludes)
return@computeIfAbsent TestCaseGroup.ALL_DISABLED
return@computeIfAbsent TestCaseGroup.AllDisabled
val (excludedTestDataFiles, includedTestDataFiles) = testDataFiles
.filter { file -> file.isFile && file.extension == "kt" }
native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/UseEagerExtTestCaseGroupProvider.kt
+18
View File
@@ -0,0 +1,18 @@
/*
* Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.konan.test.blackbox.support.group
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.DisabledTestDataFiles
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.GeneratedSources
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.TestConfiguration
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.TestRoots
@Target(AnnotationTarget.CLASS)
@TestConfiguration(
providerClass = EagerExtTestCaseGroupProvider::class,
requiredSettings = [TestRoots::class, GeneratedSources::class, DisabledTestDataFiles::class]
)
annotation class UseEagerExtTestCaseGroupProvider
native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunProvider.kt
+12 -3
View File
@@ -38,9 +38,10 @@ internal class TestRunProvider(
* Produces a single [TestRun] per [TestCase]. So-called "one test case/one test run" mode.
*
* If [TestCase] contains multiple functions annotated with [kotlin.test.Test], then all these functions will be executed
* in one shot. If either function will fail, the whole JUnit test will be considered as failed.
* in one shot. If either function fails, the whole JUnit test will be considered as failed.
*
* Example:
* ```
* //+++ testData file (foo.kt): +++//
* @kotlin.test.Test
* fun one() { /* ... */ }
@@ -58,6 +59,7 @@ internal class TestRunProvider(
* // If either of test functions fails, the whole "testFoo()" JUnit test is marked as failed.
* }
* }
* ```
*/
fun getSingleTestRun(
testCaseId: TestCaseId,
@@ -72,10 +74,11 @@ internal class TestRunProvider(
* If [TestCase] contains multiple functions annotated with [kotlin.test.Test], then a separate [TestRun] will be produced
* for each such function.
*
* This allows to have a better granularity in tests. So that every individual test method inside [TestCase] will be considered
* This allows having a better granularity in tests. So that every test method inside [TestCase] will be considered
* as an individual JUnit test, and will be presented as a separate row in JUnit test report.
*
* Example:
* ```
* //+++ testData file (foo.kt): +++//
* @kotlin.test.Test
* fun one() { /* ... */ }
@@ -95,6 +98,7 @@ internal class TestRunProvider(
* // in the test report, and "testFoo.two" will be presented as passed.
* }
* }
* ```
*/
fun getTestRuns(
testCaseId: TestCaseId,
@@ -129,6 +133,11 @@ internal class TestRunProvider(
val testCase = testCaseGroup.getByName(testCaseId) ?: fail { "No test case for $testCaseId" }
val testCaseGroupId = if (testCaseGroup is TestCaseGroup.MetaGroup)
testCaseGroup.testCaseGroupId
else
testCaseId.testCaseGroupId
val testCompilation = when (testCase.kind) {
TestKind.STANDALONE, TestKind.STANDALONE_NO_TR, TestKind.STANDALONE_LLDB -> {
// Create a separate compilation for each standalone test case.
@@ -143,7 +152,7 @@ internal class TestRunProvider(
val testRunnerType = testCase.extras<WithTestRunnerExtras>().runnerType
cachedCompilations.computeIfAbsent(
TestCompilationCacheKey.Grouped(
testCaseGroupId = testCaseId.testCaseGroupId,
testCaseGroupId = testCaseGroupId,
freeCompilerArgs = testCase.freeCompilerArgs,
sharedModules = testCase.sharedModules,
runnerType = testRunnerType
native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestProcessSettings.kt
+15
View File
@@ -305,6 +305,21 @@ internal enum class CompilerOutputInterceptor {
NONE
}
internal enum class TestGroupCreation {
DEFAULT,
EAGER;
companion object {
private const val PROPERTY = "kotlin.internal.native.test.eagerGroupCreation"
fun getFromProperty(): TestGroupCreation = System.getProperty(PROPERTY)
?.let {
if (it.toBoolean()) EAGER
else DEFAULT
} ?: DEFAULT
}
}
internal enum class BinaryLibraryKind {
STATIC, DYNAMIC
}
native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestRunSettings.kt
+1 -1
View File
@@ -12,7 +12,7 @@ import java.io.File
/**
* All instances of test classes.
*
* [allInstances] - all test class instances ordered from innermost to outermost
* [allInstances] - all test class instances ordered from outermost to innermost
* [enclosingTestInstance] - the outermost test instance
*/
internal class BlackBoxTestInstances(val allInstances: List<Any>) {
repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt
+2
View File
@@ -28,6 +28,7 @@ private enum class TestProperty(shortName: String) {
EXECUTION_TIMEOUT("executionTimeout"),
SANITIZER("sanitizer"),
SHARED_TEST_EXECUTION("sharedTestExecution"),
EAGER_GROUP_CREATION("eagerGroupCreation"),
TEAMCITY("teamcity");
val fullName = "kotlin.internal.native.test.$shortName"
@@ -197,6 +198,7 @@ fun Project.nativeTest(
compute(EXECUTION_TIMEOUT)
compute(SANITIZER)
compute(SHARED_TEST_EXECUTION)
compute(EAGER_GROUP_CREATION)
// Pass whether tests are running at TeamCity.
computePrivate(TEAMCITY) { kotlinBuildProperties.isTeamcityBuild.toString() }