[Native][tests] Introduce TestCaseId and TestCaseGroupId

This commit is contained in:
Dmitriy Dolovov
2021-11-29 15:15:17 +03:00
parent b051d2f67d
commit f73ab305e6
9 changed files with 130 additions and 82 deletions
@@ -38,9 +38,18 @@ abstract class AbstractNativeBlackBoxTest {
* This function should be called from a method annotated with [org.junit.jupiter.api.Test].
*/
fun runTest(@TestDataFile testDataFilePath: String) {
val testDataFile = getAbsoluteFile(testDataFilePath)
val testRun = testRunProvider.getSingleTestRun(testDataFile)
runTest(testRun)
val testCaseId = TestCaseId.TestDataFile(getAbsoluteFile(testDataFilePath))
runTestCase(testCaseId)
}
/**
* Run JUnit test.
*
* This function should be called from a method annotated with [org.junit.jupiter.api.Test].
*/
internal fun runTestCase(testCaseId: TestCaseId) {
val testRun = testRunProvider.getSingleTestRun(testCaseId)
performTestRun(testRun)
}
/**
@@ -49,8 +58,17 @@ abstract class AbstractNativeBlackBoxTest {
* This function should be called from a method annotated with [org.junit.jupiter.api.TestFactory].
*/
fun dynamicTest(@TestDataFile testDataFilePath: String): Collection<DynamicNode> {
val testDataFile = getAbsoluteFile(testDataFilePath)
val rootTestRunNode = testRunProvider.getTestRuns(testDataFile)
val testCaseId = TestCaseId.TestDataFile(getAbsoluteFile(testDataFilePath))
return dynamicTestCase(testCaseId)
}
/**
* Run JUnit dynamic test.
*
* This function should be called from a method annotated with [org.junit.jupiter.api.TestFactory].
*/
internal fun dynamicTestCase(testCaseId: TestCaseId): Collection<DynamicNode> {
val rootTestRunNode = testRunProvider.getTestRuns(testCaseId)
return buildJUnitDynamicNodes(rootTestRunNode)
}
@@ -73,7 +91,7 @@ abstract class AbstractNativeBlackBoxTest {
val ownPackageSegment = joinPackageNames(parentPackageSegment, packageSegment)
items.mapTo(this@buildList) { testRun ->
val displayName = testRun.displayName.prependPackageName(ownPackageSegment)
dynamicTest(displayName) { runTest(testRun) }
dynamicTest(displayName) { performTestRun(testRun) }
}
children.forEach { it.processItems(ownPackageSegment) }
@@ -82,7 +100,7 @@ abstract class AbstractNativeBlackBoxTest {
testRunNode.processItems("")
}
private fun runTest(testRun: TestRun) {
private fun performTestRun(testRun: TestRun) {
val testRunner = testRunProvider.createRunner(testRun)
testRunner.run()
}
@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.konan.blackboxtest.support
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.konan.blackboxtest.support.runner.RunResult
import org.jetbrains.kotlin.utils.addToStdlib.safeAs
import java.io.File
import kotlin.time.Duration
import kotlin.time.DurationUnit
@@ -33,16 +34,22 @@ internal abstract class LoggedData {
private val sourceModules: Collection<TestModule>
) : LoggedData() {
private val testDataFiles: List<File>
get() = sourceModules.asSequence()
.filterIsInstance<TestModule.Exclusive>()
.map { it.testCase.origin.testDataFile }
.toMutableList()
.apply { sort() }
get() = buildList {
sourceModules.forEach { module ->
if (module !is TestModule.Exclusive) return@forEach
this += module.testCase.id.safeAs<TestCaseId.TestDataFile>()?.file ?: return@forEach
}
sort()
}
override fun computeText() = buildString {
appendArguments("COMPILER ARGUMENTS:", listOf("\$\$kotlinc-native\$\$") + compilerArgs)
appendLine()
appendList("TEST DATA FILES (COMPILED TOGETHER):", testDataFiles)
val testDataFiles = testDataFiles
if (testDataFiles.isNotEmpty()) {
appendLine()
appendList("TEST DATA FILES (COMPILED TOGETHER):", testDataFiles)
}
}
}
@@ -83,13 +90,18 @@ internal abstract class LoggedData {
class TestRunParameters(
private val compilerCall: CompilerCall,
private val origin: TestOrigin.SingleTestDataFile,
private val testCaseId: TestCaseId,
private val runArgs: Iterable<String>,
private val runParameters: List<TestRunParameter>
) : LoggedData() {
override fun computeText() = buildString {
appendLine("TEST DATA FILE:")
appendLine(origin.testDataFile)
if (testCaseId is TestCaseId.TestDataFile) {
appendLine("TEST DATA FILE:")
appendLine(testCaseId.file)
} else {
appendLine("TEST CASE ID:")
appendLine(testCaseId)
}
appendLine()
appendArguments("TEST RUN ARGUMENTS:", runArgs)
appendLine()
@@ -18,15 +18,6 @@ import java.io.File
internal typealias PackageFQN = String
internal typealias FunctionName = String
/**
* Helps to track the origin of every [TestCase], [TestCompilation] or [TestExecutable]. Used for issue reporting purposes.
*/
internal interface TestOrigin {
class SingleTestDataFile(val testDataFile: File) : TestOrigin {
override fun toString(): String = testDataFile.path
}
}
/**
* Represents a single test function (i.e. a function annotated with [kotlin.test.Test]) inside of a [TestFile].
*/
@@ -154,21 +145,35 @@ internal sealed class TestModule {
}
}
/**
* A unique identifier of [TestCase].
*
* [testCaseGroupId] - a unique ID of [TestCaseGroup] this [TestCase] belongs to.
*/
internal interface TestCaseId {
val testCaseGroupId: TestCaseGroupId
data class TestDataFile(val file: File) : TestCaseId {
override val testCaseGroupId = TestCaseGroupId.TestDataDir(file.parentFile) // The directory, containing testData file.
override fun toString(): String = file.path
}
}
/**
* A collection of one or more [TestModule]s that results in testable executable file.
*
* [modules] - the collection of [TestModule.Exclusive] modules with [TestFile]s that need to be compiled to run this test.
* Note: There can also be [TestModule.Shared] modules as dependencies of either of [TestModule.Exclusive] modules.
* See [TestModule.Exclusive.allDependencies] for details.
* [origin] - the origin of the test case.
* [nominalPackageName] - the unique package name that was computed for this [TestCase] based on [origin]'s actual path.
* [id] - the unique ID of the test case.
* [nominalPackageName] - the unique package name that was computed for this [TestCase] based on [id].
* Note: It depends on the concrete [TestKind] whether the package name will be enforced for the [TestFile]s or not.
*/
internal class TestCase(
val id: TestCaseId,
val kind: TestKind,
val modules: Set<TestModule.Exclusive>,
val freeCompilerArgs: TestCompilerArgs,
val origin: TestOrigin.SingleTestDataFile,
val nominalPackageName: PackageFQN,
val expectedOutputDataFile: File?,
val extras: Extras
@@ -204,11 +209,11 @@ internal class TestCase(
rootModules -= module.allDependencies
}
assertTrue(rootModules.isNotEmpty()) { "No root modules in test case. Origin: $origin." }
assertTrue(rootModules.isNotEmpty()) { "$id: No root modules in test case." }
val nonExclusiveRootTestModules = rootModules.filter { module -> module !is TestModule.Exclusive }
assertTrue(nonExclusiveRootTestModules.isEmpty()) {
"There are non-exclusive root test modules in test case. Origin: $origin. Modules: $nonExclusiveRootTestModules"
"$id: There are non-exclusive root test modules in test case. Modules: $nonExclusiveRootTestModules"
}
@Suppress("UNCHECKED_CAST")
@@ -218,19 +223,19 @@ internal class TestCase(
fun initialize(findSharedModule: ((moduleName: String) -> TestModule.Shared?)?) {
// Check that there are no duplicated files among different modules.
val duplicatedFiles = modules.flatMap { it.files }.groupingBy { it }.eachCount().filterValues { it > 1 }.keys
assertTrue(duplicatedFiles.isEmpty()) { "$origin: Duplicated test files encountered: $duplicatedFiles" }
assertTrue(duplicatedFiles.isEmpty()) { "$id: Duplicated test files encountered: $duplicatedFiles" }
// Check that there are modules with duplicated names.
val exclusiveModules: Map</* regular module name */ String, TestModule.Exclusive> = modules.toIdentitySet()
.groupingBy { module -> module.name }
.aggregate { moduleName, _: TestModule.Exclusive?, module, isFirst ->
assertTrue(isFirst) { "$origin: Multiple test modules with the same name found: $moduleName" }
assertTrue(isFirst) { "$id: Multiple test modules with the same name found: $moduleName" }
module
}
fun findModule(moduleName: String): TestModule = exclusiveModules[moduleName]
?: findSharedModule?.invoke(moduleName)
?: fail { "$origin: Module $moduleName not found" }
?: fail { "$id: Module $moduleName not found" }
modules.forEach { module ->
module.commit() // Save to the file system and release the memory.
@@ -241,6 +246,13 @@ internal class TestCase(
}
}
/**
* A unique identified of [TestCaseGroup].
*/
internal interface TestCaseGroupId {
data class TestDataDir(val dir: File) : TestCaseGroupId
}
/**
* A group of [TestCase]s that were obtained from the same origin (ex: same testData directory).
*
@@ -248,27 +260,27 @@ internal class TestCase(
* executable file to reduce the time spent for compiling and speed-up overall test execution.
*/
internal interface TestCaseGroup {
fun isEnabled(testDataFileName: String): Boolean
fun getByName(testDataFileName: String): TestCase?
fun isEnabled(testCaseId: TestCaseId): Boolean
fun getByName(testCaseId: TestCaseId): TestCase?
fun getRegularOnlyByCompilerArgs(freeCompilerArgs: TestCompilerArgs): Collection<TestCase>
class Default(
private val disabledTestDataFileNames: Set<String>,
private val disabledTestCaseIds: Set<TestCaseId>,
testCases: Iterable<TestCase>
) : TestCaseGroup {
private val testCasesByTestDataFileNames = testCases.associateBy { it.origin.testDataFile.name }
private val testCasesById = testCases.associateBy { it.id }
override fun isEnabled(testDataFileName: String) = testDataFileName !in disabledTestDataFileNames
override fun getByName(testDataFileName: String) = testCasesByTestDataFileNames[testDataFileName]
override fun isEnabled(testCaseId: TestCaseId) = testCaseId !in disabledTestCaseIds
override fun getByName(testCaseId: TestCaseId) = testCasesById[testCaseId]
override fun getRegularOnlyByCompilerArgs(freeCompilerArgs: TestCompilerArgs) =
testCasesByTestDataFileNames.values.filter { it.kind == TestKind.REGULAR && it.freeCompilerArgs == freeCompilerArgs }
testCasesById.values.filter { it.kind == TestKind.REGULAR && it.freeCompilerArgs == freeCompilerArgs }
}
companion object {
val ALL_DISABLED = object : TestCaseGroup {
override fun isEnabled(testDataFileName: String) = false
override fun getByName(testDataFileName: String) = fail { "This function should not be called" }
override fun isEnabled(testCaseId: TestCaseId) = false
override fun getByName(testCaseId: TestCaseId) = fail { "This function should not be called" }
override fun getRegularOnlyByCompilerArgs(freeCompilerArgs: TestCompilerArgs) = fail { "This function should not be called" }
}
}
@@ -18,7 +18,7 @@ internal class TestRun(
val displayName: String,
val executable: TestExecutable,
val runParameters: List<TestRunParameter>,
val origin: TestOrigin.SingleTestDataFile
val testCaseId: TestCaseId
)
internal sealed interface TestRunParameter {
@@ -37,9 +37,9 @@ internal class TestRunProvider(
}
/**
* Produces a single [TestRun] per testData file.
* Produces a single [TestRun] per [TestCase]. So-called "one test case/one test run" mode.
*
* If testData file contains multiple functions annotated with [kotlin.test.Test], then all these functions will be executed
* If [TestCase] contains multiple functions annotated with [kotlin.test.Test], then all these functions will be executed
* in one shot. If either function will fail, the whole JUnit test will be considered as failed.
*
* Example:
@@ -61,18 +61,18 @@ internal class TestRunProvider(
* }
* }
*/
fun getSingleTestRun(testDataFile: File): TestRun = withTestExecutable(testDataFile) { testCase, executable ->
fun getSingleTestRun(testCaseId: TestCaseId): TestRun = withTestExecutable(testCaseId) { testCase, executable ->
val runParameters = getRunParameters(testCase, testFunction = null)
TestRun(displayName = testDataFile.nameWithoutExtension, executable, runParameters, testCase.origin)
TestRun(displayName = /* Unimportant. Used only in dynamic tests. */ "", executable, runParameters, testCase.id)
}
/**
* Produces at least one [TestRun] per testData file.
* Produces at least one [TestRun] per [TestCase]. So-called "one test function/one test run" mode.
*
* If testData file contains multiple functions annotated with [kotlin.test.Test], then a separate [TestRun] will be produced
* If [TestCase] contains multiple functions annotated with [kotlin.test.Test], then a separate [TestRun] will be produced
* for each such function.
*
* This allows to have a better granularity in tests. So that every individual test method inside testData file will be considered
* This allows to have a better granularity in tests. So that every individual test method inside [TestCase] will be considered
* as an individual JUnit test, and will be presented as a separate row in JUnit test report.
*
* Example:
@@ -96,10 +96,10 @@ internal class TestRunProvider(
* }
* }
*/
fun getTestRuns(testDataFile: File): TreeNode<TestRun> = withTestExecutable(testDataFile) { testCase, executable ->
fun getTestRuns(testCaseId: TestCaseId): TreeNode<TestRun> = withTestExecutable(testCaseId) { testCase, executable ->
fun createTestRun(testRunName: String, testFunction: TestFunction?): TestRun {
val runParameters = getRunParameters(testCase, testFunction)
return TestRun(testRunName, executable, runParameters, testCase.origin)
return TestRun(testRunName, executable, runParameters, testCase.id)
}
when (testCase.kind) {
@@ -110,33 +110,32 @@ internal class TestRunProvider(
}
TestKind.REGULAR, TestKind.STANDALONE -> {
val testFunctions = testCase.extras<WithTestRunnerExtras>().testFunctions
testFunctions.buildTree(TestFunction::packageName) { testFunction -> createTestRun(testFunction.functionName, testFunction) }
testFunctions.buildTree(TestFunction::packageName) { testFunction ->
createTestRun(testFunction.functionName, testFunction)
}
}
}
}
private fun <T> withTestExecutable(testDataFile: File, action: (TestCase, TestExecutable) -> T): T {
private fun <T> withTestExecutable(testCaseId: TestCaseId, action: (TestCase, TestExecutable) -> T): T {
settings.assertNotDisposed()
val testDataDir = testDataFile.parentFile
val testDataFileName = testDataFile.name
val testCaseGroup = testCaseGroupProvider.getTestCaseGroup(testCaseId.testCaseGroupId) ?: fail { "No test case for $testCaseId" }
assumeTrue(testCaseGroup.isEnabled(testCaseId), "Test case is disabled")
val testCaseGroup = testCaseGroupProvider.getTestCaseGroup(testDataDir) ?: fail { "No test case for $testDataFile" }
assumeTrue(testCaseGroup.isEnabled(testDataFileName), "Test case is disabled")
val testCase = testCaseGroup.getByName(testDataFileName) ?: fail { "No test case for $testDataFile" }
val testCase = testCaseGroup.getByName(testCaseId) ?: fail { "No test case for $testCaseId" }
val testCompilation = when (testCase.kind) {
TestKind.STANDALONE, TestKind.STANDALONE_NO_TR -> {
// Create a separate compilation for each standalone test case.
val cacheKey = TestCompilationCacheKey.Standalone(testDataFile)
val cacheKey = TestCompilationCacheKey.Standalone(testCaseId)
cachedCompilations.computeIfAbsent(cacheKey) {
compilationFactory.testCasesToExecutable(listOf(testCase))
}
}
TestKind.REGULAR -> {
// Group regular test cases by compiler arguments.
val cacheKey = TestCompilationCacheKey.Grouped(testDataDir, testCase.freeCompilerArgs)
val cacheKey = TestCompilationCacheKey.Grouped(testCaseId.testCaseGroupId, testCase.freeCompilerArgs)
cachedCompilations.computeIfAbsent(cacheKey) {
val testCases = testCaseGroup.getRegularOnlyByCompilerArgs(testCase.freeCompilerArgs)
assertTrue(testCases.isNotEmpty())
@@ -193,7 +192,7 @@ internal class TestRunProvider(
}
private sealed class TestCompilationCacheKey {
data class Standalone(val testDataFile: File) : TestCompilationCacheKey()
data class Grouped(val testDataDir: File, val freeCompilerArgs: TestCompilerArgs) : TestCompilationCacheKey()
data class Standalone(val testCaseId: TestCaseId) : TestCompilationCacheKey()
data class Grouped(val testCaseGroupId: TestCaseGroupId, val freeCompilerArgs: TestCompilerArgs) : TestCompilationCacheKey()
}
}
@@ -37,6 +37,7 @@ import org.jetbrains.kotlin.test.InTextDirectivesUtils.isIgnoredTarget
import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
import org.jetbrains.kotlin.utils.addIfNotNull
import org.jetbrains.kotlin.utils.addToStdlib.cast
import java.io.File
internal class ExtTestCaseGroupProvider(
@@ -46,16 +47,16 @@ internal class ExtTestCaseGroupProvider(
private val structureFactory = ExtTestDataFileStructureFactory(parentDisposable = this)
private val sharedModules = ThreadSafeCache<String, TestModule.Shared?>()
private val lazyTestCaseGroups = ThreadSafeFactory<File, TestCaseGroup?> { testDataDir ->
if (testDataDir in excludes) return@ThreadSafeFactory TestCaseGroup.ALL_DISABLED
private val lazyTestCaseGroups = ThreadSafeFactory<TestCaseGroupId.TestDataDir, TestCaseGroup?> { testCaseGroupId ->
if (testCaseGroupId.dir in excludes) return@ThreadSafeFactory TestCaseGroup.ALL_DISABLED
val (excludedTestDataFiles, testDataFiles) = testDataDir.listFiles()
val (excludedTestDataFiles, testDataFiles) = testCaseGroupId.dir.listFiles()
?.filter { file -> file.isFile && file.extension == "kt" }
?.partition { file -> file in excludes }
?: return@ThreadSafeFactory null
val disabledTestDataFileNames = hashSetOf<String>()
excludedTestDataFiles.mapTo(disabledTestDataFileNames) { it.name }
val disabledTestCaseIds = hashSetOf<TestCaseId>()
excludedTestDataFiles.mapTo(disabledTestCaseIds, TestCaseId::TestDataFile)
val testCases = mutableListOf<TestCase>()
@@ -70,10 +71,10 @@ internal class ExtTestCaseGroupProvider(
sharedModules = sharedModules
)
else
disabledTestDataFileNames += testDataFile.name
disabledTestCaseIds += TestCaseId.TestDataFile(testDataFile)
}
TestCaseGroup.Default(disabledTestDataFileNames, testCases)
TestCaseGroup.Default(disabledTestCaseIds, testCases)
}
override fun setPreprocessors(testDataDir: File, preprocessors: List<(String) -> String>) {
@@ -83,9 +84,10 @@ internal class ExtTestCaseGroupProvider(
sourceTransformers.remove(testDataDir.canonicalPath)
}
override fun getTestCaseGroup(testDataDir: File): TestCaseGroup? {
override fun getTestCaseGroup(testCaseGroupId: TestCaseGroupId): TestCaseGroup? {
assertNotDisposed()
return lazyTestCaseGroups[testDataDir]
assertTrue(testCaseGroupId is TestCaseGroupId.TestDataDir)
return lazyTestCaseGroups[testCaseGroupId.cast()]
}
companion object {
@@ -586,10 +588,10 @@ private class ExtTestDataFile(
)
val testCase = TestCase(
id = TestCaseId.TestDataFile(testDataFile),
kind = if (isStandaloneTest) TestKind.STANDALONE else TestKind.REGULAR,
modules = modules,
freeCompilerArgs = assembleFreeCompilerArgs(),
origin = TestOrigin.SingleTestDataFile(testDataFile),
nominalPackageName = testDataFileSettings.nominalPackageName,
expectedOutputDataFile = null,
extras = WithTestRunnerExtras.EMPTY
@@ -23,14 +23,15 @@ import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertNotEquals
import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
import org.jetbrains.kotlin.test.services.impl.RegisteredDirectivesParser
import org.jetbrains.kotlin.utils.addToStdlib.cast
import java.io.File
internal class StandardTestCaseGroupProvider(private val settings: Settings) : TestCaseGroupProvider {
val sourceTransformers: MutableMap<String, List<(String) -> String>> = mutableMapOf()
// Load test cases in groups on demand.
private val lazyTestCaseGroups = ThreadSafeFactory<File, TestCaseGroup?> { testDataDir ->
val testDataFiles = testDataDir.listFiles()
private val lazyTestCaseGroups = ThreadSafeFactory<TestCaseGroupId.TestDataDir, TestCaseGroup?> { testCaseGroupId ->
val testDataFiles = testCaseGroupId.dir.listFiles()
?: return@ThreadSafeFactory null // `null` means that there is no such testDataDir.
val testCases = testDataFiles.mapNotNull { testDataFile ->
@@ -40,7 +41,7 @@ internal class StandardTestCaseGroupProvider(private val settings: Settings) : T
createTestCase(testDataFile)
}
TestCaseGroup.Default(disabledTestDataFileNames = emptySet(), testCases = testCases)
TestCaseGroup.Default(disabledTestCaseIds = emptySet(), testCases = testCases)
}
override fun setPreprocessors(testDataDir: File, preprocessors: List<(String) -> String>) {
@@ -50,7 +51,10 @@ internal class StandardTestCaseGroupProvider(private val settings: Settings) : T
sourceTransformers.remove(testDataDir.canonicalPath)
}
override fun getTestCaseGroup(testDataDir: File) = lazyTestCaseGroups[testDataDir]
override fun getTestCaseGroup(testCaseGroupId: TestCaseGroupId): TestCaseGroup? {
assertTrue(testCaseGroupId is TestCaseGroupId.TestDataDir)
return lazyTestCaseGroups[testCaseGroupId.cast()]
}
private fun createTestCase(testDataFile: File): TestCase {
val generatedSourcesDir = computeGeneratedSourcesDir(
@@ -211,10 +215,10 @@ internal class StandardTestCaseGroupProvider(private val settings: Settings) : T
}
val testCase = TestCase(
id = TestCaseId.TestDataFile(testDataFile),
kind = testKind,
modules = testModules.values.toSet(),
freeCompilerArgs = freeCompilerArgs,
origin = TestOrigin.SingleTestDataFile(testDataFile),
nominalPackageName = nominalPackageName,
expectedOutputDataFile = expectedOutputDataFile,
extras = extras
@@ -6,11 +6,12 @@
package org.jetbrains.kotlin.konan.blackboxtest.support.group
import org.jetbrains.kotlin.konan.blackboxtest.support.TestCaseGroup
import org.jetbrains.kotlin.konan.blackboxtest.support.TestCaseGroupId
import java.io.File
internal interface TestCaseGroupProvider {
fun setPreprocessors(testDataDir: File, preprocessors: List<(String) -> String>)
fun getTestCaseGroup(testDataDir: File): TestCaseGroup?
fun getTestCaseGroup(testCaseGroupId: TestCaseGroupId): TestCaseGroup?
}
internal fun String.applySourceTransformers(sourceTransformers: List<(String) -> String>) =
@@ -24,7 +24,7 @@ internal class LocalTestRunner(
override fun getLoggedParameters() = LoggedData.TestRunParameters(
compilerCall = executable.loggedCompilerCall,
origin = testRun.origin,
testCaseId = testRun.testCaseId,
runArgs = programArgs,
runParameters = testRun.runParameters
)