azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Test framework implementation refactoring

Browse Source 
* Extract argument processor and settings out of TestRunner.
This makes it possible to add external runners to the framework.
* Add methods and docs to TestCase and TestSuite classes that
describe Kotlin test methods.
This commit is contained in:
Pavel Punegov
2023-06-29 13:59:42 +02:00
committed by Space Team
parent a5893d4bee
commit f727934f6b
5 changed files with 230 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/GTestLogger.kt
+4 -4
View File
@@ -16,11 +16,11 @@ internal class GTestLogger : TestLoggerWithStatistics() {
private val Collection<TestSuite>.totalNotIgnored: Int
get() = filter { !it.ignored }.size
override fun startIteration(runner: TestRunner, iteration: Int, suites: Collection<TestSuite>) {
if (runner.iterations != 1) {
override fun startIteration(settings: TestSettings, iteration: Int, suites: Collection<TestSuite>) {
if (settings.iterations != 1) {
println("\nRepeating all tests (iteration $iteration) . . .\n")
}
super.startIteration(runner, iteration, suites)
super.startIteration(settings, iteration, suites)
println("[==========] Running ${suites.totalTestsNotIgnored} tests from ${suites.totalNotIgnored} test cases.")
// Just hack to deal with GTest output parsers.
println("[----------] Global test environment set-up.")
@@ -47,7 +47,7 @@ internal class GTestLogger : TestLoggerWithStatistics() {
}
}
override fun finishIteration(runner: TestRunner, iteration: Int, timeMillis: Long) = printResults(timeMillis)
override fun finishIteration(settings: TestSettings, iteration: Int, timeMillis: Long) = printResults(timeMillis)
override fun startSuite(suite: TestSuite) = println("[----------] ${suite.size} tests from ${suite.name}")
kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestListener.kt
+8 -8
View File
@@ -9,11 +9,11 @@ import kotlin.experimental.ExperimentalNativeApi
@ExperimentalNativeApi
internal interface TestListener {
fun startTesting(runner: TestRunner)
fun finishTesting(runner: TestRunner, timeMillis: Long)
fun startTesting(settings: TestSettings)
fun finishTesting(settings: TestSettings, timeMillis: Long)
fun startIteration(runner: TestRunner, iteration: Int, suites: Collection<TestSuite>)
fun finishIteration(runner: TestRunner, iteration: Int, timeMillis: Long)
fun startIteration(settings: TestSettings, iteration: Int, suites: Collection<TestSuite>)
fun finishIteration(settings: TestSettings, iteration: Int, timeMillis: Long)
fun startSuite(suite: TestSuite)
fun finishSuite(suite: TestSuite, timeMillis: Long)
@@ -27,10 +27,10 @@ internal interface TestListener {
@ExperimentalNativeApi
internal open class BaseTestListener: TestListener {
override fun startTesting(runner: TestRunner) {}
override fun finishTesting(runner: TestRunner, timeMillis: Long) {}
override fun startIteration(runner: TestRunner, iteration: Int, suites: Collection<TestSuite>) {}
override fun finishIteration(runner: TestRunner, iteration: Int, timeMillis: Long) {}
override fun startTesting(settings: TestSettings) {}
override fun finishTesting(settings: TestSettings, timeMillis: Long) {}
override fun startIteration(settings: TestSettings, iteration: Int, suites: Collection<TestSuite>) {}
override fun finishIteration(settings: TestSettings, iteration: Int, timeMillis: Long) {}
override fun startSuite(suite: TestSuite) {}
override fun finishSuite(suite: TestSuite, timeMillis: Long) {}
override fun ignoreSuite(suite: TestSuite) {}
kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestLogger.kt
+9 -9
View File
@@ -9,14 +9,14 @@ import kotlin.experimental.ExperimentalNativeApi
@ExperimentalNativeApi
internal interface TestLogger: TestListener {
fun logTestList(runner: TestRunner, suites: Collection<TestSuite>)
fun logTestList(suites: Collection<TestSuite>)
fun log(message: String)
}
@ExperimentalNativeApi
internal open class BaseTestLogger: BaseTestListener(), TestLogger {
override fun log(message: String) = println(message)
override fun logTestList(runner: TestRunner, suites: Collection<TestSuite>) {
override fun logTestList(suites: Collection<TestSuite>) {
suites.forEach { suite ->
println("${suite.name}.")
suite.testCases.values.forEach {
@@ -31,8 +31,8 @@ internal open class TestLoggerWithStatistics: BaseTestLogger() {
protected val statistics = MutableTestStatistics()
override fun startTesting(runner: TestRunner) = statistics.reset()
override fun startIteration(runner: TestRunner, iteration: Int, suites: Collection<TestSuite>) = statistics.reset()
override fun startTesting(settings: TestSettings) = statistics.reset()
override fun startIteration(settings: TestSettings, iteration: Int, suites: Collection<TestSuite>) = statistics.reset()
override fun finishSuite(suite: TestSuite, timeMillis: Long) = statistics.registerSuite()
override fun ignoreSuite(suite: TestSuite) = statistics.registerIgnore(suite)
@@ -43,19 +43,19 @@ internal open class TestLoggerWithStatistics: BaseTestLogger() {
@ExperimentalNativeApi
internal class SilentTestLogger: BaseTestLogger() {
override fun logTestList(runner: TestRunner, suites: Collection<TestSuite>) {}
override fun logTestList(suites: Collection<TestSuite>) {}
override fun log(message: String) {}
override fun fail(testCase: TestCase, e: Throwable, timeMillis: Long) = e.printStackTrace()
}
@ExperimentalNativeApi
internal class SimpleTestLogger: BaseTestLogger() {
override fun startTesting(runner: TestRunner) = println("Starting testing")
override fun finishTesting(runner: TestRunner, timeMillis: Long) = println("Testing finished")
override fun startTesting(settings: TestSettings) = println("Starting testing")
override fun finishTesting(settings: TestSettings, timeMillis: Long) = println("Testing finished")
override fun startIteration(runner: TestRunner, iteration: Int, suites: Collection<TestSuite>) =
override fun startIteration(settings: TestSettings, iteration: Int, suites: Collection<TestSuite>) =
println("Starting iteration: $iteration")
override fun finishIteration(runner: TestRunner, iteration: Int, timeMillis: Long) =
override fun finishIteration(settings: TestSettings, iteration: Int, timeMillis: Long) =
println("Iteration finished: $iteration")
override fun startSuite(suite: TestSuite) = println("Starting test suite: $suite")
kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestRunner.kt
+107 -95
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
@@ -11,20 +11,39 @@ import kotlin.text.StringBuilder
import kotlin.time.TimeSource
import kotlin.time.measureTime
@OptIn(kotlin.time.ExperimentalTime::class)
/**
* Test execution settings.
*/
@ExperimentalNativeApi
internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
private val filters = mutableListOf<(TestCase) -> Boolean>()
private val listeners = mutableSetOf<TestListener>()
private var logger: TestLogger = GTestLogger()
private var runTests = true
private var useExitCode = true
var iterations = 1
private set
var exitCode = 0
private set
internal data class TestSettings(
val testSuites: List<TestSuite>,
val listeners: Set<TestListener>,
val logger: TestLogger,
val runTests: Boolean,
val iterations: Int,
val useExitCode: Boolean
)
/**
* Test processor that initializes the [TestSettings] using the command line options passed in [args].
*
* See the [help] for the description of arguments
*/
@ExperimentalNativeApi
internal class TestProcessor(val suites: List<TestSuite>, val args: Array<String>) {
/**
* Processes given arguments to set test settings and filter tests to be run
*
* @see TestSettings
*/
fun process(): TestSettings {
val listeners = mutableSetOf<TestListener>()
val filters = mutableListOf<TestFilter>()
var logger: TestLogger = GTestLogger()
var runTests = true
var useExitCode = true
var iterations = 1
init {
args.filter {
it.startsWith("--gtest_") || it.startsWith("--ktest_") || it == "--help" || it == "-h"
}.forEach {
@@ -33,11 +52,13 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
1 -> when (arg[0]) {
"--gtest_list_tests",
"--ktest_list_tests" -> {
logger.logTestList(this, filterSuites()); runTests = false
logger.logTestList(suites.filterWith(filters))
runTests = false
}
"-h",
"--help" -> {
logger.log(help); runTests = false
logger.log(help)
runTests = false
}
"--ktest_no_exit_code" -> useExitCode = false
else -> throw IllegalArgumentException("Unknown option: $it\n$help")
@@ -46,70 +67,28 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
val key = arg[0]
val value = arg[1]
when (key) {
"--ktest_logger" -> setLoggerFromArg(value)
"--ktest_logger" -> logger = loggerFromArg(value)
"--gtest_filter",
"--ktest_filter" -> setGTestFilterFromArg(value)
"--ktest_regex_filter" -> setRegexFilterFromArg(value, true)
"--ktest_negative_regex_filter" -> setRegexFilterFromArg(value, false)
"--ktest_gradle_filter" -> setGradleFilterFromArg(value, true)
"--ktest_negative_gradle_filter" -> setGradleFilterFromArg(value, false)
"--ktest_filter" -> filters.add(gTestFilterFromArg(value))
"--ktest_regex_filter" -> filters.add(regexFilterFromArg(value, true))
"--ktest_negative_regex_filter" -> filters.add(regexFilterFromArg(value, false))
"--ktest_gradle_filter" -> filters.add(gradleFilterFromArg(value, true))
"--ktest_negative_gradle_filter" -> filters.add(gradleFilterFromArg(value, false))
"--ktest_repeat",
"--gtest_repeat" -> iterations = value.toIntOrNull() ?: throw IllegalArgumentException("Cannot parse number: $value")
"--gtest_repeat" ->
iterations = value.toIntOrNull() ?: throw IllegalArgumentException("Cannot parse number: $value")
else -> throw IllegalArgumentException("Unknown option: $it\n$help")
}
}
else -> throw IllegalArgumentException("Unknown option: $it\n$help")
}
}
val testSuites = suites.filterWith(filters).toList()
return TestSettings(testSuites, listeners, logger, runTests, iterations, useExitCode)
}
inner class FilteredSuite(val innerSuite: TestSuite) : TestSuite by innerSuite {
private val TestCase.matchFilters: Boolean
get() = filters.all { it(this) }
override val size: Int
get() = testCases.size
override val testCases: Map<String, TestCase> = innerSuite.testCases.filter { it.value.matchFilters }
override fun toString() = innerSuite.toString()
}
private fun filterSuites(): Collection<TestSuite> = suites.map { FilteredSuite(it) }
// TODO: Support short aliases.
// TODO: Support several test iterations.
/**
* Initialize the TestRunner using the command line options passed in [args].
* Returns: true if tests may be ran, false otherwise (there are unrecognized options or just help).
* The following options are available:
*
* --gtest_list_tests
* --ktest_list_tests - Show all available tests.
*
* --gtest_filter=POSITIVE_PATTERNS[-NEGATIVE_PATTERNS]
* --ktest_filter=POSITIVE_PATTERNS[-NEGATIVE_PATTERNS] - Run only the tests whose name matches one of the
* positive patterns but none of the negative patterns.
* '?' matches any single character; '*' matches any
* substring; ':' separates two patterns.
*
* --ktest_regex_filter=PATTERN - Run only the tests whose name matches the pattern.
* The pattern is a Kotlin regular expression.
*
* --ktest_negative_regex_filter=PATTERN - Run only the tests whose name doesn't match the pattern.
* The pattern is a Kotlin regular expression.
*
* --gtest_repeat=COUNT
* --ktest_repeat=COUNT - Run the tests repeatedly.
* Use a negative count to repeat forever.
*
* --ktest_logger=GTEST|TEAMCITY|SIMPLE|SILENT - Use the specified output format. The default one is GTEST.
*
* --ktest_no_exit_code - Don't return a non-zero exit code if there are failing tests.
*/
private fun String.substringEscaped(range: IntRange) =
this.substring(range).let { if (it.isNotEmpty()) Regex.escape(it) else "" }
private fun String.substringEscaped(range: IntRange) = this.substring(range).let { if (it.isNotEmpty()) Regex.escape(it) else "" }
private fun String.toGTestPatterns() = splitToSequence(':').map { pattern ->
val result = StringBuilder()
@@ -125,7 +104,9 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
return@map result.toString().toRegex()
}.toList()
private fun setGTestFilterFromArg(filter: String) {
// region filters
private fun gTestFilterFromArg(filter: String): TestFilter {
if (filter.isEmpty()) {
throw IllegalArgumentException("Empty filter")
}
@@ -137,23 +118,23 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
val positivePatterns = filters[0].toGTestPatterns()
val negativePatterns = filters.getOrNull(1)?.toGTestPatterns() ?: emptyList()
this.filters.add { testCase ->
return { testCase ->
positivePatterns.any { testCase.prettyName.matches(it) } &&
negativePatterns.none { testCase.prettyName.matches(it) }
}
}
private fun setRegexFilterFromArg(filter: String, positive: Boolean = true) {
private fun regexFilterFromArg(filter: String, positive: Boolean = true): TestFilter {
if (filter.isEmpty()) {
throw IllegalArgumentException("Empty filter")
}
val pattern = filter.toRegex()
filters.add { testCase ->
return { testCase ->
testCase.prettyName.matches(pattern) == positive
}
}
private fun setGradleFilterFromArg(filter: String, positive: Boolean = true) {
private fun gradleFilterFromArg(filter: String, positive: Boolean = true): TestFilter {
if (filter.isEmpty()) {
throw IllegalArgumentException("Empty filter")
}
@@ -165,25 +146,39 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
fun TestCase.matches(pattern: Regex) =
prettyName.matches(pattern) || suite.name.matches(pattern)
if (positive) {
filters.add { testCase ->
return if (positive) {
{ testCase ->
patterns.any { testCase.matches(it) }
}
} else {
filters.add { testCase ->
{ testCase ->
patterns.none { testCase.matches(it) }
}
}
}
private fun setLoggerFromArg(logger: String) {
when (logger.uppercase()) {
"GTEST" -> this.logger = GTestLogger()
"TEAMCITY" -> this.logger = TeamCityLogger()
"SIMPLE" -> this.logger = SimpleTestLogger()
"SILENT" -> this.logger = SilentTestLogger()
else -> throw IllegalArgumentException("Unknown logger type. Available types: GTEST, TEAMCITY, SIMPLE")
}
inner class FilteredSuite(val innerSuite: TestSuite, val filters: Collection<TestFilter>) : TestSuite by innerSuite {
private val TestCase.matchFilters: Boolean
get() = filters.all { it(this) }
override val size: Int
get() = testCases.size
override val testCases: Map<String, TestCase> = innerSuite.testCases.filter { it.value.matchFilters }
override fun toString() = innerSuite.toString()
}
private fun Collection<TestSuite>.filterWith(filters: Collection<TestFilter>): Collection<TestSuite> =
map { FilteredSuite(it, filters) }
// endregion
private fun loggerFromArg(logger: String): TestLogger = when (logger.uppercase()) {
"GTEST" -> GTestLogger()
"TEAMCITY" -> TeamCityLogger()
"SIMPLE" -> SimpleTestLogger()
"SILENT" -> SilentTestLogger()
else -> throw IllegalArgumentException("Unknown logger type. Available types: GTEST, TEAMCITY, SIMPLE")
}
private val help: String
@@ -221,10 +216,27 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
|
|--ktest_no_exit_code - Don't return a non-zero exit code if there are failing tests.
""".trimMargin()
}
@OptIn(kotlin.experimental.ExperimentalNativeApi::class)
private typealias TestFilter = (TestCase) -> Boolean
/**
* Test runner.
*
* Runs tests for the given [TestSettings].
*/
@OptIn(kotlin.time.ExperimentalTime::class)
@ExperimentalNativeApi
internal class TestRunner(val settings: TestSettings) {
constructor(suites: List<TestSuite>, args: Array<String>) : this(TestProcessor(suites, args).process())
var exitCode = 0
private set
private inline fun sendToListeners(event: TestListener.() -> Unit) {
logger.event()
listeners.forEach(event)
settings.logger.event()
settings.listeners.forEach(event)
}
private fun TestSuite.run() {
@@ -249,7 +261,7 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
sendToListeners { pass(testCase, startTime.elapsedNow().inWholeMilliseconds) }
} catch (e: Throwable) {
sendToListeners { fail(testCase, e, startTime.elapsedNow().inWholeMilliseconds) }
if (useExitCode) {
if (settings.useExitCode) {
exitCode = 1
}
}
@@ -259,8 +271,8 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
}
private fun runIteration(iteration: Int) {
val suitesFiltered = filterSuites()
sendToListeners { startIteration(this@TestRunner, iteration, suitesFiltered) }
val suitesFiltered = settings.testSuites
sendToListeners { startIteration(settings, iteration, suitesFiltered) }
val iterationTime = measureTime {
suitesFiltered.forEach {
if (it.ignored) {
@@ -276,21 +288,21 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
}
}
}.inWholeMilliseconds
sendToListeners { finishIteration(this@TestRunner, iteration, iterationTime) }
sendToListeners { finishIteration(settings, iteration, iterationTime) }
}
fun run(): Int {
if (!runTests)
if (!settings.runTests)
return 0
sendToListeners { startTesting(this@TestRunner) }
sendToListeners { startTesting(settings) }
val totalTime = measureTime {
var i = 1
while (i <= iterations || iterations < 0) {
while (i <= settings.iterations || settings.iterations < 0) {
runIteration(i)
i++
}
}.inWholeMilliseconds
sendToListeners { finishTesting(this@TestRunner, totalTime) }
sendToListeners { finishTesting(settings, totalTime) }
return exitCode
}
}
kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestSuite.kt
+102 -16
View File
@@ -10,26 +10,106 @@ import kotlin.IllegalArgumentException
import kotlin.system.getTimeMillis
import kotlin.system.measureTimeMillis
/**
* Represents a test case, that is a Kotlin test method marked with `@Test` annotaion.
*
* @see kotlin.test.Test
*/
@ExperimentalNativeApi
public interface TestCase {
val name: String
/**
* Shows if this test is ignored by the `@Ignore` annotation.
*
* @see kotlin.test.Ignore
*/
val ignored: Boolean
/**
* Test Suite the test belongs to.
*/
val suite: TestSuite
fun run()
/**
* Runs all methods that were marked with `@BeforeTest` annotation.
*
* @see kotlin.test.BeforeTest
*/
fun doBefore()
/**
* Runs the test method itself
*
* @see kotlin.test.Test
*/
fun doRun()
/**
* Runs test with its before and after functions.
*/
fun run() {
try {
doBefore()
doRun()
} finally {
doAfter()
}
}
/**
* Runs all methods that were marked with `@AfterTest` annotation.
*
* @see kotlin.test.AfterTest
*/
fun doAfter()
}
@ExperimentalNativeApi
internal val TestCase.prettyName get() = "${suite.name}.$name"
/**
* Represents Test suite, a suite that contains test cases.
*
* Test suite can be either test class containing test cases
* or a top level suite for top level methods.
*/
@ExperimentalNativeApi
public interface TestSuite {
/**
* Test suite name.
*/
val name: String
/**
* Shows if the suite is ignored with `@Ignore` annotation.
*
* @see kotlin.test.Ignore
*/
val ignored: Boolean
/**
* Test cases this test suite contains
*/
val testCases: Map<String, TestCase>
/**
* Number of test cases
*/
val size : Int
/**
* Executes all methods marked with `@BeforeClass` annotation
*
* @see kotlin.test.BeforeClass
*/
fun doBeforeClass()
/**
* Executes all methods marked with `@AfterClass` annotation
*
* @see @see kotlin.test.AfterClass
*/
fun doAfterClass()
}
@@ -85,14 +165,18 @@ public abstract class BaseClassSuite<INSTANCE, COMPANION>(name: String, ignored:
ignored: Boolean)
: BasicTestCase<INSTANCE.() -> Unit>(name, suite, testFunction, ignored) {
override fun run() {
val instance = suite.createInstance()
try {
suite.before.forEach { instance.it() }
instance.testFunction()
} finally {
suite.after.forEach { instance.it() }
}
internal val instance: INSTANCE by lazy { suite.createInstance() }
override fun doBefore() {
suite.before.forEach { instance.it() }
}
override fun doAfter() {
suite.after.forEach { instance.it() }
}
override fun doRun() {
instance.testFunction()
}
}
@@ -146,14 +230,16 @@ public class TopLevelSuite(name: String): AbstractTestSuite<TopLevelFun>(name, f
class TestCase(name: String, override val suite: TopLevelSuite, testFunction: TopLevelFun, ignored: Boolean)
: BasicTestCase<TopLevelFun>(name, suite, testFunction, ignored) {
override fun doBefore() {
suite.before.forEach { it() }
}
override fun run() {
try {
suite.before.forEach { it() }
testFunction()
} finally {
suite.after.forEach { it() }
}
override fun doAfter() {
suite.after.forEach { it() }
}
override fun doRun() {
testFunction()
}
}