diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/GTestLogger.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/GTestLogger.kt index a929d659e35..4ea2c1ca541 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/GTestLogger.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/GTestLogger.kt @@ -16,11 +16,11 @@ internal class GTestLogger : TestLoggerWithStatistics() { private val Collection.totalNotIgnored: Int get() = filter { !it.ignored }.size - override fun startIteration(runner: TestRunner, iteration: Int, suites: Collection) { - if (runner.iterations != 1) { + override fun startIteration(settings: TestSettings, iteration: Int, suites: Collection) { + if (settings.iterations != 1) { println("\nRepeating all tests (iteration $iteration) . . .\n") } - super.startIteration(runner, iteration, suites) + super.startIteration(settings, iteration, suites) println("[==========] Running ${suites.totalTestsNotIgnored} tests from ${suites.totalNotIgnored} test cases.") // Just hack to deal with GTest output parsers. println("[----------] Global test environment set-up.") @@ -47,7 +47,7 @@ internal class GTestLogger : TestLoggerWithStatistics() { } } - override fun finishIteration(runner: TestRunner, iteration: Int, timeMillis: Long) = printResults(timeMillis) + override fun finishIteration(settings: TestSettings, iteration: Int, timeMillis: Long) = printResults(timeMillis) override fun startSuite(suite: TestSuite) = println("[----------] ${suite.size} tests from ${suite.name}") diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestListener.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestListener.kt index ccdfebefd63..8a0714a8d21 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestListener.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestListener.kt @@ -9,11 +9,11 @@ import kotlin.experimental.ExperimentalNativeApi @ExperimentalNativeApi internal interface TestListener { - fun startTesting(runner: TestRunner) - fun finishTesting(runner: TestRunner, timeMillis: Long) + fun startTesting(settings: TestSettings) + fun finishTesting(settings: TestSettings, timeMillis: Long) - fun startIteration(runner: TestRunner, iteration: Int, suites: Collection) - fun finishIteration(runner: TestRunner, iteration: Int, timeMillis: Long) + fun startIteration(settings: TestSettings, iteration: Int, suites: Collection) + fun finishIteration(settings: TestSettings, iteration: Int, timeMillis: Long) fun startSuite(suite: TestSuite) fun finishSuite(suite: TestSuite, timeMillis: Long) @@ -27,10 +27,10 @@ internal interface TestListener { @ExperimentalNativeApi internal open class BaseTestListener: TestListener { - override fun startTesting(runner: TestRunner) {} - override fun finishTesting(runner: TestRunner, timeMillis: Long) {} - override fun startIteration(runner: TestRunner, iteration: Int, suites: Collection) {} - override fun finishIteration(runner: TestRunner, iteration: Int, timeMillis: Long) {} + override fun startTesting(settings: TestSettings) {} + override fun finishTesting(settings: TestSettings, timeMillis: Long) {} + override fun startIteration(settings: TestSettings, iteration: Int, suites: Collection) {} + override fun finishIteration(settings: TestSettings, iteration: Int, timeMillis: Long) {} override fun startSuite(suite: TestSuite) {} override fun finishSuite(suite: TestSuite, timeMillis: Long) {} override fun ignoreSuite(suite: TestSuite) {} diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestLogger.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestLogger.kt index 544e68f37a8..17f13e10e97 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestLogger.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestLogger.kt @@ -9,14 +9,14 @@ import kotlin.experimental.ExperimentalNativeApi @ExperimentalNativeApi internal interface TestLogger: TestListener { - fun logTestList(runner: TestRunner, suites: Collection) + fun logTestList(suites: Collection) fun log(message: String) } @ExperimentalNativeApi internal open class BaseTestLogger: BaseTestListener(), TestLogger { override fun log(message: String) = println(message) - override fun logTestList(runner: TestRunner, suites: Collection) { + override fun logTestList(suites: Collection) { suites.forEach { suite -> println("${suite.name}.") suite.testCases.values.forEach { @@ -31,8 +31,8 @@ internal open class TestLoggerWithStatistics: BaseTestLogger() { protected val statistics = MutableTestStatistics() - override fun startTesting(runner: TestRunner) = statistics.reset() - override fun startIteration(runner: TestRunner, iteration: Int, suites: Collection) = statistics.reset() + override fun startTesting(settings: TestSettings) = statistics.reset() + override fun startIteration(settings: TestSettings, iteration: Int, suites: Collection) = statistics.reset() override fun finishSuite(suite: TestSuite, timeMillis: Long) = statistics.registerSuite() override fun ignoreSuite(suite: TestSuite) = statistics.registerIgnore(suite) @@ -43,19 +43,19 @@ internal open class TestLoggerWithStatistics: BaseTestLogger() { @ExperimentalNativeApi internal class SilentTestLogger: BaseTestLogger() { - override fun logTestList(runner: TestRunner, suites: Collection) {} + override fun logTestList(suites: Collection) {} override fun log(message: String) {} override fun fail(testCase: TestCase, e: Throwable, timeMillis: Long) = e.printStackTrace() } @ExperimentalNativeApi internal class SimpleTestLogger: BaseTestLogger() { - override fun startTesting(runner: TestRunner) = println("Starting testing") - override fun finishTesting(runner: TestRunner, timeMillis: Long) = println("Testing finished") + override fun startTesting(settings: TestSettings) = println("Starting testing") + override fun finishTesting(settings: TestSettings, timeMillis: Long) = println("Testing finished") - override fun startIteration(runner: TestRunner, iteration: Int, suites: Collection) = + override fun startIteration(settings: TestSettings, iteration: Int, suites: Collection) = println("Starting iteration: $iteration") - override fun finishIteration(runner: TestRunner, iteration: Int, timeMillis: Long) = + override fun finishIteration(settings: TestSettings, iteration: Int, timeMillis: Long) = println("Iteration finished: $iteration") override fun startSuite(suite: TestSuite) = println("Starting test suite: $suite") diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestRunner.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestRunner.kt index 666202a5ff7..48cc1e26cfc 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestRunner.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestRunner.kt @@ -1,5 +1,5 @@ /* - * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license * that can be found in the LICENSE file. */ @@ -11,20 +11,39 @@ import kotlin.text.StringBuilder import kotlin.time.TimeSource import kotlin.time.measureTime -@OptIn(kotlin.time.ExperimentalTime::class) +/** + * Test execution settings. + */ @ExperimentalNativeApi -internal class TestRunner(val suites: List, args: Array) { - private val filters = mutableListOf<(TestCase) -> Boolean>() - private val listeners = mutableSetOf() - private var logger: TestLogger = GTestLogger() - private var runTests = true - private var useExitCode = true - var iterations = 1 - private set - var exitCode = 0 - private set +internal data class TestSettings( + val testSuites: List, + val listeners: Set, + val logger: TestLogger, + val runTests: Boolean, + val iterations: Int, + val useExitCode: Boolean +) + +/** + * Test processor that initializes the [TestSettings] using the command line options passed in [args]. + * + * See the [help] for the description of arguments + */ +@ExperimentalNativeApi +internal class TestProcessor(val suites: List, val args: Array) { + /** + * Processes given arguments to set test settings and filter tests to be run + * + * @see TestSettings + */ + fun process(): TestSettings { + val listeners = mutableSetOf() + val filters = mutableListOf() + var logger: TestLogger = GTestLogger() + var runTests = true + var useExitCode = true + var iterations = 1 - init { args.filter { it.startsWith("--gtest_") || it.startsWith("--ktest_") || it == "--help" || it == "-h" }.forEach { @@ -33,11 +52,13 @@ internal class TestRunner(val suites: List, args: Array) { 1 -> when (arg[0]) { "--gtest_list_tests", "--ktest_list_tests" -> { - logger.logTestList(this, filterSuites()); runTests = false + logger.logTestList(suites.filterWith(filters)) + runTests = false } "-h", "--help" -> { - logger.log(help); runTests = false + logger.log(help) + runTests = false } "--ktest_no_exit_code" -> useExitCode = false else -> throw IllegalArgumentException("Unknown option: $it\n$help") @@ -46,70 +67,28 @@ internal class TestRunner(val suites: List, args: Array) { val key = arg[0] val value = arg[1] when (key) { - "--ktest_logger" -> setLoggerFromArg(value) + "--ktest_logger" -> logger = loggerFromArg(value) "--gtest_filter", - "--ktest_filter" -> setGTestFilterFromArg(value) - "--ktest_regex_filter" -> setRegexFilterFromArg(value, true) - "--ktest_negative_regex_filter" -> setRegexFilterFromArg(value, false) - "--ktest_gradle_filter" -> setGradleFilterFromArg(value, true) - "--ktest_negative_gradle_filter" -> setGradleFilterFromArg(value, false) + "--ktest_filter" -> filters.add(gTestFilterFromArg(value)) + "--ktest_regex_filter" -> filters.add(regexFilterFromArg(value, true)) + "--ktest_negative_regex_filter" -> filters.add(regexFilterFromArg(value, false)) + "--ktest_gradle_filter" -> filters.add(gradleFilterFromArg(value, true)) + "--ktest_negative_gradle_filter" -> filters.add(gradleFilterFromArg(value, false)) "--ktest_repeat", - "--gtest_repeat" -> iterations = value.toIntOrNull() ?: throw IllegalArgumentException("Cannot parse number: $value") + "--gtest_repeat" -> + iterations = value.toIntOrNull() ?: throw IllegalArgumentException("Cannot parse number: $value") else -> throw IllegalArgumentException("Unknown option: $it\n$help") } } else -> throw IllegalArgumentException("Unknown option: $it\n$help") } } + val testSuites = suites.filterWith(filters).toList() + + return TestSettings(testSuites, listeners, logger, runTests, iterations, useExitCode) } - inner class FilteredSuite(val innerSuite: TestSuite) : TestSuite by innerSuite { - - private val TestCase.matchFilters: Boolean - get() = filters.all { it(this) } - - override val size: Int - get() = testCases.size - - override val testCases: Map = innerSuite.testCases.filter { it.value.matchFilters } - override fun toString() = innerSuite.toString() - } - - private fun filterSuites(): Collection = suites.map { FilteredSuite(it) } - - // TODO: Support short aliases. - // TODO: Support several test iterations. - /** - * Initialize the TestRunner using the command line options passed in [args]. - * Returns: true if tests may be ran, false otherwise (there are unrecognized options or just help). - * The following options are available: - * - * --gtest_list_tests - * --ktest_list_tests - Show all available tests. - * - * --gtest_filter=POSITIVE_PATTERNS[-NEGATIVE_PATTERNS] - * --ktest_filter=POSITIVE_PATTERNS[-NEGATIVE_PATTERNS] - Run only the tests whose name matches one of the - * positive patterns but none of the negative patterns. - * '?' matches any single character; '*' matches any - * substring; ':' separates two patterns. - * - * --ktest_regex_filter=PATTERN - Run only the tests whose name matches the pattern. - * The pattern is a Kotlin regular expression. - * - * --ktest_negative_regex_filter=PATTERN - Run only the tests whose name doesn't match the pattern. - * The pattern is a Kotlin regular expression. - * - * --gtest_repeat=COUNT - * --ktest_repeat=COUNT - Run the tests repeatedly. - * Use a negative count to repeat forever. - * - * --ktest_logger=GTEST|TEAMCITY|SIMPLE|SILENT - Use the specified output format. The default one is GTEST. - * - * --ktest_no_exit_code - Don't return a non-zero exit code if there are failing tests. - */ - - private fun String.substringEscaped(range: IntRange) = - this.substring(range).let { if (it.isNotEmpty()) Regex.escape(it) else "" } + private fun String.substringEscaped(range: IntRange) = this.substring(range).let { if (it.isNotEmpty()) Regex.escape(it) else "" } private fun String.toGTestPatterns() = splitToSequence(':').map { pattern -> val result = StringBuilder() @@ -125,7 +104,9 @@ internal class TestRunner(val suites: List, args: Array) { return@map result.toString().toRegex() }.toList() - private fun setGTestFilterFromArg(filter: String) { + // region filters + + private fun gTestFilterFromArg(filter: String): TestFilter { if (filter.isEmpty()) { throw IllegalArgumentException("Empty filter") } @@ -137,23 +118,23 @@ internal class TestRunner(val suites: List, args: Array) { val positivePatterns = filters[0].toGTestPatterns() val negativePatterns = filters.getOrNull(1)?.toGTestPatterns() ?: emptyList() - this.filters.add { testCase -> + return { testCase -> positivePatterns.any { testCase.prettyName.matches(it) } && negativePatterns.none { testCase.prettyName.matches(it) } } } - private fun setRegexFilterFromArg(filter: String, positive: Boolean = true) { + private fun regexFilterFromArg(filter: String, positive: Boolean = true): TestFilter { if (filter.isEmpty()) { throw IllegalArgumentException("Empty filter") } val pattern = filter.toRegex() - filters.add { testCase -> + return { testCase -> testCase.prettyName.matches(pattern) == positive } } - private fun setGradleFilterFromArg(filter: String, positive: Boolean = true) { + private fun gradleFilterFromArg(filter: String, positive: Boolean = true): TestFilter { if (filter.isEmpty()) { throw IllegalArgumentException("Empty filter") } @@ -165,25 +146,39 @@ internal class TestRunner(val suites: List, args: Array) { fun TestCase.matches(pattern: Regex) = prettyName.matches(pattern) || suite.name.matches(pattern) - if (positive) { - filters.add { testCase -> + return if (positive) { + { testCase -> patterns.any { testCase.matches(it) } } } else { - filters.add { testCase -> + { testCase -> patterns.none { testCase.matches(it) } } } } - private fun setLoggerFromArg(logger: String) { - when (logger.uppercase()) { - "GTEST" -> this.logger = GTestLogger() - "TEAMCITY" -> this.logger = TeamCityLogger() - "SIMPLE" -> this.logger = SimpleTestLogger() - "SILENT" -> this.logger = SilentTestLogger() - else -> throw IllegalArgumentException("Unknown logger type. Available types: GTEST, TEAMCITY, SIMPLE") - } + inner class FilteredSuite(val innerSuite: TestSuite, val filters: Collection) : TestSuite by innerSuite { + private val TestCase.matchFilters: Boolean + get() = filters.all { it(this) } + + override val size: Int + get() = testCases.size + + override val testCases: Map = innerSuite.testCases.filter { it.value.matchFilters } + override fun toString() = innerSuite.toString() + } + + private fun Collection.filterWith(filters: Collection): Collection = + map { FilteredSuite(it, filters) } + + // endregion + + private fun loggerFromArg(logger: String): TestLogger = when (logger.uppercase()) { + "GTEST" -> GTestLogger() + "TEAMCITY" -> TeamCityLogger() + "SIMPLE" -> SimpleTestLogger() + "SILENT" -> SilentTestLogger() + else -> throw IllegalArgumentException("Unknown logger type. Available types: GTEST, TEAMCITY, SIMPLE") } private val help: String @@ -221,10 +216,27 @@ internal class TestRunner(val suites: List, args: Array) { | |--ktest_no_exit_code - Don't return a non-zero exit code if there are failing tests. """.trimMargin() +} + +@OptIn(kotlin.experimental.ExperimentalNativeApi::class) +private typealias TestFilter = (TestCase) -> Boolean + +/** + * Test runner. + * + * Runs tests for the given [TestSettings]. + */ +@OptIn(kotlin.time.ExperimentalTime::class) +@ExperimentalNativeApi +internal class TestRunner(val settings: TestSettings) { + constructor(suites: List, args: Array) : this(TestProcessor(suites, args).process()) + + var exitCode = 0 + private set private inline fun sendToListeners(event: TestListener.() -> Unit) { - logger.event() - listeners.forEach(event) + settings.logger.event() + settings.listeners.forEach(event) } private fun TestSuite.run() { @@ -249,7 +261,7 @@ internal class TestRunner(val suites: List, args: Array) { sendToListeners { pass(testCase, startTime.elapsedNow().inWholeMilliseconds) } } catch (e: Throwable) { sendToListeners { fail(testCase, e, startTime.elapsedNow().inWholeMilliseconds) } - if (useExitCode) { + if (settings.useExitCode) { exitCode = 1 } } @@ -259,8 +271,8 @@ internal class TestRunner(val suites: List, args: Array) { } private fun runIteration(iteration: Int) { - val suitesFiltered = filterSuites() - sendToListeners { startIteration(this@TestRunner, iteration, suitesFiltered) } + val suitesFiltered = settings.testSuites + sendToListeners { startIteration(settings, iteration, suitesFiltered) } val iterationTime = measureTime { suitesFiltered.forEach { if (it.ignored) { @@ -276,21 +288,21 @@ internal class TestRunner(val suites: List, args: Array) { } } }.inWholeMilliseconds - sendToListeners { finishIteration(this@TestRunner, iteration, iterationTime) } + sendToListeners { finishIteration(settings, iteration, iterationTime) } } fun run(): Int { - if (!runTests) + if (!settings.runTests) return 0 - sendToListeners { startTesting(this@TestRunner) } + sendToListeners { startTesting(settings) } val totalTime = measureTime { var i = 1 - while (i <= iterations || iterations < 0) { + while (i <= settings.iterations || settings.iterations < 0) { runIteration(i) i++ } }.inWholeMilliseconds - sendToListeners { finishTesting(this@TestRunner, totalTime) } + sendToListeners { finishTesting(settings, totalTime) } return exitCode } } diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestSuite.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestSuite.kt index 59480b6559b..a0b7d110921 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestSuite.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestSuite.kt @@ -10,26 +10,106 @@ import kotlin.IllegalArgumentException import kotlin.system.getTimeMillis import kotlin.system.measureTimeMillis +/** + * Represents a test case, that is a Kotlin test method marked with `@Test` annotaion. + * + * @see kotlin.test.Test + */ @ExperimentalNativeApi public interface TestCase { val name: String + + /** + * Shows if this test is ignored by the `@Ignore` annotation. + * + * @see kotlin.test.Ignore + */ val ignored: Boolean + + /** + * Test Suite the test belongs to. + */ val suite: TestSuite - fun run() + /** + * Runs all methods that were marked with `@BeforeTest` annotation. + * + * @see kotlin.test.BeforeTest + */ + fun doBefore() + + /** + * Runs the test method itself + * + * @see kotlin.test.Test + */ + fun doRun() + + /** + * Runs test with its before and after functions. + */ + fun run() { + try { + doBefore() + doRun() + } finally { + doAfter() + } + } + + /** + * Runs all methods that were marked with `@AfterTest` annotation. + * + * @see kotlin.test.AfterTest + */ + fun doAfter() } @ExperimentalNativeApi internal val TestCase.prettyName get() = "${suite.name}.$name" +/** + * Represents Test suite, a suite that contains test cases. + * + * Test suite can be either test class containing test cases + * or a top level suite for top level methods. + */ @ExperimentalNativeApi public interface TestSuite { + /** + * Test suite name. + */ val name: String + + /** + * Shows if the suite is ignored with `@Ignore` annotation. + * + * @see kotlin.test.Ignore + */ val ignored: Boolean + + /** + * Test cases this test suite contains + */ val testCases: Map + + /** + * Number of test cases + */ val size : Int + /** + * Executes all methods marked with `@BeforeClass` annotation + * + * @see kotlin.test.BeforeClass + */ fun doBeforeClass() + + /** + * Executes all methods marked with `@AfterClass` annotation + * + * @see @see kotlin.test.AfterClass + */ fun doAfterClass() } @@ -85,14 +165,18 @@ public abstract class BaseClassSuite(name: String, ignored: ignored: Boolean) : BasicTestCase Unit>(name, suite, testFunction, ignored) { - override fun run() { - val instance = suite.createInstance() - try { - suite.before.forEach { instance.it() } - instance.testFunction() - } finally { - suite.after.forEach { instance.it() } - } + internal val instance: INSTANCE by lazy { suite.createInstance() } + + override fun doBefore() { + suite.before.forEach { instance.it() } + } + + override fun doAfter() { + suite.after.forEach { instance.it() } + } + + override fun doRun() { + instance.testFunction() } } @@ -146,14 +230,16 @@ public class TopLevelSuite(name: String): AbstractTestSuite(name, f class TestCase(name: String, override val suite: TopLevelSuite, testFunction: TopLevelFun, ignored: Boolean) : BasicTestCase(name, suite, testFunction, ignored) { + override fun doBefore() { + suite.before.forEach { it() } + } - override fun run() { - try { - suite.before.forEach { it() } - testFunction() - } finally { - suite.after.forEach { it() } - } + override fun doAfter() { + suite.after.forEach { it() } + } + + override fun doRun() { + testFunction() } }