[K/N] Test framework implementation refactoring

* Extract argument processor and settings out of TestRunner.
This makes it possible to add external runners to the framework.
* Add methods and docs to TestCase and TestSuite classes that
describe Kotlin test methods.

kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/GTestLogger.kt

@@ -16,11 +16,11 @@ internal class GTestLogger : TestLoggerWithStatistics() {
     private val Collection<TestSuite>.totalNotIgnored: Int
         get() = filter { !it.ignored }.size
 
-    override fun startIteration(runner: TestRunner, iteration: Int, suites: Collection<TestSuite>) {
+    override fun startIteration(settings: TestSettings, iteration: Int, suites: Collection<TestSuite>) {
-        if (runner.iterations != 1) {
+        if (settings.iterations != 1) {
             println("\nRepeating all tests (iteration $iteration) . . .\n")
         }
-        super.startIteration(runner, iteration, suites)
+        super.startIteration(settings, iteration, suites)
         println("[==========] Running ${suites.totalTestsNotIgnored} tests from ${suites.totalNotIgnored} test cases.")
         // Just hack to deal with GTest output parsers.
         println("[----------] Global test environment set-up.")
@@ -47,7 +47,7 @@ internal class GTestLogger : TestLoggerWithStatistics() {
         }
     }
 
-    override fun finishIteration(runner: TestRunner, iteration: Int, timeMillis: Long) = printResults(timeMillis)
+    override fun finishIteration(settings: TestSettings, iteration: Int, timeMillis: Long) = printResults(timeMillis)
 
     override fun startSuite(suite: TestSuite) = println("[----------] ${suite.size} tests from ${suite.name}")
 

kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestListener.kt

@@ -9,11 +9,11 @@ import kotlin.experimental.ExperimentalNativeApi
 
 @ExperimentalNativeApi
 internal interface TestListener {
-    fun startTesting(runner: TestRunner)
+    fun startTesting(settings: TestSettings)
-    fun finishTesting(runner: TestRunner, timeMillis: Long)
+    fun finishTesting(settings: TestSettings, timeMillis: Long)
 
-    fun startIteration(runner: TestRunner, iteration: Int, suites: Collection<TestSuite>)
+    fun startIteration(settings: TestSettings, iteration: Int, suites: Collection<TestSuite>)
-    fun finishIteration(runner: TestRunner, iteration: Int, timeMillis: Long)
+    fun finishIteration(settings: TestSettings, iteration: Int, timeMillis: Long)
 
     fun startSuite(suite: TestSuite)
     fun finishSuite(suite: TestSuite, timeMillis: Long)
@@ -27,10 +27,10 @@ internal interface TestListener {
 
 @ExperimentalNativeApi
 internal open class BaseTestListener: TestListener {
-    override fun startTesting(runner: TestRunner) {}
+    override fun startTesting(settings: TestSettings) {}
-    override fun finishTesting(runner: TestRunner, timeMillis: Long) {}
+    override fun finishTesting(settings: TestSettings, timeMillis: Long) {}
-    override fun startIteration(runner: TestRunner, iteration: Int, suites: Collection<TestSuite>) {}
+    override fun startIteration(settings: TestSettings, iteration: Int, suites: Collection<TestSuite>) {}
-    override fun finishIteration(runner: TestRunner, iteration: Int, timeMillis: Long) {}
+    override fun finishIteration(settings: TestSettings, iteration: Int, timeMillis: Long) {}
     override fun startSuite(suite: TestSuite) {}
     override fun finishSuite(suite: TestSuite, timeMillis: Long) {}
     override fun ignoreSuite(suite: TestSuite) {}

kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestLogger.kt

@@ -9,14 +9,14 @@ import kotlin.experimental.ExperimentalNativeApi
 
 @ExperimentalNativeApi
 internal interface TestLogger: TestListener {
-    fun logTestList(runner: TestRunner, suites: Collection<TestSuite>)
+    fun logTestList(suites: Collection<TestSuite>)
     fun log(message: String)
 }
 
 @ExperimentalNativeApi
 internal open class BaseTestLogger: BaseTestListener(), TestLogger {
     override fun log(message: String) = println(message)
-    override fun logTestList(runner: TestRunner, suites: Collection<TestSuite>) {
+    override fun logTestList(suites: Collection<TestSuite>) {
         suites.forEach { suite ->
             println("${suite.name}.")
             suite.testCases.values.forEach {
@@ -31,8 +31,8 @@ internal open class TestLoggerWithStatistics: BaseTestLogger() {
 
     protected val statistics = MutableTestStatistics()
 
-    override fun startTesting(runner: TestRunner) = statistics.reset()
+    override fun startTesting(settings: TestSettings) = statistics.reset()
-    override fun startIteration(runner: TestRunner, iteration: Int, suites: Collection<TestSuite>) = statistics.reset()
+    override fun startIteration(settings: TestSettings, iteration: Int, suites: Collection<TestSuite>) = statistics.reset()
 
     override fun finishSuite(suite: TestSuite, timeMillis: Long) = statistics.registerSuite()
     override fun ignoreSuite(suite: TestSuite) = statistics.registerIgnore(suite)
@@ -43,19 +43,19 @@ internal open class TestLoggerWithStatistics: BaseTestLogger() {
 
 @ExperimentalNativeApi
 internal class SilentTestLogger: BaseTestLogger() {
-    override fun logTestList(runner: TestRunner, suites: Collection<TestSuite>) {}
+    override fun logTestList(suites: Collection<TestSuite>) {}
     override fun log(message: String) {}
     override fun fail(testCase: TestCase, e: Throwable, timeMillis: Long) = e.printStackTrace()
 }
 
 @ExperimentalNativeApi
 internal class SimpleTestLogger: BaseTestLogger() {
-    override fun startTesting(runner: TestRunner) = println("Starting testing")
+    override fun startTesting(settings: TestSettings) = println("Starting testing")
-    override fun finishTesting(runner: TestRunner, timeMillis: Long) = println("Testing finished")
+    override fun finishTesting(settings: TestSettings, timeMillis: Long) = println("Testing finished")
 
-    override fun startIteration(runner: TestRunner, iteration: Int, suites: Collection<TestSuite>) =
+    override fun startIteration(settings: TestSettings, iteration: Int, suites: Collection<TestSuite>) =
             println("Starting iteration: $iteration")
-    override fun finishIteration(runner: TestRunner, iteration: Int, timeMillis: Long) =
+    override fun finishIteration(settings: TestSettings, iteration: Int, timeMillis: Long) =
             println("Iteration finished: $iteration")
 
     override fun startSuite(suite: TestSuite) = println("Starting test suite: $suite")

kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestRunner.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * Copyright 2010-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
  * that can be found in the LICENSE file.
  */
 
@@ -11,20 +11,39 @@ import kotlin.text.StringBuilder
 import kotlin.time.TimeSource
 import kotlin.time.measureTime
 
-@OptIn(kotlin.time.ExperimentalTime::class)
+/**
+ * Test execution settings.
+ */
 @ExperimentalNativeApi
-internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
+internal data class TestSettings(
-    private val filters = mutableListOf<(TestCase) -> Boolean>()
+        val testSuites: List<TestSuite>,
-    private val listeners = mutableSetOf<TestListener>()
+        val listeners: Set<TestListener>,
-    private var logger: TestLogger = GTestLogger()
+        val logger: TestLogger,
-    private var runTests = true
+        val runTests: Boolean,
-    private var useExitCode = true
+        val iterations: Int,
-    var iterations = 1
+        val useExitCode: Boolean
-        private set
+)
-    var exitCode = 0
+
-        private set
+/**
+ *  Test processor that initializes the [TestSettings] using the command line options passed in [args].
+ *
+ *  See the [help] for the description of arguments
+ */
+@ExperimentalNativeApi
+internal class TestProcessor(val suites: List<TestSuite>, val args: Array<String>) {
+    /**
+     * Processes given arguments to set test settings and filter tests to be run
+     *
+     * @see TestSettings
+     */
+    fun process(): TestSettings {
+        val listeners = mutableSetOf<TestListener>()
+        val filters = mutableListOf<TestFilter>()
+        var logger: TestLogger = GTestLogger()
+        var runTests = true
+        var useExitCode = true
+        var iterations = 1
 
-    init {
         args.filter {
             it.startsWith("--gtest_") || it.startsWith("--ktest_") || it == "--help" || it == "-h"
         }.forEach {
@@ -33,11 +52,13 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
                 1 -> when (arg[0]) {
                     "--gtest_list_tests",
                     "--ktest_list_tests" -> {
-                        logger.logTestList(this, filterSuites()); runTests = false
+                        logger.logTestList(suites.filterWith(filters))
+                        runTests = false
                     }
                     "-h",
                     "--help" -> {
-                        logger.log(help); runTests = false
+                        logger.log(help)
+                        runTests = false
                     }
                     "--ktest_no_exit_code" -> useExitCode = false
                     else -> throw IllegalArgumentException("Unknown option: $it\n$help")
@@ -46,70 +67,28 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
                     val key = arg[0]
                     val value = arg[1]
                     when (key) {
-                        "--ktest_logger" -> setLoggerFromArg(value)
+                        "--ktest_logger" -> logger = loggerFromArg(value)
                         "--gtest_filter",
-                        "--ktest_filter" -> setGTestFilterFromArg(value)
+                        "--ktest_filter" -> filters.add(gTestFilterFromArg(value))
-                        "--ktest_regex_filter" -> setRegexFilterFromArg(value, true)
+                        "--ktest_regex_filter" -> filters.add(regexFilterFromArg(value, true))
-                        "--ktest_negative_regex_filter" -> setRegexFilterFromArg(value, false)
+                        "--ktest_negative_regex_filter" -> filters.add(regexFilterFromArg(value, false))
-                        "--ktest_gradle_filter" -> setGradleFilterFromArg(value, true)
+                        "--ktest_gradle_filter" -> filters.add(gradleFilterFromArg(value, true))
-                        "--ktest_negative_gradle_filter" -> setGradleFilterFromArg(value, false)
+                        "--ktest_negative_gradle_filter" -> filters.add(gradleFilterFromArg(value, false))
                         "--ktest_repeat",
-                        "--gtest_repeat" -> iterations = value.toIntOrNull() ?: throw IllegalArgumentException("Cannot parse number: $value")
+                        "--gtest_repeat" ->
+                            iterations = value.toIntOrNull() ?: throw IllegalArgumentException("Cannot parse number: $value")
                         else -> throw IllegalArgumentException("Unknown option: $it\n$help")
                     }
                 }
                 else -> throw IllegalArgumentException("Unknown option: $it\n$help")
             }
         }
+        val testSuites = suites.filterWith(filters).toList()
+
+        return TestSettings(testSuites, listeners, logger, runTests, iterations, useExitCode)
     }
 
-    inner class FilteredSuite(val innerSuite: TestSuite) : TestSuite by innerSuite {
+    private fun String.substringEscaped(range: IntRange) = this.substring(range).let { if (it.isNotEmpty()) Regex.escape(it) else "" }
-
-        private val TestCase.matchFilters: Boolean
-            get() = filters.all { it(this) }
-
-        override val size: Int
-            get() = testCases.size
-
-        override val testCases: Map<String, TestCase> = innerSuite.testCases.filter { it.value.matchFilters }
-        override fun toString() = innerSuite.toString()
-    }
-
-    private fun filterSuites(): Collection<TestSuite> = suites.map { FilteredSuite(it) }
-
-    // TODO: Support short aliases.
-    // TODO: Support several test iterations.
-    /**
-     *  Initialize the TestRunner using the command line options passed in [args].
-     *  Returns: true if tests may be ran, false otherwise (there are unrecognized options or just help).
-     *  The following options are available:
-     *
-     *  --gtest_list_tests
-     *  --ktest_list_tests                                  - Show all available tests.
-     *
-     *  --gtest_filter=POSITIVE_PATTERNS[-NEGATIVE_PATTERNS]
-     *  --ktest_filter=POSITIVE_PATTERNS[-NEGATIVE_PATTERNS] - Run only the tests whose name matches one of the
-     *                                                        positive patterns but none of the negative patterns.
-     *                                                        '?' matches any single character; '*' matches any
-     *                                                        substring; ':' separates two patterns.
-     *
-     *  --ktest_regex_filter=PATTERN                        - Run only the tests whose name matches the pattern.
-     *                                                        The pattern is a Kotlin regular expression.
-     *
-     *  --ktest_negative_regex_filter=PATTERN               - Run only the tests whose name doesn't match the pattern.
-     *                                                        The pattern is a Kotlin regular expression.
-     *
-     *  --gtest_repeat=COUNT
-     *  --ktest_repeat=COUNT                                - Run the tests repeatedly.
-     *                                                        Use a negative count to repeat forever.
-     *
-     *  --ktest_logger=GTEST|TEAMCITY|SIMPLE|SILENT         - Use the specified output format. The default one is GTEST.
-     *
-     *  --ktest_no_exit_code                                - Don't return a non-zero exit code if there are failing tests.
-     */
-
-    private fun String.substringEscaped(range: IntRange) =
-            this.substring(range).let { if (it.isNotEmpty()) Regex.escape(it) else "" }
 
     private fun String.toGTestPatterns() = splitToSequence(':').map { pattern ->
         val result = StringBuilder()
@@ -125,7 +104,9 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
         return@map result.toString().toRegex()
     }.toList()
 
-    private fun setGTestFilterFromArg(filter: String) {
+    // region filters
+
+    private fun gTestFilterFromArg(filter: String): TestFilter {
         if (filter.isEmpty()) {
             throw IllegalArgumentException("Empty filter")
         }
@@ -137,23 +118,23 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
         val positivePatterns = filters[0].toGTestPatterns()
         val negativePatterns = filters.getOrNull(1)?.toGTestPatterns() ?: emptyList()
 
-        this.filters.add { testCase ->
+        return { testCase ->
             positivePatterns.any { testCase.prettyName.matches(it) } &&
                     negativePatterns.none { testCase.prettyName.matches(it) }
         }
     }
 
-    private fun setRegexFilterFromArg(filter: String, positive: Boolean = true) {
+    private fun regexFilterFromArg(filter: String, positive: Boolean = true): TestFilter {
         if (filter.isEmpty()) {
             throw IllegalArgumentException("Empty filter")
         }
         val pattern = filter.toRegex()
-        filters.add { testCase ->
+        return { testCase ->
             testCase.prettyName.matches(pattern) == positive
         }
     }
 
-    private fun setGradleFilterFromArg(filter: String, positive: Boolean = true) {
+    private fun gradleFilterFromArg(filter: String, positive: Boolean = true): TestFilter {
         if (filter.isEmpty()) {
             throw IllegalArgumentException("Empty filter")
         }
@@ -165,25 +146,39 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
         fun TestCase.matches(pattern: Regex) =
             prettyName.matches(pattern) || suite.name.matches(pattern)
 
-        if (positive) {
+        return if (positive) {
-            filters.add { testCase ->
+            { testCase ->
                 patterns.any { testCase.matches(it) }
             }
         } else {
-            filters.add { testCase ->
+            { testCase ->
                 patterns.none { testCase.matches(it) }
             }
         }
     }
 
-    private fun setLoggerFromArg(logger: String) {
+    inner class FilteredSuite(val innerSuite: TestSuite, val filters: Collection<TestFilter>) : TestSuite by innerSuite {
-        when (logger.uppercase()) {
+        private val TestCase.matchFilters: Boolean
-            "GTEST" -> this.logger = GTestLogger()
+            get() = filters.all { it(this) }
-            "TEAMCITY" -> this.logger = TeamCityLogger()
+
-            "SIMPLE" -> this.logger = SimpleTestLogger()
+        override val size: Int
-            "SILENT" -> this.logger = SilentTestLogger()
+            get() = testCases.size
-            else -> throw IllegalArgumentException("Unknown logger type. Available types: GTEST, TEAMCITY, SIMPLE")
+
-        }
+        override val testCases: Map<String, TestCase> = innerSuite.testCases.filter { it.value.matchFilters }
+        override fun toString() = innerSuite.toString()
+    }
+
+    private fun Collection<TestSuite>.filterWith(filters: Collection<TestFilter>): Collection<TestSuite> =
+            map { FilteredSuite(it, filters) }
+
+    // endregion
+
+    private fun loggerFromArg(logger: String): TestLogger = when (logger.uppercase()) {
+        "GTEST" -> GTestLogger()
+        "TEAMCITY" -> TeamCityLogger()
+        "SIMPLE" -> SimpleTestLogger()
+        "SILENT" -> SilentTestLogger()
+        else -> throw IllegalArgumentException("Unknown logger type. Available types: GTEST, TEAMCITY, SIMPLE")
     }
 
     private val help: String
@@ -221,10 +216,27 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
             |
             |--ktest_no_exit_code                                - Don't return a non-zero exit code if there are failing tests.
         """.trimMargin()
+}
+
+@OptIn(kotlin.experimental.ExperimentalNativeApi::class)
+private typealias TestFilter = (TestCase) -> Boolean
+
+/**
+ * Test runner.
+ *
+ * Runs tests for the given [TestSettings].
+ */
+@OptIn(kotlin.time.ExperimentalTime::class)
+@ExperimentalNativeApi
+internal class TestRunner(val settings: TestSettings) {
+    constructor(suites: List<TestSuite>, args: Array<String>) : this(TestProcessor(suites, args).process())
+
+    var exitCode = 0
+        private set
 
     private inline fun sendToListeners(event: TestListener.() -> Unit) {
-        logger.event()
+        settings.logger.event()
-        listeners.forEach(event)
+        settings.listeners.forEach(event)
     }
 
     private fun TestSuite.run() {
@@ -249,7 +261,7 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
                     sendToListeners { pass(testCase, startTime.elapsedNow().inWholeMilliseconds) }
                 } catch (e: Throwable) {
                     sendToListeners { fail(testCase, e, startTime.elapsedNow().inWholeMilliseconds) }
-                    if (useExitCode) {
+                    if (settings.useExitCode) {
                         exitCode = 1
                     }
                 }
@@ -259,8 +271,8 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
     }
 
     private fun runIteration(iteration: Int) {
-        val suitesFiltered = filterSuites()
+        val suitesFiltered = settings.testSuites
-        sendToListeners { startIteration(this@TestRunner, iteration, suitesFiltered) }
+        sendToListeners { startIteration(settings, iteration, suitesFiltered) }
         val iterationTime = measureTime {
             suitesFiltered.forEach {
                 if (it.ignored) {
@@ -276,21 +288,21 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
                 }
             }
         }.inWholeMilliseconds
-        sendToListeners { finishIteration(this@TestRunner, iteration, iterationTime) }
+        sendToListeners { finishIteration(settings, iteration, iterationTime) }
     }
 
     fun run(): Int {
-        if (!runTests)
+        if (!settings.runTests)
             return 0
-        sendToListeners { startTesting(this@TestRunner) }
+        sendToListeners { startTesting(settings) }
         val totalTime = measureTime {
             var i = 1
-            while (i <= iterations || iterations < 0) {
+            while (i <= settings.iterations || settings.iterations < 0) {
                 runIteration(i)
                 i++
             }
         }.inWholeMilliseconds
-        sendToListeners { finishTesting(this@TestRunner, totalTime) }
+        sendToListeners { finishTesting(settings, totalTime) }
         return exitCode
     }
 }

kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestSuite.kt

@@ -10,26 +10,106 @@ import kotlin.IllegalArgumentException
 import kotlin.system.getTimeMillis
 import kotlin.system.measureTimeMillis
 
+/**
+ * Represents a test case, that is a Kotlin test method marked with `@Test` annotaion.
+ *
+ * @see kotlin.test.Test
+ */
 @ExperimentalNativeApi
 public interface TestCase {
     val name: String
+
+    /**
+     * Shows if this test is ignored by the `@Ignore` annotation.
+     *
+     * @see kotlin.test.Ignore
+     */
     val ignored: Boolean
+
+    /**
+     * Test Suite the test belongs to.
+     */
     val suite: TestSuite
 
-    fun run()
+    /**
+     * Runs all methods that were marked with `@BeforeTest` annotation.
+     *
+     * @see kotlin.test.BeforeTest
+     */
+    fun doBefore()
+
+    /**
+     * Runs the test method itself
+     *
+     * @see kotlin.test.Test
+     */
+    fun doRun()
+
+    /**
+     * Runs test with its before and after functions.
+     */
+    fun run() {
+        try {
+            doBefore()
+            doRun()
+        } finally {
+            doAfter()
+        }
+    }
+
+    /**
+     * Runs all methods that were marked with `@AfterTest` annotation.
+     *
+     * @see kotlin.test.AfterTest
+     */
+    fun doAfter()
 }
 
 @ExperimentalNativeApi
 internal val TestCase.prettyName get() = "${suite.name}.$name"
 
+/**
+ * Represents Test suite, a suite that contains test cases.
+ *
+ * Test suite can be either test class containing test cases
+ * or a top level suite for top level methods.
+ */
 @ExperimentalNativeApi
 public interface TestSuite {
+    /**
+     * Test suite name.
+     */
     val name: String
+
+    /**
+     * Shows if the suite is ignored with `@Ignore` annotation.
+     *
+     * @see kotlin.test.Ignore
+     */
     val ignored: Boolean
+
+    /**
+     * Test cases this test suite contains
+     */
     val testCases: Map<String, TestCase>
+
+    /**
+     * Number of test cases
+     */
     val size : Int
 
+    /**
+     * Executes all methods marked with `@BeforeClass` annotation
+     *
+     * @see kotlin.test.BeforeClass
+     */
     fun doBeforeClass()
+
+    /**
+     * Executes all methods marked with `@AfterClass` annotation
+     *
+     * @see @see kotlin.test.AfterClass
+     */
     fun doAfterClass()
 }
 
@@ -85,14 +165,18 @@ public abstract class BaseClassSuite<INSTANCE, COMPANION>(name: String, ignored:
                                         ignored: Boolean)
         : BasicTestCase<INSTANCE.() -> Unit>(name, suite, testFunction, ignored) {
 
-        override fun run() {
+        internal val instance: INSTANCE by lazy { suite.createInstance() }
-            val instance = suite.createInstance()
+
-            try {
+        override fun doBefore() {
-                suite.before.forEach { instance.it() }
+            suite.before.forEach { instance.it() }
-                instance.testFunction()
+        }
-            } finally {
+
-                suite.after.forEach { instance.it() }
+        override fun doAfter() {
-            }
+            suite.after.forEach { instance.it() }
+        }
+
+        override fun doRun() {
+            instance.testFunction()
         }
     }
 
@@ -146,14 +230,16 @@ public class TopLevelSuite(name: String): AbstractTestSuite<TopLevelFun>(name, f
 
     class TestCase(name: String, override val suite: TopLevelSuite, testFunction: TopLevelFun, ignored: Boolean)
         : BasicTestCase<TopLevelFun>(name, suite, testFunction, ignored) {
+        override fun doBefore() {
+            suite.before.forEach { it() }
+        }
 
-        override fun run() {
+        override fun doAfter() {
-            try {
+            suite.after.forEach { it() }
-                suite.before.forEach { it() }
+        }
-                testFunction()
+
-            } finally {
+        override fun doRun() {
-                suite.after.forEach { it() }
+            testFunction()
-            }
         }
     }