diff --git a/runtime/src/main/kotlin/konan/test/GTestLogger.kt b/runtime/src/main/kotlin/konan/test/GTestLogger.kt index 8a9d5becc07..709b6ae067d 100644 --- a/runtime/src/main/kotlin/konan/test/GTestLogger.kt +++ b/runtime/src/main/kotlin/konan/test/GTestLogger.kt @@ -9,6 +9,12 @@ class GTestLogger : TestLoggerWithStatistics() { println("[----------] Global test environment set-up.") } + override fun startIteration(runner: TestRunner, iteration: Int) { + if (runner.iterations != 1) { + println("\nRepeating all tests (iteration $iteration) . . .\n") + } + } + private fun printResults(timeMillis: Long) = with (statistics) { println("[----------] Global test environment tear-down") // Just hack to deal with the Clion parser. println("[==========] $total tests from $totalSuites test cases ran. ($timeMillis ms total)") diff --git a/runtime/src/main/kotlin/konan/test/TestListener.kt b/runtime/src/main/kotlin/konan/test/TestListener.kt index b62065737a2..9ec3153728a 100644 --- a/runtime/src/main/kotlin/konan/test/TestListener.kt +++ b/runtime/src/main/kotlin/konan/test/TestListener.kt @@ -4,6 +4,9 @@ interface TestListener { fun startTesting(runner: TestRunner) fun finishTesting(runner: TestRunner, timeMillis: Long) + fun startIteration(runner: TestRunner, iteration: Int) + fun finishIteration(runner: TestRunner, iteration: Int, timeMillis: Long) + fun startSuite(suite: TestSuite) fun finishSuite(suite: TestSuite, timeMillis: Long) fun ignoreSuite(suite: TestSuite) @@ -17,6 +20,8 @@ interface TestListener { open class BaseTestListener: TestListener { override fun startTesting(runner: TestRunner) {} override fun finishTesting(runner: TestRunner, timeMillis: Long) {} + override fun startIteration(runner: TestRunner, iteration: Int) {} + override fun finishIteration(runner: TestRunner, iteration: Int, timeMillis: Long) {} override fun startSuite(suite: TestSuite) {} override fun finishSuite(suite: TestSuite, timeMillis: Long) {} override fun ignoreSuite(suite: TestSuite) {} diff --git a/runtime/src/main/kotlin/konan/test/TestLogger.kt b/runtime/src/main/kotlin/konan/test/TestLogger.kt index f25739e01a2..34b431cfbad 100644 --- a/runtime/src/main/kotlin/konan/test/TestLogger.kt +++ b/runtime/src/main/kotlin/konan/test/TestLogger.kt @@ -29,6 +29,8 @@ open class TestLoggerWithStatistics: BaseTestLogger() { protected val statistics = MutableTestStatistics() override fun startTesting(runner: TestRunner) = statistics.reset() + override fun startIteration(runner: TestRunner, iteration: Int) = statistics.reset() + override fun finishSuite(suite: TestSuite, timeMillis: Long) = statistics.registerSuite() override fun pass(testCase: TestCase, timeMillis: Long) = statistics.registerPass() override fun fail(testCase: TestCase, e: Throwable, timeMillis: Long) = statistics.registerFail(testCase) @@ -44,6 +46,10 @@ class SimpleTestLogger: BaseTestLogger() { override fun startTesting(runner: TestRunner) = println("Starting testing") override fun finishTesting(runner: TestRunner, timeMillis: Long) = println("Testing finished") + override fun startIteration(runner: TestRunner, iteration: Int) = println("Starting iteration: $iteration") + override fun finishIteration(runner: TestRunner, iteration: Int, timeMillis: Long) = + println("Iteration finished: $iteration") + override fun startSuite(suite: TestSuite) = println("Starting test suite: $suite") override fun finishSuite(suite: TestSuite, timeMillis: Long) = println("Test suite finished: $suite") override fun ignoreSuite(suite: TestSuite) = println("Test suite ignored: $suite") diff --git a/runtime/src/main/kotlin/konan/test/TestRunner.kt b/runtime/src/main/kotlin/konan/test/TestRunner.kt index 6df6c97d1b3..c3247c8e6f7 100644 --- a/runtime/src/main/kotlin/konan/test/TestRunner.kt +++ b/runtime/src/main/kotlin/konan/test/TestRunner.kt @@ -12,6 +12,7 @@ object TestRunner { get() = suites_ var logger: TestLogger = GTestLogger() + var iterations = 1 val listeners = mutableSetOf() val filters = mutableListOf<(TestCase) -> Boolean>() @@ -42,8 +43,9 @@ object TestRunner { * --gtest_list_tests * --ktest_list_tests - Show all available tests. * - * --gtest_filter=POSTIVE_PATTERNS[-NEGATIVE_PATTERNS] - Run only the tests whose name matches one of the - * --ktest_filter=POSTIVE_PATTERNS[-NEGATIVE_PATTERNS] positive patterns but none of the negative patterns. + * --gtest_filter=POSTIVE_PATTERNS[-NEGATIVE_PATTERNS] + * --ktest_filter=POSTIVE_PATTERNS[-NEGATIVE_PATTERNS] - Run only the tests whose name matches one of the + * positive patterns but none of the negative patterns. * '?' matches any single character; '*' matches any * substring; ':' separates two patterns. * @@ -53,6 +55,10 @@ object TestRunner { * --ktest_negative_regex_filter=PATTERN - Run only the tests whose name doesn't match the pattern. * The pattern is a Kotlin regular expression. * + * --gtest_repeat=COUNT + * --ktest_repeat=COUNT - Run the tests repeatedly. + * Use a negative count to repeat forever. + * * --ktest_logger=GTEST|TEAMCITY|SIMPLE|SILENT - Use the specified output format. The default one is GTEST. */ fun useArgs(args: Array): Boolean { @@ -67,13 +73,14 @@ object TestRunner { private fun parseArgs(args: Array): Boolean { var result = true args.filter { - it.startsWith("--gtest_") || it.startsWith("--ktest_") || it == "--help" + it.startsWith("--gtest_") || it.startsWith("--ktest_") || it == "--help" || it == "-h" }.forEach { val arg = it.split('=') when (arg.size) { 1 -> when (arg[0]) { "--gtest_list_tests", "--ktest_list_tests" -> { logger.logTestList(this); result = false } + "-h", "--help" -> { logger.log(help); result = false } else -> throw IllegalArgumentException("Unknown option: $it\n$help") } @@ -84,8 +91,11 @@ object TestRunner { "--ktest_logger" -> setLoggerFromArg(value) "--gtest_filter", "--ktest_filter" -> setGTestFilterFromArg(value) - "--ktest_regex_filter"-> setRegexFilterFromArg(value, true) + "--ktest_regex_filter" -> setRegexFilterFromArg(value, true) "--ktest_negative_regex_filter" -> setRegexFilterFromArg(value, false) + "--ktest_repeat", + "--gtest_repeat" -> iterations = value.toIntOrNull() ?: + throw IllegalArgumentException("Cannot parse number: $value") else -> throw IllegalArgumentException("Unknown option: $it\n$help") } } @@ -167,6 +177,9 @@ object TestRunner { | |--ktest_negative_regex_filter=PATTERN - Run only the tests whose name doesn't match the pattern. | The pattern is a Kotlin regular expression. + |--gtest_repeat=COUNT + |--ktest_repeat=COUNT - Run the tests repeatedly. + | Use a negative count to repeat forever. | |--ktest_logger=GTEST|TEAMCITY|SIMPLE|SILENT - Use the specified output format. The default one is GTEST. """.trimMargin() @@ -196,6 +209,22 @@ object TestRunner { doAfterClass() } + private fun runIteration(iteration: Int) { + sendToListeners { startIteration(this@TestRunner, iteration) } + val iterationTime = measureTimeMillis { + suites.forEach { + if (it.ignored) { + sendToListeners { ignoreSuite(it) } + } else { + sendToListeners { startSuite(it) } + val time = measureTimeMillis { it.run() } + sendToListeners { finishSuite(it, time) } + } + } + } + sendToListeners { finishIteration(this@TestRunner, iteration, iterationTime) } + } + fun run(args: Array): Int { return if (useArgs(args)) { run() @@ -207,14 +236,10 @@ object TestRunner { fun run(): Int { sendToListeners { startTesting(this@TestRunner) } val totalTime = measureTimeMillis { - suites.forEach { - if (it.ignored) { - sendToListeners { ignoreSuite(it) } - } else { - sendToListeners { startSuite(it) } - val time = measureTimeMillis { it.run() } - sendToListeners { finishSuite(it, time) } - } + var i = 1 + while (i <= iterations || iterations < 0) { + runIteration(i) + i++ } } sendToListeners { finishTesting(this@TestRunner, totalTime) }