diff --git a/runtime/src/main/kotlin/konan/test/Listeners.kt b/runtime/src/main/kotlin/konan/test/GTestLogger.kt similarity index 63% rename from runtime/src/main/kotlin/konan/test/Listeners.kt rename to runtime/src/main/kotlin/konan/test/GTestLogger.kt index 0b9cbc65f9c..04b542fdc8e 100644 --- a/runtime/src/main/kotlin/konan/test/Listeners.kt +++ b/runtime/src/main/kotlin/konan/test/GTestLogger.kt @@ -1,25 +1,16 @@ package konan.test -val TestRunner.totalTests - get() = suites.map { it.size }.reduce { sum, size -> sum + size } - -val TestRunner.totalSuites - get() = suites.size - -class GTestListener: TestListener, AbstractTestStatistics() { - - private val TestCase.prettyName get() = "${suite.name}.$name" - private val failedTests_ = mutableListOf() - - private var totalSuites = 0 +class GTestLogger : TestLoggerWithStatistics() { override fun startTesting(runner: TestRunner) { + super.startTesting(runner) println("[==========] Running ${runner.totalTests} tests from ${runner.totalSuites} test case.") - println("[----------] Global test environment set-up.") // TODO: just emulation + // Just hack to deal with the Clion parser. TODO: Remove it after changes in the parser. + println("[----------] Global test environment set-up.") } - private fun printResults(timeMillis: Long) { - println("[----------] Global test environment tear-down") + private fun printResults(timeMillis: Long) = with (statistics) { + println("[----------] Global test environment tear-down") // Just hack to deal with the Clion parser. println("[==========] $total tests from $totalSuites test cases ran. ($timeMillis ms total)") println("[ PASSED ] $passed tests.") if (hasFailedTests) { @@ -37,31 +28,22 @@ class GTestListener: TestListener, AbstractTestStatistics() { override fun endTesting(runner: TestRunner, timeMillis: Long) = printResults(timeMillis) override fun startSuite(suite: TestSuite) = println("[----------] ${suite.size} tests from ${suite.name}") + override fun endSuite(suite: TestSuite, timeMillis: Long) { - totalSuites++ + super.endSuite(suite, timeMillis) println("[----------] ${suite.size} tests from ${suite.name} ($timeMillis ms total)\n") } - override fun ignoreSuite(suite: TestSuite) {} - override fun start(testCase: TestCase) = println("[ RUN ] ${testCase.prettyName}") override fun pass(testCase: TestCase, timeMillis: Long) { - registerPass() + super.pass(testCase, timeMillis) println("[ OK ] ${testCase.prettyName} ($timeMillis ms)") } override fun fail(testCase: TestCase, e: Throwable, timeMillis: Long) { - registerFail() - failedTests_.add(testCase) + super.fail(testCase, e, timeMillis) e.printStackTrace() println("[ FAILED ] ${testCase.prettyName} ($timeMillis ms)") } - - override fun ignore(testCase: TestCase) = registerIgnore() - - override val failedTests: Collection - get() = failedTests_ - override val hasFailedTests: Boolean - get() = failed != 0 } diff --git a/runtime/src/main/kotlin/konan/test/Launcher.kt b/runtime/src/main/kotlin/konan/test/Launcher.kt index ec8b18bf799..81ca741ce89 100644 --- a/runtime/src/main/kotlin/konan/test/Launcher.kt +++ b/runtime/src/main/kotlin/konan/test/Launcher.kt @@ -2,30 +2,7 @@ package konan.test import kotlin.system.exitProcess -class SimpleTestListener: TestListener { - - var hasFails = false - private set - - override fun startTesting(runner: TestRunner) = println("Starting testing") - override fun endTesting(runner: TestRunner, timeMillis: Long) = println("Testing finished") - - override fun startSuite(suite: TestSuite) = println("Starting test suite: $suite") - override fun endSuite(suite: TestSuite, timeMillis: Long) = println("Test suite finished: $suite") - override fun ignoreSuite(suite: TestSuite) = println("Test suite ignored: $suite") - - override fun start(testCase: TestCase) = println("Starting test case: $testCase") - override fun pass(testCase: TestCase, timeMillis: Long) = println("Passed: $testCase") - override fun fail(testCase: TestCase, e: Throwable, timeMillis: Long) { - println("Failed: $testCase. Exception:") - e.printStackTrace() - hasFails = true - } - override fun ignore(testCase: TestCase) = println("Ignore: $testCase") -} - fun main(args:Array) { - val listener = GTestListener() - TestRunner.run(listener) - exitProcess( if (listener.hasFailedTests) -1 else 0 ) + val exitCode = TestRunner.run(args) + exitProcess(exitCode) } diff --git a/runtime/src/main/kotlin/konan/test/TestListener.kt b/runtime/src/main/kotlin/konan/test/TestListener.kt index 9e1873fa9d3..307fdecb2ea 100644 --- a/runtime/src/main/kotlin/konan/test/TestListener.kt +++ b/runtime/src/main/kotlin/konan/test/TestListener.kt @@ -14,24 +14,14 @@ interface TestListener { fun ignore(testCase: TestCase) } -interface TestStatistics { - val total: Int - val passed: Int - val failed: Int - val ignored: Int - - // TODO: Do we need such properties for other cases? - val failedTests: Collection - val hasFailedTests: Boolean +open class BaseTestListener: TestListener { + override fun startTesting(runner: TestRunner) {} + override fun endTesting(runner: TestRunner, timeMillis: Long) {} + override fun startSuite(suite: TestSuite) {} + override fun endSuite(suite: TestSuite, timeMillis: Long) {} + override fun ignoreSuite(suite: TestSuite) {} + override fun start(testCase: TestCase) {} + override fun pass(testCase: TestCase, timeMillis: Long) {} + override fun fail(testCase: TestCase, e: Throwable, timeMillis: Long) {} + override fun ignore(testCase: TestCase) {} } - -abstract class AbstractTestStatistics: TestStatistics { - override var total: Int = 0; protected set - override var passed: Int = 0; protected set - override var failed: Int = 0; protected set - override var ignored: Int = 0; protected set - - protected fun registerPass() { total++; passed++ } - protected fun registerFail() { total++; failed++ } - protected fun registerIgnore() { total++; ignored++ } -} \ No newline at end of file diff --git a/runtime/src/main/kotlin/konan/test/TestLogger.kt b/runtime/src/main/kotlin/konan/test/TestLogger.kt new file mode 100644 index 00000000000..b557b28610e --- /dev/null +++ b/runtime/src/main/kotlin/konan/test/TestLogger.kt @@ -0,0 +1,60 @@ +package konan.test + +interface TestLogger: TestListener { + fun logTestList(runner: TestRunner) + fun log(message: String) +} + +open class BaseTestLogger: BaseTestListener(), TestLogger { + + protected val TestRunner.totalTests + get() = filterTests().filter { !it.ignored }.size + + protected val TestRunner.totalSuites + get() = suites.filter { !it.ignored }.size + + override fun log(message: String) = println(message) + override fun logTestList(runner: TestRunner) { + runner.filterTests().groupBy { it.suite }.forEach { (suite, tests) -> + println("${suite.name}.") + tests.forEach { + println(" ${it.name}") + } + } + } +} + +open class TestLoggerWithStatistics: BaseTestLogger() { + + protected val statistics = MutableTestStatistics() + + override fun startTesting(runner: TestRunner) = statistics.reset() + override fun endSuite(suite: TestSuite, timeMillis: Long) = statistics.registerSuite() + override fun pass(testCase: TestCase, timeMillis: Long) = statistics.registerPass() + override fun fail(testCase: TestCase, e: Throwable, timeMillis: Long) = statistics.registerFail(testCase) + override fun ignore(testCase: TestCase) = statistics.registerIgnore() +} + +class SilentTestLogger: BaseTestLogger() { + override fun logTestList(runner: TestRunner) {} + override fun log(message: String) {} +} + +class SimpleTestLogger: BaseTestLogger() { + override fun startTesting(runner: TestRunner) = println("Starting testing") + override fun endTesting(runner: TestRunner, timeMillis: Long) = println("Testing finished") + + override fun startSuite(suite: TestSuite) = println("Starting test suite: $suite") + override fun endSuite(suite: TestSuite, timeMillis: Long) = println("Test suite finished: $suite") + override fun ignoreSuite(suite: TestSuite) = println("Test suite ignored: $suite") + + override fun start(testCase: TestCase) = println("Starting test case: $testCase") + override fun pass(testCase: TestCase, timeMillis: Long) = println("Passed: $testCase") + override fun fail(testCase: TestCase, e: Throwable, timeMillis: Long) { + println("Failed: $testCase. Exception:") + e.printStackTrace() + } + override fun ignore(testCase: TestCase) = println("Ignore: $testCase") +} + + diff --git a/runtime/src/main/kotlin/konan/test/TestRunner.kt b/runtime/src/main/kotlin/konan/test/TestRunner.kt index 262ddcce47e..ec7cc7ed295 100644 --- a/runtime/src/main/kotlin/konan/test/TestRunner.kt +++ b/runtime/src/main/kotlin/konan/test/TestRunner.kt @@ -1,29 +1,223 @@ package konan.test +import kotlin.IllegalArgumentException +import kotlin.system.getTimeMillis import kotlin.system.measureTimeMillis +import kotlin.text.StringBuilder object TestRunner { - private val _suites = mutableListOf() - val suites: Collection get() = _suites + private val suites_ = mutableListOf() + val suites: Collection + get() = suites_ - fun register(suite: TestSuite) = _suites.add(suite) - fun register(suites: Iterable) = _suites.addAll(suites) - fun register(vararg suites: TestSuite) = _suites.addAll(suites) + var logger: TestLogger = GTestLogger() - fun run(listener: TestListener) = with(listener) { - startTesting(this@TestRunner) - val totalTime = measureTimeMillis { - suites.forEach { - if (it.ignored) { - ignoreSuite(it) - } else { - startSuite(it) - val time = measureTimeMillis { it.run(listener) } - endSuite(it, time) + val listeners = mutableSetOf() + val filters = mutableListOf<(TestCase) -> Boolean>() + + var exitCode = 0 + private set + + private val TestCase.matchFilters: Boolean + get() = filters.map { it(this) }.all { it } + + private val TestSuite.testCasesFiltered + get() = testCases.values.filter { it.matchFilters } + + // TODO: We can cache it. + fun filterTests(): Collection = suites.flatMap { it.testCasesFiltered } + + fun register(suite: TestSuite) = suites_.add(suite) + fun register(suites: Iterable) = suites_.addAll(suites) + fun register(vararg suites: TestSuite) = suites_.addAll(suites) + + // TODO: Support short aliases. + // TODO: Support several test iterations. + /** + * Initialize the TestRunner using the command line options passed in [args]. + * Returns: true if tests may be ran, false otherwise (there are unrecognized options or just help). + * The following options are available: + * + * --gtest_list_tests + * --ktest_list_tests - Show all available tests. + * + * --gtest_filter=POSTIVE_PATTERNS[-NEGATIVE_PATTERNS] - Run only the tests whose name matches one of the + * --ktest_filter=POSTIVE_PATTERNS[-NEGATIVE_PATTERNS] positive patterns but none of the negative patterns. + * '?' matches any single character; '*' matches any + * substring; ':' separates two patterns. + * + * --ktest_regex_filter=PATTERN - Run only the tests whose name matches the pattern. + * The pattern is a Kotlin regular expression. + * + * --ktest_negative_regex_filter=PATTERN - Run only the tests whose name doesn't match the pattern. + * The pattern is a Kotlin regular expression. + * + * --ktest_logger=GTEST|TEAMCITY|SIMPLE|SILENT - Use the specified output format. The default one is GTEST. + */ + fun useArgs(args: Array): Boolean { + try { + return parseArgs(args) + } catch (e: IllegalArgumentException) { + logger.log("Error: ${e.message}") + return false + } + } + + private fun parseArgs(args: Array): Boolean { + var result = true + args.filter { + it.startsWith("--gtest_") || it.startsWith("--ktest_") || it == "--help" + }.forEach { + val arg = it.split('=') + when (arg.size) { + 1 -> when (arg[0]) { + "--gtest_list_tests", + "--ktest_list_tests" -> { logger.logTestList(this); result = false } + "--help" -> { logger.log(help); result = false } + else -> throw IllegalArgumentException("Unknown option: $it\n$help") + } + 2 -> { + val key = arg[0] + val value = arg[1] + when (key) { + "--ktest_logger" -> setLoggerFromArg(value) + "--gtest_filter", + "--ktest_filter" -> setGTestFilterFromArg(value) + "--ktest_regex_filter"-> setRegexFilterFromArg(value, true) + "--ktest_negative_regex_filter" -> setRegexFilterFromArg(value, false) + else -> throw IllegalArgumentException("Unknown option: $it\n$help") + } + } + else -> throw IllegalArgumentException("Unknown option: $it\n$help") + } + } + return result + } + + private fun String.substringEscaped(range: IntRange) = + this.substring(range).let { if(it.isNotEmpty()) Regex.escape(it) else "" } + + private fun String.toGTestPatterns() = splitToSequence(':').map { pattern -> + val result = StringBuilder() + var prevIndex = 0 + pattern.forEachIndexed { index, c -> + if (c == '*' || c == '?') { + result.append(pattern.substringEscaped(prevIndex until index)) + prevIndex = index+1 + result.append( if (c == '*') ".*" else ".") + } + } + result.append(pattern.substringEscaped(prevIndex until pattern.length)) + return@map result.toString().toRegex() + }.toList() + + private fun setGTestFilterFromArg(filter: String) { + if (filter.isEmpty()) { + throw IllegalArgumentException("Empty filter") + } + val filters = filter.split('-') + if (filters.size > 2) { + throw IllegalArgumentException("Wrong pattern syntax: $filter.") + } + + val positivePatterns = filters[0].toGTestPatterns() + val negativePatterns = filters.getOrNull(1)?.toGTestPatterns() ?: emptyList() + + this.filters.add { testCase -> + positivePatterns.all { testCase.prettyName.matches(it) } && + negativePatterns.none { testCase.prettyName.matches(it) } + } + } + + private fun setRegexFilterFromArg(filter: String, positive: Boolean = true) { + if (filter.isEmpty()) { + throw IllegalArgumentException("Empty filter") + } + val pattern = filter.toRegex() + filters.add { testCase -> + testCase.prettyName.matches(pattern) == positive + } + } + + private fun setLoggerFromArg(logger: String) { + when(logger.toUpperCase()) { + "GTEST" -> this.logger = GTestLogger() + "TEAMCITY" -> TODO() //this.logger = TeamCityLogger() + "SIMPLE" -> this.logger = SimpleTestLogger() + "SILENT" -> this.logger = SilentTestLogger() + else -> throw IllegalArgumentException("Unknown logger type. Available types: GTEST, TEAMCITY, SIMPLE") + } + } + + private val help: String + get() = """ + |Available options: + |--gtest_list_tests + |--ktest_list_tests - Show all available tests. + | + |--gtest_filter=POSTIVE_PATTERNS[-NEGATIVE_PATTERNS] + |--ktest_filter=POSTIVE_PATTERNS[-NEGATIVE_PATTERNS] - Run only the tests whose name matches one of the + | positive patterns but none of the negative patterns. + | '?' matches any single character; '*' matches any + | substring; ':' separates two patterns. + | + |--ktest_regex_filter=PATTERN - Run only the tests whose name matches the pattern. + | The pattern is a Kotlin regular expression. + | + |--ktest_negative_regex_filter=PATTERN - Run only the tests whose name doesn't match the pattern. + | The pattern is a Kotlin regular expression. + | + |--ktest_logger=GTEST|TEAMCITY|SIMPLE|SILENT - Use the specified output format. The default one is GTEST. + """.trimMargin() + + private inline fun sendToListeners(event: TestListener.() -> Unit) { + logger.event() + listeners.forEach(event) + } + + private fun TestSuite.run() { + doBeforeClass() + testCasesFiltered.forEach { testCase -> + if (testCase.ignored) { + sendToListeners { ignore(testCase) } + } else { + val startTime = getTimeMillis() + try { + sendToListeners { start(testCase) } + testCase.run() + sendToListeners { pass(testCase, getTimeMillis() - startTime) } + } catch (e: Throwable) { + sendToListeners { fail(testCase, e, getTimeMillis() - startTime) } + exitCode = 1 } } } - endTesting(this@TestRunner, totalTime) + doAfterClass() + } + + fun run(args: Array): Int { + return if (useArgs(args)) { + run() + } else { + 0 + } + } + + fun run(): Int { + sendToListeners { startTesting(this@TestRunner) } + val totalTime = measureTimeMillis { + suites.forEach { + if (it.ignored) { + sendToListeners { ignoreSuite(it) } + } else { + sendToListeners { startSuite(it) } + val time = measureTimeMillis { it.run() } + sendToListeners { endSuite(it, time) } + } + } + } + sendToListeners { endTesting(this@TestRunner, totalTime) } + return exitCode } } diff --git a/runtime/src/main/kotlin/konan/test/TestStatistics.kt b/runtime/src/main/kotlin/konan/test/TestStatistics.kt new file mode 100644 index 00000000000..b26f8a29d26 --- /dev/null +++ b/runtime/src/main/kotlin/konan/test/TestStatistics.kt @@ -0,0 +1,57 @@ +package konan.test + +interface TestStatistics { + val total: Int + val passed: Int + val failed: Int + val ignored: Int + + val totalSuites: Int + + val failedTests: Collection + val hasFailedTests: Boolean + + object EMPTY: TestStatistics { + override val total: Int get() = 0 + override val passed: Int get() = 0 + override val failed: Int get() = 0 + override val ignored: Int get() = 0 + override val totalSuites: Int get() = 0 + + override val failedTests: Collection = emptyList() + override val hasFailedTests: Boolean = false + } +} + +class MutableTestStatistics: TestStatistics { + + override var total: Int = 0; private set + override var passed: Int = 0; private set + override var ignored: Int = 0; private set + + override var totalSuites: Int = 0; private set + + override val failed: Int + get() = failedTests_.size + + override val hasFailedTests: Boolean + get() = failedTests_.isNotEmpty() + + private val failedTests_ = mutableListOf() + override val failedTests: Collection + get() = failedTests_ + + fun registerSuite() { totalSuites++ } + + fun registerPass() { total++; passed++ } + fun registerFail(testCase: TestCase) { total++; failedTests_.add(testCase) } + fun registerIgnore() { total++; ignored++ } + + fun reset() { + total = 0 + passed = 0 + ignored = 0 + totalSuites = 0 + failedTests_.clear() + } +} diff --git a/runtime/src/main/kotlin/konan/test/TestSuite.kt b/runtime/src/main/kotlin/konan/test/TestSuite.kt index febe1c3fda8..a6126aef91d 100644 --- a/runtime/src/main/kotlin/konan/test/TestSuite.kt +++ b/runtime/src/main/kotlin/konan/test/TestSuite.kt @@ -8,14 +8,20 @@ interface TestCase { val name: String val ignored: Boolean val suite: TestSuite + + fun run() } +internal val TestCase.prettyName get() = "${suite.name}.$name" + interface TestSuite { val name: String val ignored: Boolean val testCases: Map val size : Int - fun run(listener: TestListener) + + fun doBeforeClass() + fun doAfterClass() } enum class TestFunctionKind { @@ -30,7 +36,7 @@ abstract class AbstractTestSuite>(override val name: String, o override fun toString(): String = name // TODO: Make inner and remove the type param when the bug is fixed. - class BasicTestCase>( + abstract class BasicTestCase>( override val name: String, override val suite: AbstractTestSuite, val testFunction: F, @@ -48,13 +54,7 @@ abstract class AbstractTestSuite>(override val name: String, o fun registerTestCase(name: String, testFunction: F, ignored: Boolean) = registerTestCase(createTestCase(name, testFunction, ignored)) - fun createTestCase(name: String, testFunction: F, ignored: Boolean) = - BasicTestCase(name, this, testFunction, ignored) - - protected abstract fun doBeforeClass() - protected abstract fun doAfterClass() - - protected abstract fun doTest(testCase: BasicTestCase) + abstract fun createTestCase(name: String, testFunction: F, ignored: Boolean): BasicTestCase init { TestRunner.register(this) @@ -62,30 +62,28 @@ abstract class AbstractTestSuite>(override val name: String, o override val size: Int get() = testCases.size - - override fun run(listener: TestListener) { - doBeforeClass() - testCases.values.forEach { - if (it.ignored) { - listener.ignore(it) - } else { - val startTime = getTimeMillis() - try { - listener.start(it) - doTest(it) - listener.pass(it, getTimeMillis() - startTime) - } catch (e: Throwable) { - listener.fail(it, e, getTimeMillis() - startTime) - } - } - } - doAfterClass() - } } abstract class BaseClassSuite(name: String, ignored: Boolean) : AbstractTestSuite Unit>(name, ignored) { + class TestCase(name: String, + override val suite: BaseClassSuite, + testFunction: INSTANCE.() -> Unit, + ignored: Boolean) + : BasicTestCase Unit>(name, suite, testFunction, ignored) { + + override fun run() { + val instance = suite.createInstance() + try { + suite.before.forEach { instance.it() } + instance.testFunction() + } finally { + suite.after.forEach { instance.it() } + } + } + } + // These two methods are overrided in test suite classes generated by the compiler. abstract fun createInstance(): INSTANCE open fun getCompanion(): COMPANION = throw NotImplementedError("Test class has no companion object") @@ -124,22 +122,28 @@ abstract class BaseClassSuite(name: String, ignored: Boolea override fun doBeforeClass() = beforeClass.forEach { getCompanion().it() } override fun doAfterClass() = afterClass.forEach { getCompanion().it() } - override fun doTest(testCase: BasicTestCase Unit>) { - val instance = createInstance() - val testFunction = testCase.testFunction - try { - before.forEach { instance.it() } - instance.testFunction() - } finally { - after.forEach { instance.it() } - } - } + override fun createTestCase(name: String, testFunction: INSTANCE.() -> Unit, ignored: Boolean) + : BasicTestCase Unit> = + TestCase(name, this, testFunction, ignored) } private typealias TopLevelFun = () -> Unit class TopLevelSuite(name: String): AbstractTestSuite(name, false) { + class TestCase(name: String, override val suite: TopLevelSuite, testFunction: TopLevelFun, ignored: Boolean) + : BasicTestCase(name, suite, testFunction, ignored) { + + override fun run() { + try { + suite.before.forEach { it() } + testFunction() + } finally { + suite.after.forEach { it() } + } + } + } + private val specialFunctions = mutableMapOf>() private fun getFunctions(type: TestFunctionKind) = specialFunctions.getOrPut(type) { mutableSetOf() } @@ -153,11 +157,6 @@ class TopLevelSuite(name: String): AbstractTestSuite(name, false) { override fun doBeforeClass() = beforeClass.forEach { it() } override fun doAfterClass() = afterClass.forEach { it() } - override fun doTest(testCase: BasicTestCase<() -> Unit>) = - try { - before.forEach { it() } - testCase.testFunction() - } finally { - after.forEach { it() } - } + override fun createTestCase(name: String, testFunction: TopLevelFun, ignored: Boolean) + : BasicTestCase = TestCase(name, this, testFunction, ignored) } \ No newline at end of file