unit-tests: Add runner parameters and refactor test runner

This commit is contained in:
Ilya Matveev
2017-09-20 17:18:21 +03:00
committed by ilmat192
parent f3fd6579a0
commit 3e02eab4c9
7 changed files with 393 additions and 134 deletions
@@ -1,25 +1,16 @@
package konan.test
val TestRunner.totalTests
get() = suites.map { it.size }.reduce { sum, size -> sum + size }
val TestRunner.totalSuites
get() = suites.size
class GTestListener: TestListener, AbstractTestStatistics() {
private val TestCase.prettyName get() = "${suite.name}.$name"
private val failedTests_ = mutableListOf<TestCase>()
private var totalSuites = 0
class GTestLogger : TestLoggerWithStatistics() {
override fun startTesting(runner: TestRunner) {
super.startTesting(runner)
println("[==========] Running ${runner.totalTests} tests from ${runner.totalSuites} test case.")
println("[----------] Global test environment set-up.") // TODO: just emulation
// Just hack to deal with the Clion parser. TODO: Remove it after changes in the parser.
println("[----------] Global test environment set-up.")
}
private fun printResults(timeMillis: Long) {
println("[----------] Global test environment tear-down")
private fun printResults(timeMillis: Long) = with (statistics) {
println("[----------] Global test environment tear-down") // Just hack to deal with the Clion parser.
println("[==========] $total tests from $totalSuites test cases ran. ($timeMillis ms total)")
println("[ PASSED ] $passed tests.")
if (hasFailedTests) {
@@ -37,31 +28,22 @@ class GTestListener: TestListener, AbstractTestStatistics() {
override fun endTesting(runner: TestRunner, timeMillis: Long) = printResults(timeMillis)
override fun startSuite(suite: TestSuite) = println("[----------] ${suite.size} tests from ${suite.name}")
override fun endSuite(suite: TestSuite, timeMillis: Long) {
totalSuites++
super.endSuite(suite, timeMillis)
println("[----------] ${suite.size} tests from ${suite.name} ($timeMillis ms total)\n")
}
override fun ignoreSuite(suite: TestSuite) {}
override fun start(testCase: TestCase) = println("[ RUN ] ${testCase.prettyName}")
override fun pass(testCase: TestCase, timeMillis: Long) {
registerPass()
super.pass(testCase, timeMillis)
println("[ OK ] ${testCase.prettyName} ($timeMillis ms)")
}
override fun fail(testCase: TestCase, e: Throwable, timeMillis: Long) {
registerFail()
failedTests_.add(testCase)
super.fail(testCase, e, timeMillis)
e.printStackTrace()
println("[ FAILED ] ${testCase.prettyName} ($timeMillis ms)")
}
override fun ignore(testCase: TestCase) = registerIgnore()
override val failedTests: Collection<TestCase>
get() = failedTests_
override val hasFailedTests: Boolean
get() = failed != 0
}
+2 -25
View File
@@ -2,30 +2,7 @@ package konan.test
import kotlin.system.exitProcess
class SimpleTestListener: TestListener {
var hasFails = false
private set
override fun startTesting(runner: TestRunner) = println("Starting testing")
override fun endTesting(runner: TestRunner, timeMillis: Long) = println("Testing finished")
override fun startSuite(suite: TestSuite) = println("Starting test suite: $suite")
override fun endSuite(suite: TestSuite, timeMillis: Long) = println("Test suite finished: $suite")
override fun ignoreSuite(suite: TestSuite) = println("Test suite ignored: $suite")
override fun start(testCase: TestCase) = println("Starting test case: $testCase")
override fun pass(testCase: TestCase, timeMillis: Long) = println("Passed: $testCase")
override fun fail(testCase: TestCase, e: Throwable, timeMillis: Long) {
println("Failed: $testCase. Exception:")
e.printStackTrace()
hasFails = true
}
override fun ignore(testCase: TestCase) = println("Ignore: $testCase")
}
fun main(args:Array<String>) {
val listener = GTestListener()
TestRunner.run(listener)
exitProcess( if (listener.hasFailedTests) -1 else 0 )
val exitCode = TestRunner.run(args)
exitProcess(exitCode)
}
@@ -14,24 +14,14 @@ interface TestListener {
fun ignore(testCase: TestCase)
}
interface TestStatistics {
val total: Int
val passed: Int
val failed: Int
val ignored: Int
// TODO: Do we need such properties for other cases?
val failedTests: Collection<TestCase>
val hasFailedTests: Boolean
open class BaseTestListener: TestListener {
override fun startTesting(runner: TestRunner) {}
override fun endTesting(runner: TestRunner, timeMillis: Long) {}
override fun startSuite(suite: TestSuite) {}
override fun endSuite(suite: TestSuite, timeMillis: Long) {}
override fun ignoreSuite(suite: TestSuite) {}
override fun start(testCase: TestCase) {}
override fun pass(testCase: TestCase, timeMillis: Long) {}
override fun fail(testCase: TestCase, e: Throwable, timeMillis: Long) {}
override fun ignore(testCase: TestCase) {}
}
abstract class AbstractTestStatistics: TestStatistics {
override var total: Int = 0; protected set
override var passed: Int = 0; protected set
override var failed: Int = 0; protected set
override var ignored: Int = 0; protected set
protected fun registerPass() { total++; passed++ }
protected fun registerFail() { total++; failed++ }
protected fun registerIgnore() { total++; ignored++ }
}
@@ -0,0 +1,60 @@
package konan.test
interface TestLogger: TestListener {
fun logTestList(runner: TestRunner)
fun log(message: String)
}
open class BaseTestLogger: BaseTestListener(), TestLogger {
protected val TestRunner.totalTests
get() = filterTests().filter { !it.ignored }.size
protected val TestRunner.totalSuites
get() = suites.filter { !it.ignored }.size
override fun log(message: String) = println(message)
override fun logTestList(runner: TestRunner) {
runner.filterTests().groupBy { it.suite }.forEach { (suite, tests) ->
println("${suite.name}.")
tests.forEach {
println(" ${it.name}")
}
}
}
}
open class TestLoggerWithStatistics: BaseTestLogger() {
protected val statistics = MutableTestStatistics()
override fun startTesting(runner: TestRunner) = statistics.reset()
override fun endSuite(suite: TestSuite, timeMillis: Long) = statistics.registerSuite()
override fun pass(testCase: TestCase, timeMillis: Long) = statistics.registerPass()
override fun fail(testCase: TestCase, e: Throwable, timeMillis: Long) = statistics.registerFail(testCase)
override fun ignore(testCase: TestCase) = statistics.registerIgnore()
}
class SilentTestLogger: BaseTestLogger() {
override fun logTestList(runner: TestRunner) {}
override fun log(message: String) {}
}
class SimpleTestLogger: BaseTestLogger() {
override fun startTesting(runner: TestRunner) = println("Starting testing")
override fun endTesting(runner: TestRunner, timeMillis: Long) = println("Testing finished")
override fun startSuite(suite: TestSuite) = println("Starting test suite: $suite")
override fun endSuite(suite: TestSuite, timeMillis: Long) = println("Test suite finished: $suite")
override fun ignoreSuite(suite: TestSuite) = println("Test suite ignored: $suite")
override fun start(testCase: TestCase) = println("Starting test case: $testCase")
override fun pass(testCase: TestCase, timeMillis: Long) = println("Passed: $testCase")
override fun fail(testCase: TestCase, e: Throwable, timeMillis: Long) {
println("Failed: $testCase. Exception:")
e.printStackTrace()
}
override fun ignore(testCase: TestCase) = println("Ignore: $testCase")
}
+210 -16
View File
@@ -1,29 +1,223 @@
package konan.test
import kotlin.IllegalArgumentException
import kotlin.system.getTimeMillis
import kotlin.system.measureTimeMillis
import kotlin.text.StringBuilder
object TestRunner {
private val _suites = mutableListOf<TestSuite>()
val suites: Collection<TestSuite> get() = _suites
private val suites_ = mutableListOf<TestSuite>()
val suites: Collection<TestSuite>
get() = suites_
fun register(suite: TestSuite) = _suites.add(suite)
fun register(suites: Iterable<TestSuite>) = _suites.addAll(suites)
fun register(vararg suites: TestSuite) = _suites.addAll(suites)
var logger: TestLogger = GTestLogger()
fun run(listener: TestListener) = with(listener) {
startTesting(this@TestRunner)
val totalTime = measureTimeMillis {
suites.forEach {
if (it.ignored) {
ignoreSuite(it)
} else {
startSuite(it)
val time = measureTimeMillis { it.run(listener) }
endSuite(it, time)
val listeners = mutableSetOf<TestListener>()
val filters = mutableListOf<(TestCase) -> Boolean>()
var exitCode = 0
private set
private val TestCase.matchFilters: Boolean
get() = filters.map { it(this) }.all { it }
private val TestSuite.testCasesFiltered
get() = testCases.values.filter { it.matchFilters }
// TODO: We can cache it.
fun filterTests(): Collection<TestCase> = suites.flatMap { it.testCasesFiltered }
fun register(suite: TestSuite) = suites_.add(suite)
fun register(suites: Iterable<TestSuite>) = suites_.addAll(suites)
fun register(vararg suites: TestSuite) = suites_.addAll(suites)
// TODO: Support short aliases.
// TODO: Support several test iterations.
/**
* Initialize the TestRunner using the command line options passed in [args].
* Returns: true if tests may be ran, false otherwise (there are unrecognized options or just help).
* The following options are available:
*
* --gtest_list_tests
* --ktest_list_tests - Show all available tests.
*
* --gtest_filter=POSTIVE_PATTERNS[-NEGATIVE_PATTERNS] - Run only the tests whose name matches one of the
* --ktest_filter=POSTIVE_PATTERNS[-NEGATIVE_PATTERNS] positive patterns but none of the negative patterns.
* '?' matches any single character; '*' matches any
* substring; ':' separates two patterns.
*
* --ktest_regex_filter=PATTERN - Run only the tests whose name matches the pattern.
* The pattern is a Kotlin regular expression.
*
* --ktest_negative_regex_filter=PATTERN - Run only the tests whose name doesn't match the pattern.
* The pattern is a Kotlin regular expression.
*
* --ktest_logger=GTEST|TEAMCITY|SIMPLE|SILENT - Use the specified output format. The default one is GTEST.
*/
fun useArgs(args: Array<String>): Boolean {
try {
return parseArgs(args)
} catch (e: IllegalArgumentException) {
logger.log("Error: ${e.message}")
return false
}
}
private fun parseArgs(args: Array<String>): Boolean {
var result = true
args.filter {
it.startsWith("--gtest_") || it.startsWith("--ktest_") || it == "--help"
}.forEach {
val arg = it.split('=')
when (arg.size) {
1 -> when (arg[0]) {
"--gtest_list_tests",
"--ktest_list_tests" -> { logger.logTestList(this); result = false }
"--help" -> { logger.log(help); result = false }
else -> throw IllegalArgumentException("Unknown option: $it\n$help")
}
2 -> {
val key = arg[0]
val value = arg[1]
when (key) {
"--ktest_logger" -> setLoggerFromArg(value)
"--gtest_filter",
"--ktest_filter" -> setGTestFilterFromArg(value)
"--ktest_regex_filter"-> setRegexFilterFromArg(value, true)
"--ktest_negative_regex_filter" -> setRegexFilterFromArg(value, false)
else -> throw IllegalArgumentException("Unknown option: $it\n$help")
}
}
else -> throw IllegalArgumentException("Unknown option: $it\n$help")
}
}
return result
}
private fun String.substringEscaped(range: IntRange) =
this.substring(range).let { if(it.isNotEmpty()) Regex.escape(it) else "" }
private fun String.toGTestPatterns() = splitToSequence(':').map { pattern ->
val result = StringBuilder()
var prevIndex = 0
pattern.forEachIndexed { index, c ->
if (c == '*' || c == '?') {
result.append(pattern.substringEscaped(prevIndex until index))
prevIndex = index+1
result.append( if (c == '*') ".*" else ".")
}
}
result.append(pattern.substringEscaped(prevIndex until pattern.length))
return@map result.toString().toRegex()
}.toList()
private fun setGTestFilterFromArg(filter: String) {
if (filter.isEmpty()) {
throw IllegalArgumentException("Empty filter")
}
val filters = filter.split('-')
if (filters.size > 2) {
throw IllegalArgumentException("Wrong pattern syntax: $filter.")
}
val positivePatterns = filters[0].toGTestPatterns()
val negativePatterns = filters.getOrNull(1)?.toGTestPatterns() ?: emptyList()
this.filters.add { testCase ->
positivePatterns.all { testCase.prettyName.matches(it) } &&
negativePatterns.none { testCase.prettyName.matches(it) }
}
}
private fun setRegexFilterFromArg(filter: String, positive: Boolean = true) {
if (filter.isEmpty()) {
throw IllegalArgumentException("Empty filter")
}
val pattern = filter.toRegex()
filters.add { testCase ->
testCase.prettyName.matches(pattern) == positive
}
}
private fun setLoggerFromArg(logger: String) {
when(logger.toUpperCase()) {
"GTEST" -> this.logger = GTestLogger()
"TEAMCITY" -> TODO() //this.logger = TeamCityLogger()
"SIMPLE" -> this.logger = SimpleTestLogger()
"SILENT" -> this.logger = SilentTestLogger()
else -> throw IllegalArgumentException("Unknown logger type. Available types: GTEST, TEAMCITY, SIMPLE")
}
}
private val help: String
get() = """
|Available options:
|--gtest_list_tests
|--ktest_list_tests - Show all available tests.
|
|--gtest_filter=POSTIVE_PATTERNS[-NEGATIVE_PATTERNS]
|--ktest_filter=POSTIVE_PATTERNS[-NEGATIVE_PATTERNS] - Run only the tests whose name matches one of the
| positive patterns but none of the negative patterns.
| '?' matches any single character; '*' matches any
| substring; ':' separates two patterns.
|
|--ktest_regex_filter=PATTERN - Run only the tests whose name matches the pattern.
| The pattern is a Kotlin regular expression.
|
|--ktest_negative_regex_filter=PATTERN - Run only the tests whose name doesn't match the pattern.
| The pattern is a Kotlin regular expression.
|
|--ktest_logger=GTEST|TEAMCITY|SIMPLE|SILENT - Use the specified output format. The default one is GTEST.
""".trimMargin()
private inline fun sendToListeners(event: TestListener.() -> Unit) {
logger.event()
listeners.forEach(event)
}
private fun TestSuite.run() {
doBeforeClass()
testCasesFiltered.forEach { testCase ->
if (testCase.ignored) {
sendToListeners { ignore(testCase) }
} else {
val startTime = getTimeMillis()
try {
sendToListeners { start(testCase) }
testCase.run()
sendToListeners { pass(testCase, getTimeMillis() - startTime) }
} catch (e: Throwable) {
sendToListeners { fail(testCase, e, getTimeMillis() - startTime) }
exitCode = 1
}
}
}
endTesting(this@TestRunner, totalTime)
doAfterClass()
}
fun run(args: Array<String>): Int {
return if (useArgs(args)) {
run()
} else {
0
}
}
fun run(): Int {
sendToListeners { startTesting(this@TestRunner) }
val totalTime = measureTimeMillis {
suites.forEach {
if (it.ignored) {
sendToListeners { ignoreSuite(it) }
} else {
sendToListeners { startSuite(it) }
val time = measureTimeMillis { it.run() }
sendToListeners { endSuite(it, time) }
}
}
}
sendToListeners { endTesting(this@TestRunner, totalTime) }
return exitCode
}
}
@@ -0,0 +1,57 @@
package konan.test
interface TestStatistics {
val total: Int
val passed: Int
val failed: Int
val ignored: Int
val totalSuites: Int
val failedTests: Collection<TestCase>
val hasFailedTests: Boolean
object EMPTY: TestStatistics {
override val total: Int get() = 0
override val passed: Int get() = 0
override val failed: Int get() = 0
override val ignored: Int get() = 0
override val totalSuites: Int get() = 0
override val failedTests: Collection<TestCase> = emptyList()
override val hasFailedTests: Boolean = false
}
}
class MutableTestStatistics: TestStatistics {
override var total: Int = 0; private set
override var passed: Int = 0; private set
override var ignored: Int = 0; private set
override var totalSuites: Int = 0; private set
override val failed: Int
get() = failedTests_.size
override val hasFailedTests: Boolean
get() = failedTests_.isNotEmpty()
private val failedTests_ = mutableListOf<TestCase>()
override val failedTests: Collection<TestCase>
get() = failedTests_
fun registerSuite() { totalSuites++ }
fun registerPass() { total++; passed++ }
fun registerFail(testCase: TestCase) { total++; failedTests_.add(testCase) }
fun registerIgnore() { total++; ignored++ }
fun reset() {
total = 0
passed = 0
ignored = 0
totalSuites = 0
failedTests_.clear()
}
}
+44 -45
View File
@@ -8,14 +8,20 @@ interface TestCase {
val name: String
val ignored: Boolean
val suite: TestSuite
fun run()
}
internal val TestCase.prettyName get() = "${suite.name}.$name"
interface TestSuite {
val name: String
val ignored: Boolean
val testCases: Map<String, TestCase>
val size : Int
fun run(listener: TestListener)
fun doBeforeClass()
fun doAfterClass()
}
enum class TestFunctionKind {
@@ -30,7 +36,7 @@ abstract class AbstractTestSuite<F: Function<Unit>>(override val name: String, o
override fun toString(): String = name
// TODO: Make inner and remove the type param when the bug is fixed.
class BasicTestCase<F: Function<Unit>>(
abstract class BasicTestCase<F: Function<Unit>>(
override val name: String,
override val suite: AbstractTestSuite<F>,
val testFunction: F,
@@ -48,13 +54,7 @@ abstract class AbstractTestSuite<F: Function<Unit>>(override val name: String, o
fun registerTestCase(name: String, testFunction: F, ignored: Boolean) =
registerTestCase(createTestCase(name, testFunction, ignored))
fun createTestCase(name: String, testFunction: F, ignored: Boolean) =
BasicTestCase(name, this, testFunction, ignored)
protected abstract fun doBeforeClass()
protected abstract fun doAfterClass()
protected abstract fun doTest(testCase: BasicTestCase<F>)
abstract fun createTestCase(name: String, testFunction: F, ignored: Boolean): BasicTestCase<F>
init {
TestRunner.register(this)
@@ -62,30 +62,28 @@ abstract class AbstractTestSuite<F: Function<Unit>>(override val name: String, o
override val size: Int
get() = testCases.size
override fun run(listener: TestListener) {
doBeforeClass()
testCases.values.forEach {
if (it.ignored) {
listener.ignore(it)
} else {
val startTime = getTimeMillis()
try {
listener.start(it)
doTest(it)
listener.pass(it, getTimeMillis() - startTime)
} catch (e: Throwable) {
listener.fail(it, e, getTimeMillis() - startTime)
}
}
}
doAfterClass()
}
}
abstract class BaseClassSuite<INSTANCE, COMPANION>(name: String, ignored: Boolean)
: AbstractTestSuite<INSTANCE.() -> Unit>(name, ignored) {
class TestCase<INSTANCE, COMPANION>(name: String,
override val suite: BaseClassSuite<INSTANCE, COMPANION>,
testFunction: INSTANCE.() -> Unit,
ignored: Boolean)
: BasicTestCase<INSTANCE.() -> Unit>(name, suite, testFunction, ignored) {
override fun run() {
val instance = suite.createInstance()
try {
suite.before.forEach { instance.it() }
instance.testFunction()
} finally {
suite.after.forEach { instance.it() }
}
}
}
// These two methods are overrided in test suite classes generated by the compiler.
abstract fun createInstance(): INSTANCE
open fun getCompanion(): COMPANION = throw NotImplementedError("Test class has no companion object")
@@ -124,22 +122,28 @@ abstract class BaseClassSuite<INSTANCE, COMPANION>(name: String, ignored: Boolea
override fun doBeforeClass() = beforeClass.forEach { getCompanion().it() }
override fun doAfterClass() = afterClass.forEach { getCompanion().it() }
override fun doTest(testCase: BasicTestCase<INSTANCE.() -> Unit>) {
val instance = createInstance()
val testFunction = testCase.testFunction
try {
before.forEach { instance.it() }
instance.testFunction()
} finally {
after.forEach { instance.it() }
}
}
override fun createTestCase(name: String, testFunction: INSTANCE.() -> Unit, ignored: Boolean)
: BasicTestCase<INSTANCE.() -> Unit> =
TestCase<INSTANCE, COMPANION>(name, this, testFunction, ignored)
}
private typealias TopLevelFun = () -> Unit
class TopLevelSuite(name: String): AbstractTestSuite<TopLevelFun>(name, false) {
class TestCase(name: String, override val suite: TopLevelSuite, testFunction: TopLevelFun, ignored: Boolean)
: BasicTestCase<TopLevelFun>(name, suite, testFunction, ignored) {
override fun run() {
try {
suite.before.forEach { it() }
testFunction()
} finally {
suite.after.forEach { it() }
}
}
}
private val specialFunctions = mutableMapOf<TestFunctionKind, MutableSet<TopLevelFun>>()
private fun getFunctions(type: TestFunctionKind) = specialFunctions.getOrPut(type) { mutableSetOf() }
@@ -153,11 +157,6 @@ class TopLevelSuite(name: String): AbstractTestSuite<TopLevelFun>(name, false) {
override fun doBeforeClass() = beforeClass.forEach { it() }
override fun doAfterClass() = afterClass.forEach { it() }
override fun doTest(testCase: BasicTestCase<() -> Unit>) =
try {
before.forEach { it() }
testCase.testFunction()
} finally {
after.forEach { it() }
}
override fun createTestCase(name: String, testFunction: TopLevelFun, ignored: Boolean)
: BasicTestCase<TopLevelFun> = TestCase(name, this, testFunction, ignored)
}