[Native][tests] Introduce TestRunner that knows how exactly to run an arbitrary test

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/AbstractNativeBlackBoxTest.kt

@@ -13,7 +13,10 @@ import org.junit.jupiter.api.extension.ExtendWith
 abstract class AbstractNativeBlackBoxTest {
     internal lateinit var testRunProvider: TestRunProvider
 
-    fun runTest(@TestDataFile testDataFilePath: String) {
+    fun runTest(@TestDataFile testDataFilePath: String): Unit = with(testRunProvider) {
-        testRunProvider.getSingleTestRun(getAbsoluteFile(testDataFilePath)).runAndVerify()
+        val testDataFile = getAbsoluteFile(testDataFilePath)
+        val testRun = getSingleTestRun(testDataFile)
+        val testRunner = createRunner(testRun)
+        testRunner.run()
     }
 }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestEnvironment.kt

@@ -29,6 +29,8 @@ internal class GlobalTestEnvironment(
     val cacheSettings: TestCacheSettings = defaultCacheSettings,
     val baseBuildDir: File = projectBuildDir
 ) {
+    val hostTarget: KonanTarget = HostManager.host
+
     fun getRootCacheDirectory(debuggable: Boolean): File? =
         (cacheSettings as? TestCacheSettings.WithCache)?.getRootCacheDirectory(this, debuggable)
 

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestExecutable.kt

@@ -22,12 +22,27 @@ internal class TestRun(
 internal sealed interface TestRunParameter {
     fun applyTo(programArgs: MutableList<String>)
 
-    class WithPackageName(val packageName: String) : TestRunParameter {
+    sealed class WithFilter : TestRunParameter {
-        override fun applyTo(programArgs: MutableList<String>) {
+        protected abstract val wildcard: String
-            programArgs += "--ktest_filter=$packageName.*"
+        abstract fun testMatches(testName: String): Boolean
+
+        final override fun applyTo(programArgs: MutableList<String>) {
+            programArgs += "--ktest_filter=$wildcard"
         }
     }
 
+    class WithPackageFilter(private val packageFQN: PackageName) : WithFilter() {
+        override val wildcard get() = "$packageFQN.*"
+        override fun testMatches(testName: String) = testName.startsWith("$packageFQN.")
+    }
+
+/*
+    class WithFunctionFilter(private val functionFQN: String) : WithFilter() {
+        override val wildcard get() = functionFQN
+        override fun testMatches(testName: String) = testName == functionFQN
+    }
+*/
+
     object WithGTestLogger : TestRunParameter {
         override fun applyTo(programArgs: MutableList<String>) {
             programArgs += "--ktest_logger=GTEST"

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestRunProvider.kt

@@ -8,6 +8,8 @@ package org.jetbrains.kotlin.konan.blackboxtest
 import com.intellij.openapi.util.Disposer
 import org.jetbrains.kotlin.konan.blackboxtest.TestCompilationResult.Companion.assertSuccess
 import org.jetbrains.kotlin.konan.blackboxtest.group.TestCaseGroupProvider
+import org.jetbrains.kotlin.konan.blackboxtest.runner.AbstractRunner
+import org.jetbrains.kotlin.konan.blackboxtest.runner.LocalTestRunner
 import org.jetbrains.kotlin.konan.blackboxtest.util.ThreadSafeCache
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
@@ -64,7 +66,7 @@ internal class TestRunProvider(
             )
             TestKind.REGULAR -> listOfNotNull(
                 TestRunParameter.WithGTestLogger,
-                TestRunParameter.WithPackageName(packageName = testCase.nominalPackageName),
+                TestRunParameter.WithPackageFilter(testCase.nominalPackageName),
                 testCase.expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)
             )
         }
@@ -72,6 +74,18 @@ internal class TestRunProvider(
         return TestRun(executable, runParameters, testCase.origin)
     }
 
+    // Currently, only local test runner is supported.
+    fun createRunner(testRun: TestRun): AbstractRunner<*> = when (val target = environment.globalEnvironment.target) {
+        environment.globalEnvironment.hostTarget -> LocalTestRunner(testRun)
+        else -> fail {
+            """
+                Running at non-host target is not supported yet.
+                Compilation target: $target
+                Host target: ${environment.globalEnvironment.hostTarget}
+            """.trimIndent()
+        }
+    }
+
     override fun close() {
         Disposer.dispose(environment)
     }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestRunner.kt

@@ -1,128 +0,0 @@
-/*
- * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.konan.blackboxtest
-
-import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertEquals
-import kotlin.properties.Delegates
-
-internal fun TestRun.runAndVerify() {
-    val programArgs = mutableListOf<String>(executable.executableFile.path)
-    runParameters.forEach { it.applyTo(programArgs) }
-
-    val loggedParameters = LoggedData.TestRunParameters(executable.loggedCompilerCall, origin, programArgs, runParameters)
-
-    val startTimeMillis = System.currentTimeMillis()
-    val process = ProcessBuilder(programArgs).directory(executable.executableFile.parentFile).start()
-    runParameters.get<TestRunParameter.WithInputData> {
-        process.outputStream.write(inputDataFile.readBytes())
-        process.outputStream.flush()
-    }
-
-    TestOutput(runParameters, process, startTimeMillis, loggedParameters).verify()
-}
-
-private class TestOutput(
-    private val runParameters: List<TestRunParameter>,
-    private val process: Process,
-    private val startTimeMillis: Long,
-    private val loggedParameters: LoggedData.TestRunParameters
-) {
-    private var exitCode: Int by Delegates.notNull()
-    private lateinit var stdOut: String
-    private lateinit var stdErr: String
-    private var finishTimeMillis by Delegates.notNull<Long>()
-
-    fun verify() {
-        waitUntilExecutionFinished()
-
-        verifyExpectation(0, exitCode) { "Tested process exited with non-zero code." }
-
-        if (runParameters.has<TestRunParameter.WithGTestLogger>()) {
-            verifyTestWithGTestRunner()
-        } else {
-            verifyPlainTest()
-        }
-    }
-
-    private fun verifyTestWithGTestRunner() {
-        val testStatuses = hashMapOf<TestStatus, MutableSet<TestName>>()
-        val cleanStdOut = StringBuilder()
-
-        var expectStatusLine = false
-        stdOut.lines().forEach { line ->
-            when {
-                expectStatusLine -> {
-                    val matcher = GTEST_STATUS_LINE_REGEX.matchEntire(line)
-                    if (matcher != null) {
-                        // Read the line with test status.
-                        val testStatus = matcher.groupValues[1]
-                        val testName = matcher.groupValues[2]
-                        testStatuses.getOrPut(testStatus) { hashSetOf() } += testName
-                        expectStatusLine = false
-                    } else {
-                        // If current line is not a status line then it could be only the line with the process' output.
-                        cleanStdOut.appendLine(line)
-                    }
-                }
-                line.startsWith(GTEST_RUN_LINE_PREFIX) -> {
-                    expectStatusLine = true // Next line contains either  test status.
-                }
-                else -> Unit
-            }
-        }
-
-        verifyExpectation(true, testStatuses.isNotEmpty()) { "No tests have been executed." }
-
-        val passedTests = testStatuses[GTEST_STATUS_OK]?.size ?: 0
-        verifyExpectation(true, passedTests > 0) { "No passed tests." }
-
-        runParameters.get<TestRunParameter.WithPackageName> {
-            val excessiveTests = testStatuses.getValue(GTEST_STATUS_OK).filter { testName -> !testName.startsWith(packageName) }
-            verifyExpectation(true, excessiveTests.isEmpty()) { "Excessive tests have been executed: $excessiveTests." }
-        }
-
-        val failedTests = (testStatuses - GTEST_STATUS_OK).values.sumOf { it.size }
-        verifyExpectation(0, failedTests) { "There are failed tests." }
-
-        runParameters.get<TestRunParameter.WithExpectedOutputData> {
-            val mergedOutput = cleanStdOut.toString() + stdErr
-            verifyExpectation(expectedOutputDataFile.readText(), mergedOutput) { "Tested process output mismatch." }
-        }
-    }
-
-    private fun verifyPlainTest() {
-        runParameters.get<TestRunParameter.WithExpectedOutputData> {
-            val mergedOutput = stdOut + stdErr
-            verifyExpectation(expectedOutputDataFile.readText(), mergedOutput) { "Tested process output mismatch." }
-        }
-    }
-
-    private fun waitUntilExecutionFinished() {
-        exitCode = process.waitFor()
-        finishTimeMillis = System.currentTimeMillis()
-        stdOut = process.inputStream.bufferedReader().readText()
-        stdErr = process.errorStream.bufferedReader().readText()
-    }
-
-    private inline fun <T> verifyExpectation(expected: T, actual: T, crossinline errorMessageHeader: () -> String) {
-        assertEquals(expected, actual) {
-            buildString {
-                appendLine(errorMessageHeader())
-                appendLine()
-                appendLine(LoggedData.TestRun(loggedParameters, exitCode, stdOut, stdErr, finishTimeMillis - startTimeMillis))
-            }
-        }
-    }
-
-    companion object {
-        private const val GTEST_RUN_LINE_PREFIX = "[ RUN      ]"
-        private val GTEST_STATUS_LINE_REGEX = Regex("^\\[\\s+([A-Z]+)\\s+]\\s+(\\S+)\\s+.*")
-        private const val GTEST_STATUS_OK = "OK"
-    }
-}
-
-private typealias TestStatus = String
-private typealias TestName = String

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/runner/AbstractLocalProcessRunner.kt

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.konan.blackboxtest.runner
+
+import org.jetbrains.kotlin.konan.blackboxtest.TestExecutable
+
+internal abstract class AbstractLocalProcessRunner<R> : AbstractRunner<R>() {
+    protected abstract val visibleProcessName: String
+    protected abstract val executable: TestExecutable
+    protected abstract val programArgs: List<String>
+
+    protected open fun customizeProcess(process: Process) = Unit
+
+    final override fun buildRun() = object : AbstractRun {
+        override fun run(): RunResult {
+            val startTimeMillis = System.currentTimeMillis()
+
+            val process = ProcessBuilder(programArgs).directory(executable.executableFile.parentFile).start()
+            customizeProcess(process)
+
+            val exitCode = process.waitFor()
+            val finishTimeMillis = System.currentTimeMillis()
+
+            val stdOut = process.inputStream.bufferedReader().readText()
+            val stdErr = process.errorStream.bufferedReader().readText()
+
+            return RunResult(exitCode, finishTimeMillis - startTimeMillis, stdOut, stdErr)
+        }
+    }
+
+    abstract override fun buildResultHandler(runResult: RunResult): ResultHandler // Narrow returned type.
+
+    abstract inner class ResultHandler(runResult: RunResult) : AbstractRunner<R>.ResultHandler(runResult) {
+        override fun handle(): R {
+            verifyExpectation(0, runResult.exitCode) { "$visibleProcessName exited with non-zero code." }
+
+            return doHandle()
+        }
+
+        protected abstract fun doHandle(): R
+    }
+}

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/runner/AbstractRunner.kt

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.konan.blackboxtest.runner
+
+import org.jetbrains.kotlin.konan.blackboxtest.LoggedData
+import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertEquals
+import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
+
+internal abstract class AbstractRunner<R> {
+    protected abstract fun buildRun(): AbstractRun
+    protected abstract fun buildResultHandler(runResult: RunResult): ResultHandler
+
+    fun run(): R {
+        val run = buildRun()
+        val runResult = run.run()
+
+        val resultHandler = buildResultHandler(runResult)
+        return resultHandler.handle()
+    }
+
+    interface AbstractRun {
+        fun run(): RunResult
+    }
+
+    data class RunResult(val exitCode: Int, val durationMillis: Long, val stdOut: String, val stdErr: String)
+
+    abstract inner class ResultHandler(protected val runResult: RunResult) {
+        abstract fun getLoggedRun(): LoggedData
+        abstract fun handle(): R
+
+        val exitCode get() = runResult.exitCode
+        val durationMillis get() = runResult.durationMillis
+        val stdOut get() = runResult.stdOut
+        val stdErr get() = runResult.stdErr
+
+        protected inline fun <T> verifyExpectation(expected: T, actual: T, crossinline errorMessageHeader: () -> String) {
+            assertEquals(expected, actual) { formatErrorMessage(errorMessageHeader) }
+        }
+
+        protected inline fun verifyExpectation(shouldBeTrue: Boolean, crossinline errorMessageHeader: () -> String) {
+            assertTrue(shouldBeTrue) { formatErrorMessage(errorMessageHeader) }
+        }
+
+        private inline fun formatErrorMessage(errorMessageHeader: () -> String) = buildString {
+            appendLine(errorMessageHeader())
+            appendLine()
+            appendLine(getLoggedRun())
+        }
+    }
+}

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/runner/LocalTestRunner.kt

@@ -0,0 +1,112 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.konan.blackboxtest.runner
+
+import org.jetbrains.kotlin.konan.blackboxtest.*
+import org.jetbrains.kotlin.konan.blackboxtest.LoggedData
+import org.jetbrains.kotlin.konan.blackboxtest.TestRun
+import org.jetbrains.kotlin.konan.blackboxtest.TestRunParameter
+import org.jetbrains.kotlin.konan.blackboxtest.get
+
+internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProcessRunner<Unit>() {
+    override val visibleProcessName get() = "Tested process"
+    override val executable get() = testRun.executable
+
+    override val programArgs = buildList<String> {
+        add(executable.executableFile.path)
+        testRun.runParameters.forEach { it.applyTo(this) }
+    }
+
+    private fun getLoggedParameters() = LoggedData.TestRunParameters(
+        compilerCall = executable.loggedCompilerCall,
+        origin = testRun.origin,
+        runArgs = programArgs,
+        runParameters = testRun.runParameters
+    )
+
+    override fun customizeProcess(process: Process) {
+        testRun.runParameters.get<TestRunParameter.WithInputData> {
+            process.outputStream.write(inputDataFile.readBytes())
+            process.outputStream.flush()
+        }
+    }
+
+    override fun buildResultHandler(runResult: RunResult) = ResultHandler(runResult)
+
+    inner class ResultHandler(runResult: RunResult) : AbstractLocalProcessRunner<Unit>.ResultHandler(runResult) {
+        override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), exitCode, stdOut, stdErr, durationMillis)
+
+        override fun doHandle() {
+            if (testRun.runParameters.has<TestRunParameter.WithGTestLogger>()) {
+                verifyTestWithGTestRunner()
+            } else {
+                verifyPlainTest()
+            }
+        }
+
+        private fun verifyTestWithGTestRunner() {
+            val testStatuses = hashMapOf<TestStatus, MutableSet<TestName>>()
+            val cleanStdOut = StringBuilder()
+
+            var expectStatusLine = false
+            stdOut.lines().forEach { line ->
+                when {
+                    expectStatusLine -> {
+                        val matcher = GTEST_STATUS_LINE_REGEX.matchEntire(line)
+                        if (matcher != null) {
+                            // Read the line with test status.
+                            val testStatus = matcher.groupValues[1]
+                            val testName = matcher.groupValues[2]
+                            testStatuses.getOrPut(testStatus) { hashSetOf() } += testName
+                            expectStatusLine = false
+                        } else {
+                            // If current line is not a status line then it could be only the line with the process' output.
+                            cleanStdOut.appendLine(line)
+                        }
+                    }
+                    line.startsWith(GTEST_RUN_LINE_PREFIX) -> {
+                        expectStatusLine = true // Next line contains either  test status.
+                    }
+                    else -> Unit
+                }
+            }
+
+            verifyExpectation(true, testStatuses.isNotEmpty()) { "No tests have been executed." }
+
+            val passedTests = testStatuses[GTEST_STATUS_OK]?.size ?: 0
+            verifyExpectation(true, passedTests > 0) { "No passed tests." }
+
+            testRun.runParameters.get<TestRunParameter.WithFilter> {
+                val excessiveTests = testStatuses.getValue(GTEST_STATUS_OK).filter { testName -> !testMatches(testName) }
+                verifyExpectation(true, excessiveTests.isEmpty()) { "Excessive tests have been executed: $excessiveTests." }
+            }
+
+            val failedTests = (testStatuses - GTEST_STATUS_OK).values.sumOf { it.size }
+            verifyExpectation(0, failedTests) { "There are failed tests." }
+
+            testRun.runParameters.get<TestRunParameter.WithExpectedOutputData> {
+                val mergedOutput = cleanStdOut.toString() + stdErr
+                verifyExpectation(expectedOutputDataFile.readText(), mergedOutput) { "Tested process output mismatch." }
+            }
+        }
+
+        private fun verifyPlainTest() {
+            testRun.runParameters.get<TestRunParameter.WithExpectedOutputData> {
+                val mergedOutput = stdOut + stdErr
+                verifyExpectation(expectedOutputDataFile.readText(), mergedOutput) { "Tested process output mismatch." }
+            }
+        }
+    }
+
+    companion object {
+        private const val GTEST_RUN_LINE_PREFIX = "[ RUN      ]"
+        private val GTEST_STATUS_LINE_REGEX = Regex("^\\[\\s+([A-Z]+)\\s+]\\s+(\\S+)\\s+.*")
+        private const val GTEST_STATUS_OK = "OK"
+    }
+}
+
+private typealias TestStatus = String
+private typealias TestName = String