diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/AbstractNativeBlackBoxTest.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/AbstractNativeBlackBoxTest.kt index 98c6157a237..1570e98f5b1 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/AbstractNativeBlackBoxTest.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/AbstractNativeBlackBoxTest.kt @@ -13,7 +13,10 @@ import org.junit.jupiter.api.extension.ExtendWith abstract class AbstractNativeBlackBoxTest { internal lateinit var testRunProvider: TestRunProvider - fun runTest(@TestDataFile testDataFilePath: String) { - testRunProvider.getSingleTestRun(getAbsoluteFile(testDataFilePath)).runAndVerify() + fun runTest(@TestDataFile testDataFilePath: String): Unit = with(testRunProvider) { + val testDataFile = getAbsoluteFile(testDataFilePath) + val testRun = getSingleTestRun(testDataFile) + val testRunner = createRunner(testRun) + testRunner.run() } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestEnvironment.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestEnvironment.kt index a4155c702d4..c19cc07369a 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestEnvironment.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestEnvironment.kt @@ -29,6 +29,8 @@ internal class GlobalTestEnvironment( val cacheSettings: TestCacheSettings = defaultCacheSettings, val baseBuildDir: File = projectBuildDir ) { + val hostTarget: KonanTarget = HostManager.host + fun getRootCacheDirectory(debuggable: Boolean): File? = (cacheSettings as? TestCacheSettings.WithCache)?.getRootCacheDirectory(this, debuggable) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestExecutable.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestExecutable.kt index 7d9ebf55287..94ae2137098 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestExecutable.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestExecutable.kt @@ -22,12 +22,27 @@ internal class TestRun( internal sealed interface TestRunParameter { fun applyTo(programArgs: MutableList) - class WithPackageName(val packageName: String) : TestRunParameter { - override fun applyTo(programArgs: MutableList) { - programArgs += "--ktest_filter=$packageName.*" + sealed class WithFilter : TestRunParameter { + protected abstract val wildcard: String + abstract fun testMatches(testName: String): Boolean + + final override fun applyTo(programArgs: MutableList) { + programArgs += "--ktest_filter=$wildcard" } } + class WithPackageFilter(private val packageFQN: PackageName) : WithFilter() { + override val wildcard get() = "$packageFQN.*" + override fun testMatches(testName: String) = testName.startsWith("$packageFQN.") + } + +/* + class WithFunctionFilter(private val functionFQN: String) : WithFilter() { + override val wildcard get() = functionFQN + override fun testMatches(testName: String) = testName == functionFQN + } +*/ + object WithGTestLogger : TestRunParameter { override fun applyTo(programArgs: MutableList) { programArgs += "--ktest_logger=GTEST" diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestRunProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestRunProvider.kt index a59fde3acb7..6bf2f89fdaf 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestRunProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestRunProvider.kt @@ -8,6 +8,8 @@ package org.jetbrains.kotlin.konan.blackboxtest import com.intellij.openapi.util.Disposer import org.jetbrains.kotlin.konan.blackboxtest.TestCompilationResult.Companion.assertSuccess import org.jetbrains.kotlin.konan.blackboxtest.group.TestCaseGroupProvider +import org.jetbrains.kotlin.konan.blackboxtest.runner.AbstractRunner +import org.jetbrains.kotlin.konan.blackboxtest.runner.LocalTestRunner import org.jetbrains.kotlin.konan.blackboxtest.util.ThreadSafeCache import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail @@ -64,7 +66,7 @@ internal class TestRunProvider( ) TestKind.REGULAR -> listOfNotNull( TestRunParameter.WithGTestLogger, - TestRunParameter.WithPackageName(packageName = testCase.nominalPackageName), + TestRunParameter.WithPackageFilter(testCase.nominalPackageName), testCase.expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData) ) } @@ -72,6 +74,18 @@ internal class TestRunProvider( return TestRun(executable, runParameters, testCase.origin) } + // Currently, only local test runner is supported. + fun createRunner(testRun: TestRun): AbstractRunner<*> = when (val target = environment.globalEnvironment.target) { + environment.globalEnvironment.hostTarget -> LocalTestRunner(testRun) + else -> fail { + """ + Running at non-host target is not supported yet. + Compilation target: $target + Host target: ${environment.globalEnvironment.hostTarget} + """.trimIndent() + } + } + override fun close() { Disposer.dispose(environment) } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestRunner.kt deleted file mode 100644 index c820e191454..00000000000 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestRunner.kt +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.konan.blackboxtest - -import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertEquals -import kotlin.properties.Delegates - -internal fun TestRun.runAndVerify() { - val programArgs = mutableListOf(executable.executableFile.path) - runParameters.forEach { it.applyTo(programArgs) } - - val loggedParameters = LoggedData.TestRunParameters(executable.loggedCompilerCall, origin, programArgs, runParameters) - - val startTimeMillis = System.currentTimeMillis() - val process = ProcessBuilder(programArgs).directory(executable.executableFile.parentFile).start() - runParameters.get { - process.outputStream.write(inputDataFile.readBytes()) - process.outputStream.flush() - } - - TestOutput(runParameters, process, startTimeMillis, loggedParameters).verify() -} - -private class TestOutput( - private val runParameters: List, - private val process: Process, - private val startTimeMillis: Long, - private val loggedParameters: LoggedData.TestRunParameters -) { - private var exitCode: Int by Delegates.notNull() - private lateinit var stdOut: String - private lateinit var stdErr: String - private var finishTimeMillis by Delegates.notNull() - - fun verify() { - waitUntilExecutionFinished() - - verifyExpectation(0, exitCode) { "Tested process exited with non-zero code." } - - if (runParameters.has()) { - verifyTestWithGTestRunner() - } else { - verifyPlainTest() - } - } - - private fun verifyTestWithGTestRunner() { - val testStatuses = hashMapOf>() - val cleanStdOut = StringBuilder() - - var expectStatusLine = false - stdOut.lines().forEach { line -> - when { - expectStatusLine -> { - val matcher = GTEST_STATUS_LINE_REGEX.matchEntire(line) - if (matcher != null) { - // Read the line with test status. - val testStatus = matcher.groupValues[1] - val testName = matcher.groupValues[2] - testStatuses.getOrPut(testStatus) { hashSetOf() } += testName - expectStatusLine = false - } else { - // If current line is not a status line then it could be only the line with the process' output. - cleanStdOut.appendLine(line) - } - } - line.startsWith(GTEST_RUN_LINE_PREFIX) -> { - expectStatusLine = true // Next line contains either test status. - } - else -> Unit - } - } - - verifyExpectation(true, testStatuses.isNotEmpty()) { "No tests have been executed." } - - val passedTests = testStatuses[GTEST_STATUS_OK]?.size ?: 0 - verifyExpectation(true, passedTests > 0) { "No passed tests." } - - runParameters.get { - val excessiveTests = testStatuses.getValue(GTEST_STATUS_OK).filter { testName -> !testName.startsWith(packageName) } - verifyExpectation(true, excessiveTests.isEmpty()) { "Excessive tests have been executed: $excessiveTests." } - } - - val failedTests = (testStatuses - GTEST_STATUS_OK).values.sumOf { it.size } - verifyExpectation(0, failedTests) { "There are failed tests." } - - runParameters.get { - val mergedOutput = cleanStdOut.toString() + stdErr - verifyExpectation(expectedOutputDataFile.readText(), mergedOutput) { "Tested process output mismatch." } - } - } - - private fun verifyPlainTest() { - runParameters.get { - val mergedOutput = stdOut + stdErr - verifyExpectation(expectedOutputDataFile.readText(), mergedOutput) { "Tested process output mismatch." } - } - } - - private fun waitUntilExecutionFinished() { - exitCode = process.waitFor() - finishTimeMillis = System.currentTimeMillis() - stdOut = process.inputStream.bufferedReader().readText() - stdErr = process.errorStream.bufferedReader().readText() - } - - private inline fun verifyExpectation(expected: T, actual: T, crossinline errorMessageHeader: () -> String) { - assertEquals(expected, actual) { - buildString { - appendLine(errorMessageHeader()) - appendLine() - appendLine(LoggedData.TestRun(loggedParameters, exitCode, stdOut, stdErr, finishTimeMillis - startTimeMillis)) - } - } - } - - companion object { - private const val GTEST_RUN_LINE_PREFIX = "[ RUN ]" - private val GTEST_STATUS_LINE_REGEX = Regex("^\\[\\s+([A-Z]+)\\s+]\\s+(\\S+)\\s+.*") - private const val GTEST_STATUS_OK = "OK" - } -} - -private typealias TestStatus = String -private typealias TestName = String diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/runner/AbstractLocalProcessRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/runner/AbstractLocalProcessRunner.kt new file mode 100644 index 00000000000..473bf0598d7 --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/runner/AbstractLocalProcessRunner.kt @@ -0,0 +1,45 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.blackboxtest.runner + +import org.jetbrains.kotlin.konan.blackboxtest.TestExecutable + +internal abstract class AbstractLocalProcessRunner : AbstractRunner() { + protected abstract val visibleProcessName: String + protected abstract val executable: TestExecutable + protected abstract val programArgs: List + + protected open fun customizeProcess(process: Process) = Unit + + final override fun buildRun() = object : AbstractRun { + override fun run(): RunResult { + val startTimeMillis = System.currentTimeMillis() + + val process = ProcessBuilder(programArgs).directory(executable.executableFile.parentFile).start() + customizeProcess(process) + + val exitCode = process.waitFor() + val finishTimeMillis = System.currentTimeMillis() + + val stdOut = process.inputStream.bufferedReader().readText() + val stdErr = process.errorStream.bufferedReader().readText() + + return RunResult(exitCode, finishTimeMillis - startTimeMillis, stdOut, stdErr) + } + } + + abstract override fun buildResultHandler(runResult: RunResult): ResultHandler // Narrow returned type. + + abstract inner class ResultHandler(runResult: RunResult) : AbstractRunner.ResultHandler(runResult) { + override fun handle(): R { + verifyExpectation(0, runResult.exitCode) { "$visibleProcessName exited with non-zero code." } + + return doHandle() + } + + protected abstract fun doHandle(): R + } +} diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/runner/AbstractRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/runner/AbstractRunner.kt new file mode 100644 index 00000000000..05b5ededc34 --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/runner/AbstractRunner.kt @@ -0,0 +1,53 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.blackboxtest.runner + +import org.jetbrains.kotlin.konan.blackboxtest.LoggedData +import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertEquals +import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue + +internal abstract class AbstractRunner { + protected abstract fun buildRun(): AbstractRun + protected abstract fun buildResultHandler(runResult: RunResult): ResultHandler + + fun run(): R { + val run = buildRun() + val runResult = run.run() + + val resultHandler = buildResultHandler(runResult) + return resultHandler.handle() + } + + interface AbstractRun { + fun run(): RunResult + } + + data class RunResult(val exitCode: Int, val durationMillis: Long, val stdOut: String, val stdErr: String) + + abstract inner class ResultHandler(protected val runResult: RunResult) { + abstract fun getLoggedRun(): LoggedData + abstract fun handle(): R + + val exitCode get() = runResult.exitCode + val durationMillis get() = runResult.durationMillis + val stdOut get() = runResult.stdOut + val stdErr get() = runResult.stdErr + + protected inline fun verifyExpectation(expected: T, actual: T, crossinline errorMessageHeader: () -> String) { + assertEquals(expected, actual) { formatErrorMessage(errorMessageHeader) } + } + + protected inline fun verifyExpectation(shouldBeTrue: Boolean, crossinline errorMessageHeader: () -> String) { + assertTrue(shouldBeTrue) { formatErrorMessage(errorMessageHeader) } + } + + private inline fun formatErrorMessage(errorMessageHeader: () -> String) = buildString { + appendLine(errorMessageHeader()) + appendLine() + appendLine(getLoggedRun()) + } + } +} diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/runner/LocalTestRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/runner/LocalTestRunner.kt new file mode 100644 index 00000000000..f7b70496f35 --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/runner/LocalTestRunner.kt @@ -0,0 +1,112 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.blackboxtest.runner + +import org.jetbrains.kotlin.konan.blackboxtest.* +import org.jetbrains.kotlin.konan.blackboxtest.LoggedData +import org.jetbrains.kotlin.konan.blackboxtest.TestRun +import org.jetbrains.kotlin.konan.blackboxtest.TestRunParameter +import org.jetbrains.kotlin.konan.blackboxtest.get + +internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProcessRunner() { + override val visibleProcessName get() = "Tested process" + override val executable get() = testRun.executable + + override val programArgs = buildList { + add(executable.executableFile.path) + testRun.runParameters.forEach { it.applyTo(this) } + } + + private fun getLoggedParameters() = LoggedData.TestRunParameters( + compilerCall = executable.loggedCompilerCall, + origin = testRun.origin, + runArgs = programArgs, + runParameters = testRun.runParameters + ) + + override fun customizeProcess(process: Process) { + testRun.runParameters.get { + process.outputStream.write(inputDataFile.readBytes()) + process.outputStream.flush() + } + } + + override fun buildResultHandler(runResult: RunResult) = ResultHandler(runResult) + + inner class ResultHandler(runResult: RunResult) : AbstractLocalProcessRunner.ResultHandler(runResult) { + override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), exitCode, stdOut, stdErr, durationMillis) + + override fun doHandle() { + if (testRun.runParameters.has()) { + verifyTestWithGTestRunner() + } else { + verifyPlainTest() + } + } + + private fun verifyTestWithGTestRunner() { + val testStatuses = hashMapOf>() + val cleanStdOut = StringBuilder() + + var expectStatusLine = false + stdOut.lines().forEach { line -> + when { + expectStatusLine -> { + val matcher = GTEST_STATUS_LINE_REGEX.matchEntire(line) + if (matcher != null) { + // Read the line with test status. + val testStatus = matcher.groupValues[1] + val testName = matcher.groupValues[2] + testStatuses.getOrPut(testStatus) { hashSetOf() } += testName + expectStatusLine = false + } else { + // If current line is not a status line then it could be only the line with the process' output. + cleanStdOut.appendLine(line) + } + } + line.startsWith(GTEST_RUN_LINE_PREFIX) -> { + expectStatusLine = true // Next line contains either test status. + } + else -> Unit + } + } + + verifyExpectation(true, testStatuses.isNotEmpty()) { "No tests have been executed." } + + val passedTests = testStatuses[GTEST_STATUS_OK]?.size ?: 0 + verifyExpectation(true, passedTests > 0) { "No passed tests." } + + testRun.runParameters.get { + val excessiveTests = testStatuses.getValue(GTEST_STATUS_OK).filter { testName -> !testMatches(testName) } + verifyExpectation(true, excessiveTests.isEmpty()) { "Excessive tests have been executed: $excessiveTests." } + } + + val failedTests = (testStatuses - GTEST_STATUS_OK).values.sumOf { it.size } + verifyExpectation(0, failedTests) { "There are failed tests." } + + testRun.runParameters.get { + val mergedOutput = cleanStdOut.toString() + stdErr + verifyExpectation(expectedOutputDataFile.readText(), mergedOutput) { "Tested process output mismatch." } + } + } + + private fun verifyPlainTest() { + testRun.runParameters.get { + val mergedOutput = stdOut + stdErr + verifyExpectation(expectedOutputDataFile.readText(), mergedOutput) { "Tested process output mismatch." } + } + } + } + + companion object { + private const val GTEST_RUN_LINE_PREFIX = "[ RUN ]" + private val GTEST_STATUS_LINE_REGEX = Regex("^\\[\\s+([A-Z]+)\\s+]\\s+(\\S+)\\s+.*") + private const val GTEST_STATUS_OK = "OK" + } +} + +private typealias TestStatus = String +private typealias TestName = String