diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/FrameworkTest.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/FrameworkTest.kt index 061da187949..60caf682e4a 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/FrameworkTest.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/FrameworkTest.kt @@ -14,18 +14,15 @@ import org.jetbrains.kotlin.konan.test.blackbox.support.group.FirPipeline import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestExecutable import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunChecks -import org.jetbrains.kotlin.konan.test.blackbox.support.runner.doFileCheck import org.jetbrains.kotlin.konan.test.blackbox.support.settings.* import org.jetbrains.kotlin.konan.test.blackbox.support.util.createTestProvider import org.jetbrains.kotlin.native.executors.runProcess import org.jetbrains.kotlin.test.KtAssert.fail import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue import org.junit.jupiter.api.Assumptions -import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test import java.io.File -import java.io.FileWriter import kotlin.time.Duration @TestDataPath("\$PROJECT_ROOT") @@ -83,7 +80,7 @@ abstract class FrameworkTestBase : AbstractNativeSimpleTest() { objCFrameworkCompilation.result.assertSuccess() val fileCheckDump = buildDir.resolve("out.$fileCheckStage.ll").also { assert(it.exists()) } - val result = doFileCheck(testCase.checks.fileCheckMatcher!!, fileCheckDump) + val result = testCase.checks.fileCheckMatcher!!.doFileCheck(fileCheckDump) if (!(result.stdout.isEmpty() && result.stderr.isEmpty())) { val shortOutText = result.stdout.lines().take(100) val shortErrText = result.stderr.lines().take(100) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt index 823c258e931..867d91de616 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt @@ -5,31 +5,10 @@ package org.jetbrains.kotlin.konan.test.blackbox.support.runner -import com.intellij.openapi.util.SystemInfo -import com.intellij.openapi.util.text.StringUtilRt.convertLineSeparators import org.jetbrains.kotlin.konan.test.blackbox.support.LoggedData -import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind -import org.jetbrains.kotlin.konan.test.blackbox.support.TestName -import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExecutionTimeout -import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExitCode -import org.jetbrains.kotlin.konan.test.blackbox.support.settings.configurables -import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestOutputFilter -import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestReport -import org.jetbrains.kotlin.native.executors.RunProcessResult -import org.jetbrains.kotlin.native.executors.runProcess -import org.junit.jupiter.api.Assumptions -import java.io.File -import kotlin.time.Duration - -private fun RunResult.processOutputAsString(output: TestRunCheck.Output) = when (output) { - TestRunCheck.Output.STDOUT -> processOutput.stdOut.filteredOutput - TestRunCheck.Output.STDERR -> processOutput.stdErr - TestRunCheck.Output.ALL -> processOutput.stdOut.filteredOutput + processOutput.stdErr -} internal class ResultHandler( runResult: RunResult, - private val visibleProcessName: String, private val checks: TestRunChecks, private val testRun: TestRun, private val loggedParameters: LoggedData.TestRunParameters, @@ -38,134 +17,12 @@ internal class ResultHandler( override fun getLoggedRun() = LoggedData.TestRun(loggedParameters, runResult) override fun handle() { - val diagnostics = buildList { - checks.forEach { check -> - when (check) { - is ExecutionTimeout.ShouldNotExceed -> if (!runResult.hasFinishedOnTime) add( - "Timeout exceeded during test execution." - ) - is ExecutionTimeout.ShouldExceed -> if (runResult.hasFinishedOnTime) add( - "Test is expected to fail with exceeded timeout, which hasn't happened." - ) - is ExitCode -> { - // Don't check exit code if it is unknown. - val knownExitCode: Int = runResult.exitCode ?: return@forEach - when (check) { - is ExitCode.Expected -> if (knownExitCode != check.expectedExitCode) add( - "$visibleProcessName exit code is $knownExitCode while ${check.expectedExitCode} was expected." - ) - is ExitCode.AnyNonZero -> if (knownExitCode == 0) add( - "$visibleProcessName exited with zero code, which wasn't expected." - ) - } - } - is TestRunCheck.OutputDataFile -> { - val expectedOutput = check.file.readText() - val actualFilteredOutput = runResult.processOutputAsString(check.output) - - // Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message - // and blowing up test logs. - if (convertLineSeparators(expectedOutput) != convertLineSeparators(actualFilteredOutput)) add( - "Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below." - ) - } - is TestRunCheck.OutputMatcher -> { - try { - if (!check.match(runResult.processOutputAsString(check.output))) add( - "Tested process output has not passed validation." - ) - } catch (t: Throwable) { - if (t is Exception || t is AssertionError) { - add("Tested process output has not passed validation: " + t.message) - } else { - throw t - } - } - } - is TestRunCheck.FileCheckMatcher -> { - val fileCheckDump = runResult.testExecutable.executable.fileCheckDump!! - val result = doFileCheck(check, fileCheckDump) - - if (!(result.stdout.isEmpty() && result.stderr.isEmpty())) { - val shortOutText = result.stdout.lines().take(100) - val shortErrText = result.stderr.lines().take(100) - add( - "FileCheck matching of ${fileCheckDump.absolutePath}\n" + - "with '--check-prefixes ${check.prefixes}'\n" + - "failed with result=$result:\n" + - shortOutText.joinToString("\n") + "\n" + - shortErrText.joinToString("\n") - ) - } - } - is TestRunCheck.TestFiltering -> { - if (check.testOutputFilter != TestOutputFilter.NO_FILTERING) { - val testReport = runResult.processOutput.stdOut.testReport - - checkNotNull(testReport) { "TestRun has TestFiltering enabled, but test report is null" } - - if (testReport.isEmpty()) add("No tests have been found. Test report is empty") - - testRun.runParameters.get { - verifyNoSuchTests( - testReport.passedTests.filter { testName -> !testMatches(testName) }, - "Excessive tests have been executed" - ) - - verifyNoSuchTests( - testReport.ignoredTests.filter { testName -> !testMatches(testName) }, - "Excessive tests have been ignored" - ) - - verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report") - - testReport.checkDisabled() - } - - testRun.runParameters.getAll { - verifyNoSuchTests( - testReport.passedTests.filter { testName -> !testMatches(testName) }, - "Ignored tests have been executed" - ) - verifyNoSuchTests( - testReport.failedTests.filter { testName -> - testName.packageName == testRun.testCase.nominalPackageName - }, - "Test failure found in the test report" - ) - - Assumptions.assumeFalse( - testReport.ignoredTests.any { testName -> - testName.packageName == testRun.testCase.nominalPackageName - }, - "Test case is disabled" - ) - } - - if (!testRun.runParameters.has()) { - verifyNoSuchTests( - testReport.failedTests.filter { testName -> - testName.packageName == testRun.testCase.nominalPackageName - }, - "Test ${testRun.testCase.id} failure found in the test report" - ) - - testReport.checkDisabled() - } - - if (testRun.testCase.kind == TestKind.STANDALONE) { - verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report") - - testReport.checkDisabled() - } - } - } - } - } - } + val failedResults = checks.map { check -> + check.apply(testRun, runResult) + }.filterIsInstance() if (!testRun.expectedFailure) { - verifyExpectation(diagnostics.isEmpty()) { - diagnostics.joinToString("\n") + verifyExpectation(failedResults.isEmpty()) { + failedResults.joinToString("\n") } } else { val runResultInfo = buildString { @@ -178,53 +35,14 @@ internal class ResultHandler( }) appendLine(runResult.processOutput.stdOut.testReport) } - verifyExpectation(diagnostics.isNotEmpty()) { + verifyExpectation(failedResults.isNotEmpty()) { "Test did not fail as expected: $runResultInfo" } println("Test failed as expected.\n$runResultInfo") - if (diagnostics.isNotEmpty()) { + if (failedResults.isNotEmpty()) { println("Diagnostics are:") - diagnostics.forEach(::println) + failedResults.forEach(::println) } } } - - private fun MutableList.verifyNoSuchTests(tests: Collection, subject: String) { - if (tests.isNotEmpty()) { - add( - buildString { - append(subject).append(':') - tests.forEach { appendLine().append(" - ").append(it) } - } - ) - } - } - - private fun TestReport.checkDisabled() { - Assumptions.assumeFalse( - ignoredTests.isNotEmpty() && passedTests.isEmpty(), - "Test case is disabled" - ) - } -} - -internal fun doFileCheck(check: TestRunCheck.FileCheckMatcher, fileCheckDump: File): RunProcessResult { - val fileCheckExecutable = check.settings.configurables.absoluteLlvmHome + File.separator + "bin" + File.separator + - if (SystemInfo.isWindows) "FileCheck.exe" else "FileCheck" - require(File(fileCheckExecutable).exists()) { - "$fileCheckExecutable does not exist. Make sure Distribution for `settings.configurables` " + - "was created using `propertyOverrides` to specify development variant of LLVM instead of user variant." - } - return try { - runProcess( - fileCheckExecutable, - check.testDataFile.absolutePath, - "--input-file", - fileCheckDump.absolutePath, - "--check-prefixes", check.prefixes, - "--allow-deprecated-dag-overlap" // TODO specify it via new test directive for `function_attributes_at_callsite.kt` - ) - } catch (t: Throwable) { - RunProcessResult(Duration.ZERO, "FileCheck utility failed:", t.toString()) - } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/RunnerWithExecutor.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/RunnerWithExecutor.kt index 968e1d54e92..44921393a1d 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/RunnerWithExecutor.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/RunnerWithExecutor.kt @@ -7,7 +7,6 @@ package org.jetbrains.kotlin.konan.test.blackbox.support.runner import org.jetbrains.kotlin.konan.test.blackbox.support.LoggedData import org.jetbrains.kotlin.konan.test.blackbox.support.runner.AbstractRunner.AbstractRun -import org.jetbrains.kotlin.konan.test.blackbox.support.util.TCTestOutputFilter import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestOutputFilter import org.jetbrains.kotlin.native.executors.ExecuteRequest import org.jetbrains.kotlin.native.executors.Executor @@ -66,7 +65,6 @@ internal open class RunnerWithExecutor( override fun buildResultHandler(runResult: RunResult): ResultHandler = ResultHandler( runResult = runResult, - visibleProcessName = "Test process under Executor ${executor::class.simpleName}", checks = testRun.checks, testRun = testRun, loggedParameters = getLoggedParameters() diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt index aa645c76dd5..5802190c1ca 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt @@ -112,7 +112,6 @@ internal object SharedExecutionBuilder { override fun buildResultHandler(runResult: RunResult): ResultHandler = ResultHandler( runResult = runResult, - visibleProcessName = "Test process under ${this::class.simpleName}", checks = sharedTestRun.checks, testRun = testRun.copy( runParameters = sharedTestRun.runParameters diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunChecks.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunChecks.kt index 9b623ac32d4..70c867abff4 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunChecks.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunChecks.kt @@ -5,29 +5,76 @@ package org.jetbrains.kotlin.konan.test.blackbox.support.runner +import com.intellij.openapi.util.SystemInfo +import com.intellij.openapi.util.text.StringUtilRt import org.jetbrains.kotlin.konan.target.Architecture import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.target.needSmallBinary +import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind +import org.jetbrains.kotlin.konan.test.blackbox.support.TestName import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.* +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.* import org.jetbrains.kotlin.konan.test.blackbox.support.settings.CacheMode import org.jetbrains.kotlin.konan.test.blackbox.support.settings.KotlinNativeTargets import org.jetbrains.kotlin.konan.test.blackbox.support.settings.OptimizationMode import org.jetbrains.kotlin.konan.test.blackbox.support.settings.Settings import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestOutputFilter +import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestReport +import org.jetbrains.kotlin.native.executors.RunProcessResult +import org.jetbrains.kotlin.native.executors.runProcess import org.jetbrains.kotlin.util.capitalizeDecapitalize.toUpperCaseAsciiOnly import org.jetbrains.kotlin.utils.yieldIfNotNull +import org.junit.jupiter.api.Assumptions import java.io.File import kotlin.time.Duration internal sealed interface TestRunCheck { + + fun apply(testRun: TestRun, runResult: RunResult): Result + + sealed interface Result { + data object Passed : Result + data class Failed(val reason: String) : Result + } + sealed class ExecutionTimeout(val timeout: Duration) : TestRunCheck { - class ShouldNotExceed(timeout: Duration) : ExecutionTimeout(timeout) - class ShouldExceed(timeout: Duration) : ExecutionTimeout(timeout) + class ShouldNotExceed(timeout: Duration) : ExecutionTimeout(timeout) { + override fun apply(testRun: TestRun, runResult: RunResult): Result = + if (!runResult.hasFinishedOnTime) + Result.Failed("Timeout exceeded during test execution.") + else Result.Passed + } + + class ShouldExceed(timeout: Duration) : ExecutionTimeout(timeout) { + override fun apply(testRun: TestRun, runResult: RunResult): Result = + if (runResult.hasFinishedOnTime) + Result.Failed("Test is expected to fail with exceeded timeout, which hasn't happened.") + else Result.Passed + } } sealed class ExitCode : TestRunCheck { - object AnyNonZero : ExitCode() - class Expected(val expectedExitCode: Int) : ExitCode() + data object AnyNonZero : ExitCode() + + data class Expected(val expectedExitCode: Int) : ExitCode() + + override fun apply(testRun: TestRun, runResult: RunResult): Result { + // Don't check the exit code if it is unknown. + val knownExitCode: Int = runResult.exitCode ?: return Result.Passed + + return when (this) { + AnyNonZero -> { + if (knownExitCode == 0) + Result.Failed("Test exited with zero code, which wasn't expected.") + else Result.Passed + } + is Expected -> { + if (knownExitCode != expectedExitCode) + Result.Failed("Exit code is $knownExitCode while $expectedExitCode was expected.") + else Result.Passed + } + } + } } enum class Output { @@ -40,11 +87,123 @@ internal sealed interface TestRunCheck { ALL, } - class OutputDataFile(val output: Output = Output.ALL, val file: File) : TestRunCheck + fun RunResult.processOutputAsString(output: Output) = when (output) { + Output.STDOUT -> processOutput.stdOut.filteredOutput + Output.STDERR -> processOutput.stdErr + Output.ALL -> processOutput.stdOut.filteredOutput + processOutput.stdErr + } - class OutputMatcher(val output: Output = Output.ALL, val match: (String) -> Boolean) : TestRunCheck + class OutputDataFile(val output: Output = Output.ALL, val file: File) : TestRunCheck { + override fun apply(testRun: TestRun, runResult: RunResult): Result { + val expectedOutput = file.readText() + val actualFilteredOutput = runResult.processOutputAsString(output) - class TestFiltering(val testOutputFilter: TestOutputFilter) : TestRunCheck + return if (StringUtilRt.convertLineSeparators(expectedOutput) != StringUtilRt.convertLineSeparators(actualFilteredOutput)) + Result.Failed("Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below.") + else Result.Passed + } + } + + class OutputMatcher(val output: Output = Output.ALL, val match: (String) -> Boolean) : TestRunCheck { + override fun apply(testRun: TestRun, runResult: RunResult): Result = + try { + if (!match(runResult.processOutputAsString(output))) { + Result.Failed("Tested process output has not passed validation.") + } else Result.Passed + } catch (t: Throwable) { + if (t is Exception || t is AssertionError) { + Result.Failed("Tested process output has not passed validation: ${t.message}") + } else { + throw t + } + } + } + + class TestFiltering(val testOutputFilter: TestOutputFilter) : TestRunCheck { + override fun apply(testRun: TestRun, runResult: RunResult): Result { + if (testOutputFilter != TestOutputFilter.NO_FILTERING) { + val testReport = runResult.processOutput.stdOut.testReport + + checkNotNull(testReport) { "TestRun has TestFiltering enabled, but test report is null" } + + if (testReport.isEmpty()) Result.Failed("No tests have been found. Test report is empty") + + testRun.runParameters.get { + testReport.checkDisabled() + + return listOf( + verifyNoSuchTests( + testReport.passedTests.filter { testName -> !testMatches(testName) }, + "Excessive tests have been executed" + ), + verifyNoSuchTests( + testReport.ignoredTests.filter { testName -> !testMatches(testName) }, + "Excessive tests have been ignored" + ), + verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report") + ).filterIsInstance().firstOrNull() ?: Result.Passed + } + + testRun.runParameters.getAll { + Assumptions.assumeFalse( + testReport.ignoredTests.any { testName -> + testName.packageName == testRun.testCase.nominalPackageName + }, + "Test case is disabled" + ) + + return listOf( + verifyNoSuchTests( + testReport.passedTests.filter { testName -> !testMatches(testName) }, + "Ignored tests have been executed" + ), + verifyNoSuchTests( + testReport.failedTests.filter { testName -> + testName.packageName == testRun.testCase.nominalPackageName + }, + "Test failure found in the test report" + ) + ).filterIsInstance().firstOrNull() ?: Result.Passed + } + + if (testRun.testCase.kind == TestKind.STANDALONE) { + testReport.checkDisabled() + + return verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report") + } + + if (!testRun.runParameters.has()) { + testReport.checkDisabled() + + return verifyNoSuchTests( + testReport.failedTests.filter { testName -> + testName.packageName == testRun.testCase.nominalPackageName + }, + "Test ${testRun.testCase.id} failure found in the test report" + ) + } + } + return Result.Passed + } + + private fun verifyNoSuchTests(tests: Collection, subject: String): Result { + return if (tests.isNotEmpty()) { + Result.Failed( + buildString { + append(subject).append(':') + tests.forEach { appendLine().append(" - ").append(it) } + } + ) + } else Result.Passed + } + + private fun TestReport.checkDisabled() { + Assumptions.assumeFalse( + ignoredTests.isNotEmpty() && passedTests.isEmpty(), + "Test case is disabled" + ) + } + } class FileCheckMatcher(val settings: Settings, val testDataFile: File) : TestRunCheck { val prefixes: String @@ -69,6 +228,55 @@ internal sealed interface TestRunCheck { val checkPrefixesWithCacheMode = checkPrefixes.map { "$it-CACHE_$cacheMode" } return (checkPrefixes + checkPrefixesWithOptMode + checkPrefixesWithCacheMode).joinToString(",") } + + // Shameless borrowing `val KonanTarget.abiInfo` from module `:kotlin-native:backend.native`, which cannot be imported here for now. + private val KonanTarget.abiInfoString: String + get() = when { + this == KonanTarget.MINGW_X64 -> "WINDOWSX64" + !family.isAppleFamily && architecture == Architecture.ARM64 -> "AAPCS" + else -> "DEFAULTABI" + } + + override fun apply(testRun: TestRun, runResult: RunResult): Result { + val fileCheckDump = runResult.testExecutable.executable.fileCheckDump!! + val result = doFileCheck(fileCheckDump) + + return if (!(result.stdout.isEmpty() && result.stderr.isEmpty())) { + val shortOutText = result.stdout.lines().take(100) + val shortErrText = result.stderr.lines().take(100) + + Result.Failed( + """ + FileCheck matching of ${fileCheckDump.absolutePath} + with '--check-prefixes $prefixes' + failed with result=$result: + ${shortOutText.joinToString("\n")} + ${shortErrText.joinToString("\n")} + """.trimIndent() + ) + } else Result.Passed + } + + internal fun doFileCheck(fileCheckDump: File): RunProcessResult { + val fileCheckExecutable = settings.configurables.absoluteLlvmHome + File.separator + "bin" + File.separator + + if (SystemInfo.isWindows) "FileCheck.exe" else "FileCheck" + require(File(fileCheckExecutable).exists()) { + "$fileCheckExecutable does not exist. Make sure Distribution for `settings.configurables` " + + "was created using `propertyOverrides` to specify development variant of LLVM instead of user variant." + } + return try { + runProcess( + fileCheckExecutable, + testDataFile.absolutePath, + "--input-file", + fileCheckDump.absolutePath, + "--check-prefixes", prefixes, + "--allow-deprecated-dag-overlap" // TODO specify it via new test directive for `function_attributes_at_callsite.kt` + ) + } catch (t: Throwable) { + RunProcessResult(Duration.ZERO, "FileCheck utility failed:", t.toString()) + } + } } } @@ -103,11 +311,3 @@ internal data class TestRunChecks( ) } } - -// Shameless borrowing `val KonanTarget.abiInfo` from module `:kotlin-native:backend.native`, which cannot be imported here for now. -private val KonanTarget.abiInfoString: String - get() = when { - this == KonanTarget.MINGW_X64 -> "WINDOWSX64" - !family.isAppleFamily && architecture == Architecture.ARM64 -> "AAPCS" - else -> "DEFAULTABI" - }