[K/N] Implement a concept of conditional ignore of K/N blackbox tests

^KT-59288 Fixed
2023-10-28 17:34:33 +02:00
parent 6ce33e80a5
commit c38a0e10af
93 changed files with 465 additions and 860 deletions
@@ -34,12 +34,6 @@ fun main() {
                suiteTestClassName = "NativeCodegenBoxTestGenerated",
                annotations = listOf(
                    provider<UseExtTestCaseGroupProvider>(),
-                    disabledInOneStageMode(
-                        "codegen/box/coroutines/featureIntersection/defaultExpect.kt",
-                        "codegen/box/multiplatform/defaultArguments/*.kt",
-                        "codegen/box/multiplatform/migratedOldTests/*.kt",
-                        "codegen/boxInline/multiplatform/defaultArguments/receiversAndParametersInLambda.kt"
-                    )
                )
            ) {
                model("codegen/box", targetBackend = TargetBackend.NATIVE)
@@ -9,6 +9,8 @@ import com.intellij.testFramework.TestDataFile
 import org.jetbrains.kotlin.konan.test.blackbox.support.NativeBlackBoxTestSupport
 import org.jetbrains.kotlin.konan.test.blackbox.support.PackageName
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestCaseId
+import org.jetbrains.kotlin.konan.test.blackbox.support.compilation.CompilationToolException
+import org.jetbrains.kotlin.konan.test.blackbox.support.group.isIgnoredTarget
 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRun
 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunProvider
 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunners.createProperTestRunner
@@ -17,6 +19,7 @@ import org.jetbrains.kotlin.konan.test.blackbox.support.util.TreeNode
 import org.jetbrains.kotlin.konan.test.blackbox.support.util.getAbsoluteFile
 import org.jetbrains.kotlin.konan.test.blackbox.support.util.joinPackageNames
 import org.jetbrains.kotlin.konan.test.blackbox.support.util.prependPackageName
+import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
 import org.junit.jupiter.api.DynamicNode
 import org.junit.jupiter.api.DynamicTest.dynamicTest
 import org.junit.jupiter.api.extension.ExtendWith
@@ -32,8 +35,17 @@ abstract class AbstractNativeBlackBoxTest {
     * This function should be called from a method annotated with [org.junit.jupiter.api.Test].
     */
    fun runTest(@TestDataFile testDataFilePath: String) {
-        val testCaseId = TestCaseId.TestDataFile(getAbsoluteFile(testDataFilePath))
-        runTestCase(testCaseId)
+        val absoluteTestFile = getAbsoluteFile(testDataFilePath)
+        val testCaseId = TestCaseId.TestDataFile(absoluteTestFile)
+        try {
+            runTestCase(testCaseId)
+        } catch (e: CompilationToolException) {
+            // TODO find out the way not to re-read test source file, but to re-use already extracted test directives.
+            if (testRunSettings.isIgnoredTarget(absoluteTestFile))
+                println("There was an expected failure: CompilationToolException: ${e.reason}")
+            else
+                fail { e.reason }
+        }
    }

    /**
@@ -6,8 +6,6 @@
 package org.jetbrains.kotlin.konan.test.blackbox

 import com.intellij.testFramework.TestDataFile
-import org.jetbrains.kotlin.konan.test.blackbox.support.ClassLevelProperty
-import org.jetbrains.kotlin.konan.test.blackbox.support.group.DisabledTestsIfProperty
 import org.jetbrains.kotlin.konan.test.blackbox.support.settings.ExternalSourceTransformersProvider
 import org.jetbrains.kotlin.konan.test.blackbox.support.util.ExternalSourceTransformer
 import org.jetbrains.kotlin.konan.test.blackbox.support.util.ExternalSourceTransformers
@@ -16,13 +14,6 @@ import org.jetbrains.kotlin.konan.test.blackbox.support.util.getAbsoluteFile
 import org.junit.jupiter.api.Tag
 import java.io.File

-// Disable codegen/box/properties/lateinit/isInitializedAndDeinitialize tests with ONE_STAGE_MULTI_MODULE
-//  They should be disabled only for K2 but this is not possible right now.
-@DisabledTestsIfProperty(
-    sourceLocations = ["compiler/testData/codegen/box/properties/lateinit/isInitializedAndDeinitialize/*.kt"],
-    property = ClassLevelProperty.TEST_MODE,
-    propertyValue = "ONE_STAGE_MULTI_MODULE"
-)
@Tag("codegen")
 abstract class AbstractNativeCodegenBoxTest : ExternalSourceTransformersProvider, AbstractNativeBlackBoxTest() {
    private val registeredSourceTransformers: ThreadSafeCache<File, MutableList<ExternalSourceTransformer>> = ThreadSafeCache()
@@ -54,7 +54,7 @@ internal class EnforcedProperties(testClass: Class<*>) {
        acceptableAnnotations[propertyType]?.contains(value) ?: true
 }

-internal enum class ClassLevelProperty(shortName: String) {
+internal enum class ClassLevelProperty(val shortName: String) {
    TEST_TARGET("target"),
    TEST_MODE("mode"),
    COMPILER_PLUGINS("compilerPlugins"),
@@ -283,7 +283,8 @@ private object NativeTestSupport {
            kotlinNativeTargets,
            optimizationMode,
            useStaticCacheForUserLibraries,
-            makePerFileCaches
+            makePerFileCaches,
+            cacheMode
        )
    }

@@ -148,6 +148,39 @@ internal object TestDirectives : SimpleDirectivesContainer() {
    val FILECHECK_STAGE by stringDirective(
        description = "Specify a LLVM stage to dump LLVM IR after, and check it with LLVM FileCheck using its directives in test file"
    )
+
+    val DISABLE_NATIVE by stringDirective(
+        description = "Test is not compiled/run with neither K1 nor K2 frontend and marked as disabled(GRAY)."
+    )
+
+    val DISABLE_NATIVE_K1 by stringDirective(
+        description = "Test is not compiled/run with K1 frontend and marked as disabled(GRAY)."
+    )
+
+    val DISABLE_NATIVE_K2 by stringDirective(
+        description = "Test is not compiled/run with K2 frontend and marked as disabled(GRAY)."
+    )
+
+    val IGNORE_NATIVE by stringDirective(
+        """
+            Usage: // IGNORE_NATIVE: property1=value1[ && property2=value2][ && property3=value3]
+            Declares this test is expected to fail in described run configuration on both K1 and K2 frontends.
+        """.trimIndent()
+    )
+
+    val IGNORE_NATIVE_K1 by stringDirective(
+        """
+            Usage: // IGNORE_NATIVE_K1: property1=value1[ && property2=value2][ && property3=value3]
+            Declares this test is expected to fail in described run configuration on K1 frontend.
+        """.trimIndent()
+    )
+
+    val IGNORE_NATIVE_K2 by stringDirective(
+        """
+            Usage: // IGNORE_NATIVE_K2: property1=value1[ && property2=value2][ && property3=value3]
+            Declares this test is expected to fail in described run configuration on K2 frontend.
+        """.trimIndent()
+    )
 }

 internal enum class TestKind {
@@ -8,6 +8,8 @@ package org.jetbrains.kotlin.konan.test.blackbox.support.compilation
 import org.jetbrains.kotlin.konan.test.blackbox.support.LoggedData
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail

+class CompilationToolException(val reason: String) : Exception()
+
 internal sealed interface TestCompilationResult<A : TestCompilationArtifact> {
    sealed interface ImmediateResult<A : TestCompilationArtifact> : TestCompilationResult<A> {
        val loggedData: LoggedData
@@ -25,7 +27,8 @@ internal sealed interface TestCompilationResult<A : TestCompilationArtifact> {
    companion object {
        fun <A : TestCompilationArtifact> TestCompilationResult<A>.assertSuccess(): Success<A> = when (this) {
            is Success -> this
-            is Failure -> fail { describeFailure() }
+            is UnexpectedFailure -> fail { describeFailure() }
+            is CompilationToolFailure -> throw CompilationToolException(describeFailure())
            is DependencyFailures -> fail { describeDependencyFailures() }
        }

@@ -6,11 +6,19 @@ package org.jetbrains.kotlin.konan.test.blackbox.support.group

 import org.jetbrains.kotlin.konan.test.blackbox.support.ClassLevelProperty

+// Deprecated: Use test directives instead:
+// `IGNORE_BACKEND: NATIVE`, `IGNORE_BACKEND_K1: NATIVE`, `IGNORE_BACKEND_K2: NATIVE` if test fails in any tets config
+// `IGNORE_NATIVE`, `IGNORE_NATIVE_K1`, `IGNORE_NATIVE_K2` with a property matcher, for usual easy fails
+// `DISABLE_NATIVE`, `DISABLE_NATIVE_K1`, `DISABLE_NATIVE_K2` in case compiler crashes within JVM
@Target(AnnotationTarget.CLASS)
 internal annotation class DisabledTests(
    val sourceLocations: Array<String>
 )

+// @DisabledTestsIfProperty(...) is intended primarily to turn off tests in bulk to reduce pressure on CI infrastructure for certain targets
+// To mark failed tests, please use the following test directives instead:
+// `IGNORE_NATIVE`, `IGNORE_NATIVE_K1`, `IGNORE_NATIVE_K2` with a property matcher, for usual easy fails
+// `DISABLE_NATIVE`, `DISABLE_NATIVE_K1`, `DISABLE_NATIVE_K2` for overcomplicated property combinations, or in case compiler crashes within JVM
@Target(AnnotationTarget.CLASS)
 internal annotation class DisabledTestsIfProperty(
    val sourceLocations: Array<String>,
@@ -21,7 +21,13 @@ import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.LanguageFeature
 import org.jetbrains.kotlin.konan.test.blackbox.support.*
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestCase.WithTestRunnerExtras
+import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.DISABLE_NATIVE
+import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.DISABLE_NATIVE_K1
+import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.DISABLE_NATIVE_K2
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.FILECHECK_STAGE
+import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.IGNORE_NATIVE
+import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.IGNORE_NATIVE_K1
+import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.IGNORE_NATIVE_K2
 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck
 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunChecks
 import org.jetbrains.kotlin.konan.test.blackbox.support.settings.*
@@ -35,6 +41,9 @@ import org.jetbrains.kotlin.resolve.ImportPath
 import org.jetbrains.kotlin.resolve.checkers.OptInNames
 import org.jetbrains.kotlin.test.*
 import org.jetbrains.kotlin.test.InTextDirectivesUtils.*
+import org.jetbrains.kotlin.test.directives.CodegenTestDirectives.IGNORE_BACKEND
+import org.jetbrains.kotlin.test.directives.CodegenTestDirectives.IGNORE_BACKEND_K1
+import org.jetbrains.kotlin.test.directives.CodegenTestDirectives.IGNORE_BACKEND_K2
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
 import org.jetbrains.kotlin.utils.addIfNotNull
@@ -72,14 +81,7 @@ internal class ExtTestCaseGroupProvider : TestCaseGroupProvider, TestDisposable(
                    testDataFile = testDataFile,
                    structureFactory = structureFactory,
                    customSourceTransformers = settings.get<ExternalSourceTransformersProvider>().getSourceTransformers(testDataFile),
-                    testRoots = settings.get(),
-                    generatedSources = settings.get(),
-                    customKlibs = settings.get(),
-                    pipelineType = settings.get(),
-                    testMode = settings.get(),
-                    cacheMode = settings.get(),
-                    optimizationMode = settings.get(),
-                    timeouts = settings.get(),
+                    settings = settings,
                )

                if (extTestDataFile.isRelevant)
@@ -100,15 +102,17 @@ private class ExtTestDataFile(
    private val testDataFile: File,
    structureFactory: ExtTestDataFileStructureFactory,
    customSourceTransformers: ExternalSourceTransformers?,
-    testRoots: TestRoots,
-    private val generatedSources: GeneratedSources,
-    private val customKlibs: CustomKlibs,
-    pipelineType: PipelineType,
-    testMode: TestMode,
-    cacheMode: CacheMode,
-    optimizationMode: OptimizationMode,
-    private val timeouts: Timeouts,
+    settings: Settings,
 ) {
+    private val testRoots = settings.get<TestRoots>()
+    private val generatedSources = settings.get<GeneratedSources>()
+    private val customKlibs = settings.get<CustomKlibs>()
+    private val timeouts = settings.get<Timeouts>()
+    private val pipelineType = settings.get<PipelineType>()
+    private val testMode = settings.get<TestMode>()
+    private val cacheMode = settings.get<CacheMode>()
+    private val optimizationMode = settings.get<OptimizationMode>()
+
    private val structure by lazy {
        val allSourceTransformers: ExternalSourceTransformers = if (customSourceTransformers.isNullOrEmpty())
            MANDATORY_SOURCE_TRANSFORMERS
@@ -118,6 +122,8 @@ private class ExtTestDataFile(
        structureFactory.ExtTestDataFileStructure(testDataFile, allSourceTransformers)
    }

+    private val isIgnoredTarget: Boolean = settings.isIgnoredTarget(structure.directives)
+
    private val testDataFileSettings by lazy {
        val optIns = structure.directives.multiValues(OPT_IN_DIRECTIVE)
        val optInsForSourceCode = optIns subtract OPT_INS_PURELY_FOR_COMPILER
@@ -145,7 +151,7 @@ private class ExtTestDataFile(

    val isRelevant: Boolean =
        isCompatibleTarget(TargetBackend.NATIVE, testDataFile) // Checks TARGET_BACKEND/DONT_TARGET_EXACT_BACKEND directives.
-                && !isIgnoredTarget(pipelineType, testDataFile, TargetBackend.NATIVE) // Checks IGNORE_BACKEND directives.
+                && !isDisabledNative(pipelineType, structure.directives)
                && testDataFileSettings.languageSettings.none { it in INCOMPATIBLE_LANGUAGE_SETTINGS }
                && INCOMPATIBLE_DIRECTIVES.none { it in structure.directives }
                && structure.directives[API_VERSION_DIRECTIVE] !in INCOMPATIBLE_API_VERSIONS
@@ -157,34 +163,6 @@ private class ExtTestDataFile(
                     && pipelineType == PipelineType.K2
                     && testMode == TestMode.ONE_STAGE_MULTI_MODULE)

-    private fun isIgnoredTarget(pipelineType: PipelineType, testDataFile: File, backend: TargetBackend): Boolean {
-        return when (pipelineType) {
-            PipelineType.K1 ->
-                isIgnoredTarget(
-                    backend,
-                    testDataFile,
-                    /*includeAny = */true,
-                    IGNORE_BACKEND_DIRECTIVE_PREFIX,
-                    IGNORE_BACKEND_K1_DIRECTIVE_PREFIX
-                )
-            PipelineType.K2 ->
-                isIgnoredTarget(
-                    backend,
-                    testDataFile,
-                    /*includeAny = */true,
-                    IGNORE_BACKEND_DIRECTIVE_PREFIX,
-                    IGNORE_BACKEND_K2_DIRECTIVE_PREFIX
-                )
-            PipelineType.DEFAULT ->
-                isIgnoredTarget(
-                    backend,
-                    testDataFile,
-                    /*includeAny = */true,
-                    IGNORE_BACKEND_DIRECTIVE_PREFIX,
-                )
-        }
-    }
-
    private fun assembleFreeCompilerArgs(): TestCompilerArgs {
        val args = mutableListOf<String>()
        testDataFileSettings.languageSettings.sorted().mapTo(args) { "-XXLanguage:$it" }
@@ -214,6 +192,12 @@ private class ExtTestDataFile(
    private fun determineIfStandaloneTest(): Boolean = with(structure) {
        if (directives.contains(NATIVE_STANDALONE_DIRECTIVE)) return true
        if (directives.contains(FILECHECK_STAGE.name)) return true
+        if (isIgnoredTarget) return true
+        // To make the debug of possible failed testruns easier, it makes sense to run dodgy tests alone
+        if (directives.contains(IGNORE_NATIVE.name) ||
+            directives.contains(IGNORE_NATIVE_K1.name) ||
+            directives.contains(IGNORE_NATIVE_K2.name)
+        ) return true

        var isStandaloneTest = false

@@ -531,8 +515,11 @@ private class ExtTestDataFile(
            modules = modules,
            freeCompilerArgs = assembleFreeCompilerArgs(),
            nominalPackageName = testDataFileSettings.nominalPackageName,
-            checks = TestRunChecks.Default(timeouts.executionTimeout)
-                .copy(fileCheckMatcher = fileCheckStage?.let { TestRunCheck.FileCheckMatcher(settings, testDataFile) }),
+            checks = TestRunChecks.Default(timeouts.executionTimeout).copy(
+                fileCheckMatcher = fileCheckStage?.let { TestRunCheck.FileCheckMatcher(settings, testDataFile) },
+                exitCodeCheck = TestRunCheck.ExitCode.Expected(0).takeUnless { isIgnoredTarget },
+                expectedFailureCheck = TestRunCheck.ExpectedFailure.takeIf { isIgnoredTarget },
+            ),
            fileCheckStage = fileCheckStage,
            extras = WithTestRunnerExtras(runnerType = TestRunnerType.DEFAULT)
        )
@@ -612,7 +599,7 @@ private class ExtTestDataFileSettings(
 private typealias SharedModuleGenerator = (sharedModulesDir: File) -> TestModule.Shared?
 private typealias SharedModuleCache = (moduleName: String, generator: SharedModuleGenerator) -> TestModule.Shared?

-private class ExtTestDataFileStructureFactory(parentDisposable: Disposable) : TestDisposable(parentDisposable) {
+private class ExtTestDataFileStructureFactory(parentDisposable: Disposable?) : TestDisposable(parentDisposable) {
    private val psiFactory = createPsiFactory(parentDisposable = this)

    inner class ExtTestDataFileStructure(originalTestDataFile: File, sourceTransformers: ExternalSourceTransformers) {
@@ -882,3 +869,85 @@ private class ExtTestDataFileStructureFactory(parentDisposable: Disposable) : Te
        }
    }
 }
+
+internal fun isDisabledNative(pipelineType: PipelineType, directives: Directives): Boolean {
+    return when (pipelineType) {
+        PipelineType.K1 -> directives.contains(DISABLE_NATIVE.name) || directives.contains(DISABLE_NATIVE_K1.name)
+        PipelineType.K2 -> directives.contains(DISABLE_NATIVE.name) || directives.contains(DISABLE_NATIVE_K2.name)
+        PipelineType.DEFAULT -> directives.contains(DISABLE_NATIVE.name)
+    }
+}
+
+internal fun Settings.isIgnoredTarget(testDataFile: File): Boolean {
+    val extTestDataFileStructure =
+        ExtTestDataFileStructureFactory(parentDisposable = null).ExtTestDataFileStructure(testDataFile, emptyList())
+    return isIgnoredTarget(extTestDataFileStructure.directives)
+}
+
+internal fun Settings.isIgnoredTarget(directives: Directives): Boolean {
+    return isIgnoredWithIGNORE_BACKEND(directives) ||
+            isIgnoredWithIGNORE_NATIVE(directives)
+}
+
+// Mimics `InTextDirectivesUtils.isIgnoredTarget(NATIVE, file)` but does not require file contents, but only already parsed directives.
+private fun Settings.isIgnoredWithIGNORE_BACKEND(directives: Directives): Boolean {
+    val containsNativeOrAny: (List<String>) -> Boolean = { TargetBackend.NATIVE.name in it || TargetBackend.ANY.name in it }
+
+    if (directives.listValues(IGNORE_BACKEND.name)?.let(containsNativeOrAny) == true)
+        return true
+    when (get<PipelineType>()) {
+        PipelineType.K1 ->
+            if (directives.listValues(IGNORE_BACKEND_K1.name)?.let(containsNativeOrAny) == true)
+                return true
+        PipelineType.K2 ->
+            if (directives.listValues(IGNORE_BACKEND_K2.name)?.let(containsNativeOrAny) == true)
+                return true
+        else -> {}
+    }
+    return false
+}
+
+private val CACHE_MODE_NAMES = CacheMode.Alias.entries.map { it.name }
+private val TEST_MODE_NAMES = TestMode.entries.map { it.name }
+private val OPTIMIZATION_MODE_NAMES = OptimizationMode.entries.map { it.name }
+
+private fun Settings.isIgnoredWithIGNORE_NATIVE(
+    directives: Directives,
+): Boolean {
+    val directiveValues = buildList {
+        directives.listValues(IGNORE_NATIVE.name)?.let { addAll(it) }
+        when (get<PipelineType>()) {
+            PipelineType.K1 -> directives.listValues(IGNORE_NATIVE_K1.name)?.let { addAll(it) }
+            PipelineType.K2 -> directives.listValues(IGNORE_NATIVE_K2.name)?.let { addAll(it) }
+            else -> {}
+        }
+    }
+    // Boolean evaluation of expressions like `property1=value1 && property2=value2`
+    directiveValues.forEach {
+        val split = it.split("&&")
+        val booleanList = split.map {
+            val matchResult = "(.+)=(.+)".toRegex().find(it.trim())
+                ?: throw AssertionError("Invalid format for IGNORE_NATIVE* directive ($it). Must be <property>=<value>")
+            val propName = matchResult.groups[1]?.value
+            val (actualValue, supportedValues) = when (propName) {
+                ClassLevelProperty.CACHE_MODE.shortName -> get<CacheMode>().alias.name to CACHE_MODE_NAMES
+                ClassLevelProperty.TEST_MODE.shortName -> get<TestMode>().name to TEST_MODE_NAMES
+                ClassLevelProperty.OPTIMIZATION_MODE.shortName -> get<OptimizationMode>().name to OPTIMIZATION_MODE_NAMES
+                ClassLevelProperty.TEST_TARGET.shortName -> get<KotlinNativeTargets>().testTarget.name to null
+                else -> throw AssertionError("ClassLevelProperty name: $propName is not yet supported in IGNORE_NATIVE* test directives.")
+            }
+            val valueFromTestDirective = matchResult.groups[2]?.value!!
+            supportedValues?.let {
+                if (actualValue !in it)
+                    throw AssertionError("Internal error: Test run value $propName=$actualValue is not in expected supported values: $it")
+                if (valueFromTestDirective !in it)
+                    throw AssertionError("Test directive `IGNORE_NATIVE*: $propName=$valueFromTestDirective` has unsupported value. Supported are: $it")
+            }
+            actualValue == valueFromTestDirective
+        }
+        val matches = booleanList.reduce { a, b -> a && b }
+        if (matches)
+            return true
+    }
+    return false
+}
@@ -226,6 +226,7 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
            checks = TestRunChecks(
                computeExecutionTimeoutCheck(settings, expectedTimeoutFailure),
                computeExitCodeCheck(testKind, registeredDirectives, location),
+                expectedFailureCheck = null,
                computeOutputDataFileCheck(testDataFile, registeredDirectives, location),
                lldbSpec?.let { OutputMatcher { output -> lldbSpec.checkLLDBOutput(output, settings.get()) } },
                fileCheckMatcher = null,
@@ -96,7 +96,7 @@ internal abstract class AbstractLocalProcessRunner<R>(protected val checks: Test
 internal abstract class LocalResultHandler<R>(
    runResult: RunResult,
    private val visibleProcessName: String,
-    private val checks: TestRunChecks
+    protected val checks: TestRunChecks
 ) : AbstractResultHandler<R>(runResult) {
    override fun handle(): R {
        checks.forEach { check ->
@@ -107,6 +107,21 @@ internal abstract class LocalResultHandler<R>(
                is ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) {
                    "Test is expected to fail with exceeded timeout, which hasn't happened."
                }
+                is TestRunCheck.ExpectedFailure -> {
+                    val testReport = runResult.processOutput.stdOut.testReport
+                    verifyExpectation(testReport != null) {
+                        "testReport is expected to be non-null"
+                    }
+                    verifyExpectation(!testReport!!.isEmpty()) {
+                        "testReport is expected to be non-empty"
+                    }
+                    verifyExpectation(testReport.failedTests.isNotEmpty()) {
+                        "Test did not fail as expected"
+                    }
+                    verifyExpectation(testReport.passedTests.isEmpty()) {
+                        "Test unexpectedly passed"
+                    }
+                }
                is ExitCode -> {
                    // Don't check exit code if it is unknown.
                    val knownExitCode: Int = runResult.exitCode ?: return@forEach
@@ -83,8 +83,17 @@ internal class ResultHandler(
            )
        }

-        verifyNoSuchTests(testReport.failedTests, "There are failed tests")
-
+        if (checks.expectedFailureCheck !is TestRunCheck.ExpectedFailure)
+            verifyNoSuchTests(testReport.failedTests, "There are failed tests")
+        else {
+            runResult.processOutput.stdOut.filteredOutput.let {
+                if (it.isNotEmpty())
+                    println("Failure is expected. Exit code=${runResult.exitCode}, filtered test output is below:\n$it")
+                else
+                    println("Failure is expected. Exit code=${runResult.exitCode}, filtered test output is empty.")
+            }
+            verifyNoSuchTests(testReport.passedTests, "There are unexpectedly passed tests")
+        }
        assumeFalse(testReport.ignoredTests.isNotEmpty() && testReport.passedTests.isEmpty(), "Test case is disabled")
    }

@@ -22,6 +22,8 @@ internal sealed interface TestRunCheck {
        class Expected(val expectedExitCode: Int) : ExitCode()
    }

+    object ExpectedFailure : TestRunCheck
+
    class OutputDataFile(val file: File) : TestRunCheck

    class OutputMatcher(val match: (String) -> Boolean): TestRunCheck
@@ -31,7 +33,8 @@ internal sealed interface TestRunCheck {

 internal data class TestRunChecks(
    val executionTimeoutCheck: ExecutionTimeout,
-    private val exitCodeCheck: ExitCode,
+    private val exitCodeCheck: ExitCode?,
+    val expectedFailureCheck: ExpectedFailure?,
    val outputDataFile: OutputDataFile?,
    val outputMatcher: OutputMatcher?,
    val fileCheckMatcher: FileCheckMatcher?,
@@ -39,7 +42,8 @@ internal data class TestRunChecks(

    override fun iterator() = iterator {
        yield(executionTimeoutCheck)
-        yield(exitCodeCheck)
+        yieldIfNotNull(exitCodeCheck)
+        yieldIfNotNull(expectedFailureCheck)
        yieldIfNotNull(outputDataFile)
        yieldIfNotNull(outputMatcher)
        yieldIfNotNull(fileCheckMatcher)
@@ -51,6 +55,7 @@ internal data class TestRunChecks(
        fun Default(timeout: Duration) = TestRunChecks(
            executionTimeoutCheck = ExecutionTimeout.ShouldNotExceed(timeout),
            exitCodeCheck = ExitCode.Expected(0),
+            expectedFailureCheck = null,
            outputDataFile = null,
            outputMatcher = null,
            fileCheckMatcher = null,
@@ -210,6 +210,7 @@ internal sealed class CacheMode {
    abstract val staticCacheForDistributionLibrariesRootDir: File?
    abstract val useStaticCacheForUserLibraries: Boolean
    abstract val makePerFileCaches: Boolean
+    abstract val alias: Alias

    val useStaticCacheForDistributionLibraries: Boolean get() = staticCacheForDistributionLibrariesRootDir != null

@@ -217,6 +218,7 @@ internal sealed class CacheMode {
        override val staticCacheForDistributionLibrariesRootDir: File? get() = null
        override val useStaticCacheForUserLibraries: Boolean get() = false
        override val makePerFileCaches: Boolean = false
+        override val alias = Alias.NO
    }

    class WithStaticCache(
@@ -224,7 +226,8 @@ internal sealed class CacheMode {
        kotlinNativeTargets: KotlinNativeTargets,
        optimizationMode: OptimizationMode,
        override val useStaticCacheForUserLibraries: Boolean,
-        override val makePerFileCaches: Boolean
+        override val makePerFileCaches: Boolean,
+        override val alias: Alias,
    ) : CacheMode() {
        override val staticCacheForDistributionLibrariesRootDir: File = File(distribution.klib)
            .resolve("cache")