[K/N][test] Check test filtering with different run parameters

Adds additional checks when different test filtering is used, like
ignored tests filter.
This makes SharedExecutionBuilder be able to handle tests separately
but run together in the executable.

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt

@@ -8,11 +8,13 @@ package org.jetbrains.kotlin.konan.test.blackbox.support.runner
 import com.intellij.openapi.util.SystemInfo
 import com.intellij.openapi.util.text.StringUtilRt.convertLineSeparators
 import org.jetbrains.kotlin.konan.test.blackbox.support.LoggedData
+import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestName
 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExecutionTimeout
 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExitCode
 import org.jetbrains.kotlin.konan.test.blackbox.support.settings.configurables
 import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestOutputFilter
+import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestReport
 import org.jetbrains.kotlin.native.executors.RunProcessResult
 import org.jetbrains.kotlin.native.executors.runProcess
 import org.junit.jupiter.api.Assumptions
@@ -100,10 +102,7 @@ internal class ResultHandler(
                         if (check.testOutputFilter != TestOutputFilter.NO_FILTERING) {
                             val testReport = runResult.processOutput.stdOut.testReport
 
-                            if (testReport == null) {
-                                add("TestRun has TestFiltering enabled, but test report is null")
-                            }
-                            checkNotNull(testReport)
+                            checkNotNull(testReport) { "TestRun has TestFiltering enabled, but test report is null" }
 
                             if (testReport.isEmpty()) add("No tests have been found. Test report is empty")
 
@@ -117,14 +116,48 @@ internal class ResultHandler(
                                     testReport.ignoredTests.filter { testName -> !testMatches(testName) },
                                     "Excessive tests have been ignored"
                                 )
+
+                                verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report")
+
+                                testReport.checkDisabled()
                             }
 
-                            verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report")
+                            testRun.runParameters.getAll<TestRunParameter.WithIgnoredTestFilter> {
+                                verifyNoSuchTests(
+                                    testReport.passedTests.filter { testName -> !testMatches(testName) },
+                                    "Ignored tests have been executed"
+                                )
+                                verifyNoSuchTests(
+                                    testReport.failedTests.filter { testName ->
+                                        testName.packageName == testRun.testCase.nominalPackageName
+                                    },
+                                    "Test failure found in the test report"
+                                )
 
-                            Assumptions.assumeFalse(
-                                testReport.ignoredTests.isNotEmpty() && testReport.passedTests.isEmpty(),
-                                "Test case is disabled"
-                            )
+                                Assumptions.assumeFalse(
+                                    testReport.ignoredTests.any { testName ->
+                                        testName.packageName == testRun.testCase.nominalPackageName
+                                    },
+                                    "Test case is disabled"
+                                )
+                            }
+
+                            if (!testRun.runParameters.has<TestRunParameter.WithFilter>()) {
+                                verifyNoSuchTests(
+                                    testReport.failedTests.filter { testName ->
+                                        testName.packageName == testRun.testCase.nominalPackageName
+                                    },
+                                    "Test ${testRun.testCase.id} failure found in the test report"
+                                )
+
+                                testReport.checkDisabled()
+                            }
+
+                            if (testRun.testCase.kind == TestKind.STANDALONE) {
+                                verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report")
+
+                                testReport.checkDisabled()
+                            }
                         }
                     }
                 }
@@ -136,6 +169,7 @@ internal class ResultHandler(
             }
         } else {
             val runResultInfo = buildString {
+                appendLine("TestCase Kind: ${testRun.testCase.kind}")
                 appendLine("TestCaseId: ${testRun.testCase.id}")
                 appendLine("Exit code: ${runResult.exitCode}")
                 appendLine("Filtered test output is")
@@ -165,6 +199,13 @@ internal class ResultHandler(
             )
         }
     }
+
+    private fun TestReport.checkDisabled() {
+        Assumptions.assumeFalse(
+            ignoredTests.isNotEmpty() && passedTests.isEmpty(),
+            "Test case is disabled"
+        )
+    }
 }
 
 internal fun doFileCheck(check: TestRunCheck.FileCheckMatcher, fileCheckDump: File): RunProcessResult {

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt

@@ -24,7 +24,7 @@ import java.util.concurrent.ConcurrentHashMap
  * @see SharedExecutionTestRunner
  */
 internal object SharedExecutionBuilder {
-    private val executionResults: ConcurrentHashMap<TestExecutable, AbstractRunner<Unit>> = ConcurrentHashMap()
+    private val executableToSharedRun: ConcurrentHashMap<TestExecutable, TestRun> = ConcurrentHashMap()
     private val testCasesToExecuteSeparately: ConcurrentHashMap<TestExecutable, MutableList<TestCase>> = ConcurrentHashMap()
 
     fun buildRunner(settings: Settings, executor: Executor, testRun: TestRun): AbstractRunner<Unit> {
@@ -38,7 +38,7 @@ internal object SharedExecutionBuilder {
             return RunnerWithExecutor(executor, testRun)
         }
 
-        return executionResults.computeIfAbsent(testRun.executable) {
+        val sharedTestRun = executableToSharedRun.computeIfAbsent(testRun.executable) {
             // Get ignored tests to exclude them from run by adding the test filtering option
             val ignoredTests = if (testRun.testCase.extras is TestCase.WithTestRunnerExtras) {
                 testRun.testCase.extras.ignoredTests
@@ -60,7 +60,7 @@ internal object SharedExecutionBuilder {
                 executionTimeoutCheck = TestRunCheck.ExecutionTimeout.ShouldNotExceed(timeout)
             )
 
-            val sharedTestRun = TestRun(
+            TestRun(
                 displayName = "Shared TestRun for ${testRun.executable.executable.path} made from ${testRun.displayName}",
                 executable = testRun.executable,
                 runParameters = runParameters,
@@ -68,8 +68,9 @@ internal object SharedExecutionBuilder {
                 checks = checks,
                 expectedFailure = false
             )
-            CachedRunResultRunner(executor, sharedTestRun)
         }
+
+        return CachedRunResultRunner(executor, testRun, sharedTestRun)
     }
 
     private fun Settings.computeSeparateTestCases(testRun: TestRun): MutableList<TestCase> =
@@ -100,11 +101,23 @@ internal object SharedExecutionBuilder {
             }.toMutableList()
         }
 
-    private class CachedRunResultRunner(executor: Executor, testRun: TestRun) : RunnerWithExecutor(executor, testRun) {
-        private val cachedRunResult by lazy(LazyThreadSafetyMode.SYNCHRONIZED) {
-            super.buildRun().run()
+    private val cachedRunResult = ConcurrentHashMap<TestRun, RunResult>()
+
+    private class CachedRunResultRunner(executor: Executor, private val testRun: TestRun, private val sharedTestRun: TestRun) :
+        RunnerWithExecutor(executor, sharedTestRun) {
+
+        override fun buildRun() = AbstractRun {
+            cachedRunResult.computeIfAbsent(sharedTestRun) { super.buildRun().run() }
         }
 
-        override fun buildRun() = AbstractRun { cachedRunResult }
+        override fun buildResultHandler(runResult: RunResult): ResultHandler = ResultHandler(
+            runResult = runResult,
+            visibleProcessName = "Test process under ${this::class.simpleName}",
+            checks = sharedTestRun.checks,
+            testRun = testRun.copy(
+                runParameters = sharedTestRun.runParameters
+            ),
+            loggedParameters = getLoggedParameters()
+        )
     }
 }

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt

@@ -146,6 +146,9 @@ internal inline fun <reified T : TestRunParameter> List<TestRunParameter>.get(on
     firstIsInstanceOrNull<T>()?.let(onFound)
 }
 
+internal inline fun <reified T : TestRunParameter> List<TestRunParameter>.getAll(onFound: T.() -> Unit) {
+    filterIsInstance<T>().forEach(onFound)
+}
 
 // must be in sync with `fromGTestPattern(String)` in kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestRunner.kt
 internal fun fromGTestPattern(pattern: String): Regex {