diff --git a/analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/LLFirBlackBoxCodegenBasedTestGenerated.java b/analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/LLFirBlackBoxCodegenBasedTestGenerated.java index dcb2858bb84..291053c3b4a 100644 --- a/analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/LLFirBlackBoxCodegenBasedTestGenerated.java +++ b/analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/LLFirBlackBoxCodegenBasedTestGenerated.java @@ -19522,6 +19522,12 @@ public class LLFirBlackBoxCodegenBasedTestGenerated extends AbstractLLFirBlackBo runTest("compiler/testData/codegen/box/fileCheck/escape_analysis.kt"); } + @Test + @TestMetadata("filecheck_expected_failure.kt") + public void testFilecheck_expected_failure() throws Exception { + runTest("compiler/testData/codegen/box/fileCheck/filecheck_expected_failure.kt"); + } + @Test @TestMetadata("force_arm_instruction_set.kt") public void testForce_arm_instruction_set() throws Exception { diff --git a/analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/LLFirReversedBlackBoxCodegenBasedTestGenerated.java b/analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/LLFirReversedBlackBoxCodegenBasedTestGenerated.java index 244d473286a..0239b2a6686 100644 --- a/analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/LLFirReversedBlackBoxCodegenBasedTestGenerated.java +++ b/analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/LLFirReversedBlackBoxCodegenBasedTestGenerated.java @@ -19522,6 +19522,12 @@ public class LLFirReversedBlackBoxCodegenBasedTestGenerated extends AbstractLLFi runTest("compiler/testData/codegen/box/fileCheck/escape_analysis.kt"); } + @Test + @TestMetadata("filecheck_expected_failure.kt") + public void testFilecheck_expected_failure() throws Exception { + runTest("compiler/testData/codegen/box/fileCheck/filecheck_expected_failure.kt"); + } + @Test @TestMetadata("force_arm_instruction_set.kt") public void testForce_arm_instruction_set() throws Exception { diff --git a/compiler/testData/codegen/box/fileCheck/filecheck_expected_failure.kt b/compiler/testData/codegen/box/fileCheck/filecheck_expected_failure.kt new file mode 100644 index 00000000000..b67ff010822 --- /dev/null +++ b/compiler/testData/codegen/box/fileCheck/filecheck_expected_failure.kt @@ -0,0 +1,8 @@ +// TARGET_BACKEND: NATIVE +// IGNORE_BACKEND: NATIVE +// FILECHECK_STAGE: CStubs + +// The check below is intentionally wrong and must fail. +// Test system should report test as passed due to IGNORE_BACKEND directive above. +// CHECK-LABEL: NONEXISTENT +fun box(): String = "OK" diff --git a/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/FirNativeCodegenBoxTestGenerated.java b/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/FirNativeCodegenBoxTestGenerated.java index e9453c73374..8e4956b5ec1 100644 --- a/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/FirNativeCodegenBoxTestGenerated.java +++ b/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/FirNativeCodegenBoxTestGenerated.java @@ -16198,6 +16198,12 @@ public class FirNativeCodegenBoxTestGenerated extends AbstractNativeCodegenBoxTe runTest("compiler/testData/codegen/box/fileCheck/escape_analysis.kt"); } + @Test + @TestMetadata("filecheck_expected_failure.kt") + public void testFilecheck_expected_failure() throws Exception { + runTest("compiler/testData/codegen/box/fileCheck/filecheck_expected_failure.kt"); + } + @Test @TestMetadata("force_arm_instruction_set.kt") public void testForce_arm_instruction_set() throws Exception { diff --git a/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/FirNativeCodegenBoxTestNoPLGenerated.java b/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/FirNativeCodegenBoxTestNoPLGenerated.java index 5998eb0a7b7..ac5c58fd365 100644 --- a/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/FirNativeCodegenBoxTestNoPLGenerated.java +++ b/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/FirNativeCodegenBoxTestNoPLGenerated.java @@ -16576,6 +16576,12 @@ public class FirNativeCodegenBoxTestNoPLGenerated extends AbstractNativeCodegenB runTest("compiler/testData/codegen/box/fileCheck/escape_analysis.kt"); } + @Test + @TestMetadata("filecheck_expected_failure.kt") + public void testFilecheck_expected_failure() throws Exception { + runTest("compiler/testData/codegen/box/fileCheck/filecheck_expected_failure.kt"); + } + @Test @TestMetadata("force_arm_instruction_set.kt") public void testForce_arm_instruction_set() throws Exception { diff --git a/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/NativeCodegenBoxTestGenerated.java b/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/NativeCodegenBoxTestGenerated.java index 805229298bc..352b3556e92 100644 --- a/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/NativeCodegenBoxTestGenerated.java +++ b/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/NativeCodegenBoxTestGenerated.java @@ -15820,6 +15820,12 @@ public class NativeCodegenBoxTestGenerated extends AbstractNativeCodegenBoxTest runTest("compiler/testData/codegen/box/fileCheck/escape_analysis.kt"); } + @Test + @TestMetadata("filecheck_expected_failure.kt") + public void testFilecheck_expected_failure() throws Exception { + runTest("compiler/testData/codegen/box/fileCheck/filecheck_expected_failure.kt"); + } + @Test @TestMetadata("force_arm_instruction_set.kt") public void testForce_arm_instruction_set() throws Exception { diff --git a/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/NativeCodegenBoxTestNoPLGenerated.java b/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/NativeCodegenBoxTestNoPLGenerated.java index 118855f5abd..7f444178347 100644 --- a/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/NativeCodegenBoxTestNoPLGenerated.java +++ b/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/test/blackbox/NativeCodegenBoxTestNoPLGenerated.java @@ -16199,6 +16199,12 @@ public class NativeCodegenBoxTestNoPLGenerated extends AbstractNativeCodegenBoxT runTest("compiler/testData/codegen/box/fileCheck/escape_analysis.kt"); } + @Test + @TestMetadata("filecheck_expected_failure.kt") + public void testFilecheck_expected_failure() throws Exception { + runTest("compiler/testData/codegen/box/fileCheck/filecheck_expected_failure.kt"); + } + @Test @TestMetadata("force_arm_instruction_set.kt") public void testForce_arm_instruction_set() throws Exception { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/FrameworkTest.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/FrameworkTest.kt index 8b5de04f501..b5457aaee74 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/FrameworkTest.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/FrameworkTest.kt @@ -64,20 +64,17 @@ abstract class FrameworkTestBase : AbstractNativeSimpleTest() { val objCFrameworkCompilation = testCompilationFactory.testCaseToObjCFrameworkCompilation(testCase, testRunSettings) objCFrameworkCompilation.result.assertSuccess() - val (result, outText, errText) = doFileCheck( - testCase.checks.fileCheckMatcher!!, - buildDir.resolve("out.$fileCheckStage.ll").also { assert(it.exists()) } - ) - if (!(result == 0 && errText.isEmpty() && outText.isEmpty())) { - val shortOutText = outText.lines().take(100) - val shortErrText = errText.lines().take(100) - assert(false) { - "FileCheck matching of ${buildDir.resolve("out.$fileCheckStage.ll").also { assert(it.exists()) }.absolutePath}\n" + + val fileCheckDump = buildDir.resolve("out.$fileCheckStage.ll").also { assert(it.exists()) } + val result = doFileCheck(testCase.checks.fileCheckMatcher!!, fileCheckDump) + if (!(result.stdout.isEmpty() && result.stderr.isEmpty())) { + val shortOutText = result.stdout.lines().take(100) + val shortErrText = result.stderr.lines().take(100) + fail("FileCheck matching of ${fileCheckDump.absolutePath}\n" + "with '--check-prefixes ${testCase.checks.fileCheckMatcher.prefixes}'\n" + "failed with result=$result:\n" + shortOutText.joinToString("\n") + "\n" + shortErrText.joinToString("\n") - } + ) } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt index 13c6d6bd5e9..c39ea1b88de 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt @@ -13,8 +13,11 @@ import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.Exec import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExitCode import org.jetbrains.kotlin.konan.test.blackbox.support.settings.configurables import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestReport +import org.jetbrains.kotlin.native.executors.RunProcessResult +import org.jetbrains.kotlin.native.executors.runProcess import org.junit.jupiter.api.Assumptions import java.io.File +import kotlin.time.Duration private fun RunResult.processOutputAsString(output: TestRunCheck.Output) = when (output) { TestRunCheck.Output.STDOUT -> processOutput.stdOut.filteredOutput @@ -79,10 +82,11 @@ internal class ResultHandler( } is TestRunCheck.FileCheckMatcher -> { val fileCheckDump = runResult.testExecutable.executable.fileCheckDump!! - val (result, outText, errText) = doFileCheck(check, fileCheckDump) - if (!(result == 0 && errText.isEmpty() && outText.isEmpty())) { - val shortOutText = outText.lines().take(100) - val shortErrText = errText.lines().take(100) + val result = doFileCheck(check, fileCheckDump) + + if (!(result.stdout.isEmpty() && result.stderr.isEmpty())) { + val shortOutText = result.stdout.lines().take(100) + val shortErrText = result.stderr.lines().take(100) add("FileCheck matching of ${fileCheckDump.absolutePath}\n" + "with '--check-prefixes ${check.prefixes}'\n" + "failed with result=$result:\n" + @@ -147,29 +151,23 @@ internal class ResultHandler( } } -internal fun doFileCheck(check: TestRunCheck.FileCheckMatcher, fileCheckDump: File): Triple { +internal fun doFileCheck(check: TestRunCheck.FileCheckMatcher, fileCheckDump: File): RunProcessResult { val fileCheckExecutable = check.settings.configurables.absoluteLlvmHome + File.separator + "bin" + File.separator + if (SystemInfo.isWindows) "FileCheck.exe" else "FileCheck" require(File(fileCheckExecutable).exists()) { "$fileCheckExecutable does not exist. Make sure Distribution for `settings.configurables` " + "was created using `propertyOverrides` to specify development variant of LLVM instead of user variant." } - val fileCheckOut = File(fileCheckDump.absolutePath + ".out") - val fileCheckErr = File(fileCheckDump.absolutePath + ".err") - - val result = ProcessBuilder( - fileCheckExecutable, - check.testDataFile.absolutePath, - "--input-file", - fileCheckDump.absolutePath, - "--check-prefixes", check.prefixes, - "--allow-deprecated-dag-overlap" // TODO specify it via new test directive for `function_attributes_at_callsite.kt` - ).redirectOutput(fileCheckOut) - .redirectError(fileCheckErr) - .start() - .waitFor() - val outText = fileCheckOut.readText() - val errText = fileCheckErr.readText() - - return Triple(result, outText, errText) + return try { + runProcess( + fileCheckExecutable, + check.testDataFile.absolutePath, + "--input-file", + fileCheckDump.absolutePath, + "--check-prefixes", check.prefixes, + "--allow-deprecated-dag-overlap" // TODO specify it via new test directive for `function_attributes_at_callsite.kt` + ) + } catch (t: Throwable) { + RunProcessResult(Duration.ZERO, "FileCheck utility failed:", t.toString()) + } }