diff --git a/native/native.tests/tests/org/jetbrains/kotlin/generators/tests/GenerateNativeTests.kt b/native/native.tests/tests/org/jetbrains/kotlin/generators/tests/GenerateNativeTests.kt index c17fe746c5f..6ab34e7054f 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/generators/tests/GenerateNativeTests.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/generators/tests/GenerateNativeTests.kt @@ -98,7 +98,12 @@ fun main() { private inline fun provider() = annotation(T::class.java) -private fun debugOnly() = annotation(EnforcedProperty::class.java, Pair("property", ClassLevelProperty.OPTIMIZATION_MODE), Pair("propertyValue", "DEBUG")) +private fun debugOnly() = annotation( + EnforcedProperty::class.java, + "property" to ClassLevelProperty.OPTIMIZATION_MODE, + "propertyValue" to "DEBUG" +) + private fun codegen() = annotation(Tag::class.java, "codegen") private fun debugger() = annotation(Tag::class.java, "debugger") private fun infrastructure() = annotation(Tag::class.java, "infrastructure") diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt index 00b9cd21a8a..7dcccf84e2d 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt @@ -80,7 +80,7 @@ private object NativeTestSupport { nativeHome, computeNativeClassLoader(), computeBaseDirs(), - DebugUtils() + LLDB(nativeHome) ) } as TestProcessSettings diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt index 30aaf4b7492..f34469f4a98 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt @@ -16,13 +16,14 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.LLDB_TRACE import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.TEST_RUNNER import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.OutputDataFile -import org.jetbrains.kotlin.konan.blackboxtest.support.util.LldbSessionSpecification +import org.jetbrains.kotlin.konan.blackboxtest.support.util.LLDBSessionSpec import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer import org.jetbrains.kotlin.test.directives.model.StringDirective import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail import org.jetbrains.kotlin.test.services.impl.RegisteredDirectivesParser +import org.junit.jupiter.api.Assertions import java.io.File internal object TestDirectives : SimpleDirectivesContainer() { @@ -214,10 +215,14 @@ internal fun parseEntryPoint(registeredDirectives: RegisteredDirectives, locatio return entryPoint } -internal fun parseLLDBSpec(baseDir:File, registeredDirectives: RegisteredDirectives, location: Location): LldbSessionSpecification { +internal fun parseLLDBSpec(baseDir: File, registeredDirectives: RegisteredDirectives, location: Location): LLDBSessionSpec { val specFile = parseFileBasedDirective(baseDir, LLDB_TRACE, registeredDirectives, location) - ?: fail { "$location: A lldb session specification must be provided" } - return LldbSessionSpecification.parse(specFile.readText()) + ?: fail { "$location: An LLDB session specification must be provided" } + return try { + LLDBSessionSpec.parse(specFile.readText()) + } catch (e: Exception) { + Assertions.fail("$location: Cannot parse LLDB session specification: " + e.message, e) + } } internal fun parseModule(parsedDirective: RegisteredDirectivesParser.ParsedDirective, location: Location): TestModule.Exclusive { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt index 74b4d8b1237..1d4281cf1e7 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt @@ -48,9 +48,9 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { val testCases = includedTestDataFiles.map { testDataFile -> createTestCase(testDataFile, settings) } - if (!settings.get().lldbIsAvailable) { - val lldbTests = testCases.filter { it.kind == TestKind.STANDALONE_LLDB } - lldbTests.mapTo(disabledTestCaseIds) { it.id } + val lldbTestCases = testCases.filter { it.kind == TestKind.STANDALONE_LLDB } + if (lldbTestCases.isNotEmpty() && !settings.get().isAvailable) { + lldbTestCases.mapTo(disabledTestCaseIds) { it.id } } TestCaseGroup.Default(disabledTestCaseIds, testCases) @@ -205,11 +205,13 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { fixPackageNames(testModules.values, nominalPackageName, testDataFile) } - val lldbSpec = if (testKind == TestKind.STANDALONE_LLDB) parseLLDBSpec( - baseDir = testDataFile.parentFile, - registeredDirectives, - location - ) else null + val lldbSpec = if (testKind == TestKind.STANDALONE_LLDB) + parseLLDBSpec( + baseDir = testDataFile.parentFile, + registeredDirectives, + location + ) + else null val testCase = TestCase( id = TestCaseId.TestDataFile(testDataFile), @@ -221,7 +223,7 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { computeExecutionTimeoutCheck(settings, expectedTimeoutFailure), computeExitCodeCheck(testKind, registeredDirectives, location), computeOutputDataFileCheck(testDataFile, registeredDirectives, location), - lldbSpec?.let { OutputMatcher(it::matchOutput) } + lldbSpec?.let { OutputMatcher { output -> lldbSpec.checkLLDBOutput(output, settings.get()) } } ), extras = when (testKind) { TestKind.STANDALONE_NO_TR -> { @@ -236,7 +238,7 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { TestKind.STANDALONE_LLDB -> { NoTestRunnerExtras( entryPoint = parseEntryPoint(registeredDirectives, location), - arguments = lldbSpec!!.generateCLIArguments(settings.get().lldbPrettyPrinters) + arguments = lldbSpec!!.generateCLIArguments(settings.get().prettyPrinters) ) } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt index c91b365faef..30b29098364 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt @@ -114,7 +114,7 @@ internal abstract class AbstractLocalProcessRunner(private val checks: TestRu } catch (t: Throwable) { if (t is Exception || t is AssertionError) { fail( - getLoggedRun().withErrorMessage("Tested process output has not passed validation:\n\n" + t.message), + getLoggedRun().withErrorMessage("Tested process output has not passed validation: " + t.message), t ) } else { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/BaseTestRunProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/BaseTestRunProvider.kt index ef40f20a510..79c868c4e9a 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/BaseTestRunProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/BaseTestRunProvider.kt @@ -34,7 +34,6 @@ internal open class BaseTestRunProvider { // Note: TestRunParameter.WithLLDB adds program arguments and would therefore conflict // with other TestRunParameters that do the same (such as WithTCTestLogger). add(TestRunParameter.WithLLDB(testCase.extras().arguments)) - addIfNotNull(testCase.extras().inputDataFile?.let(TestRunParameter::WithInputData)) } TestKind.STANDALONE_NO_TR -> { assertTrue(testName == null) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRun.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRun.kt index 0776825b527..69c8f87fe49 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRun.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRun.kt @@ -8,10 +8,7 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner import org.jetbrains.kotlin.konan.blackboxtest.support.* import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationArtifact.Executable import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationResult.Success -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeHome -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings import org.jetbrains.kotlin.konan.blackboxtest.support.util.DumpedTestListing -import org.jetbrains.kotlin.konan.blackboxtest.support.util.LldbSessionSpecification import org.jetbrains.kotlin.konan.blackboxtest.support.util.startsWith import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertFalse import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunProvider.kt index e4d0147d010..b23295429d4 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunProvider.kt @@ -15,7 +15,6 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilati import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationFactory import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationResult.Companion.assertSuccess import org.jetbrains.kotlin.konan.blackboxtest.support.group.TestCaseGroupProvider -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.DebugUtils import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings import org.jetbrains.kotlin.konan.blackboxtest.support.util.ThreadSafeCache import org.jetbrains.kotlin.konan.blackboxtest.support.util.TreeNode @@ -104,13 +103,7 @@ internal class TestRunProvider( fun createTestRun(testRunName: String, testName: TestName?) = createTestRun(testCase, executable, testRunName, testName) when (testCase.kind) { - TestKind.STANDALONE_LLDB -> { - if (!settings.get().lldbIsAvailable) TreeNode.oneLevel() - val testRunName = testCase.extras().entryPoint.substringAfterLast('.') - val testRun = createTestRun(testRunName, testName = null) - TreeNode.oneLevel(testRun) - } - TestKind.STANDALONE_NO_TR -> { + TestKind.STANDALONE_NO_TR, TestKind.STANDALONE_LLDB -> { val testRunName = testCase.extras().entryPoint.substringAfterLast('.') val testRun = createTestRun(testRunName, testName = null) TreeNode.oneLevel(testRun) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt index 3f5d49e642b..1ecdba4b986 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt @@ -30,14 +30,15 @@ internal class KotlinNativeTargets(val testTarget: KonanTarget, val hostTarget: internal class KotlinNativeHome(val dir: File) { val librariesDir: File = dir.resolve("klib") val stdlibFile: File = librariesDir.resolve("common/stdlib") - val lldbPrettyPrinters: File = dir.resolve("tools/konan_lldb.py") val properties: Properties by lazy { dir.resolve("konan/konan.properties").inputStream().use { Properties().apply { load(it) } } } } -internal class DebugUtils { - val lldbIsAvailable: Boolean by lazy { +internal class LLDB(nativeHome: KotlinNativeHome) { + val prettyPrinters: File = nativeHome.dir.resolve("tools/konan_lldb.py") + + val isAvailable: Boolean by lazy { try { val exitCode = ProcessBuilder("lldb", "-version").start().waitFor() exitCode == 0 diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/LLDBSessionSpec.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/LLDBSessionSpec.kt new file mode 100644 index 00000000000..ad163d33122 --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/LLDBSessionSpec.kt @@ -0,0 +1,139 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.blackboxtest.support.util + +import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeTargets +import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue +import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail +import java.io.File + +private class Step(val command: String, val body: List) { + companion object { + fun parse(block: String, expectedPrefix: String): Step { + val lines = block.lines() + + val commandWithPrefix = lines.first() + assertTrue(commandWithPrefix.startsWith(expectedPrefix)) { + "The command should start with $expectedPrefix. Got: $commandWithPrefix" + } + + val command = commandWithPrefix.removePrefix(expectedPrefix).trimStart() + val body = lines.drop(1).filterNot(String::isBlank) + + return Step(command, body) + } + } +} + +internal class LLDBSessionSpec private constructor(private val expectedSteps: List) { + fun generateCLIArguments(prettyPrinters: File): List = buildList { + this += "-b" + this += "-o" + this += "command script import ${prettyPrinters.absolutePath}" + expectedSteps.forEach { step -> + this += "-o" + this += step.command + } + } + + fun checkLLDBOutput(output: String, nativeTargets: KotlinNativeTargets): Boolean { + val blocks = output.split(LLDB_OUTPUT_SEPARATOR).filterNot(String::isBlank) + + val meaningfulBlocks = if (nativeTargets.testTarget == nativeTargets.hostTarget) { + // TODO: why are these two leading blocks only checked for the host target? + + val createTargetBlock = blocks.getOrElse(0) { "" } + assertTrue(createTargetBlock.startsWith("(lldb) target create")) { + "Missing block \"target create\". Got: $createTargetBlock" + } + + val commandScriptBlock = blocks.getOrElse(1) { "" } + assertTrue(commandScriptBlock.startsWith("(lldb) command script import")) { + "Missing block \"command script import\". Got: $commandScriptBlock" + } + + blocks.drop(2) + } else { + blocks.drop(2).dropLast(1) + } + + val recordedSteps = meaningfulBlocks.map { block -> Step.parse(block, LLDB_COMMAND_PREFIX) } + assertTrue(expectedSteps.size == recordedSteps.size) { + """ + The number of responses do not match the number of commands. + - Commands (${expectedSteps.size}): ${expectedSteps.map { it.command }} + - Responses (${recordedSteps.size}): ${recordedSteps.map { it.command }} + """.trimIndent() + } + + for ((expectedStep, recordedStep) in expectedSteps.zip(recordedSteps)) { + assertTrue(expectedStep.command == recordedStep.command) { + """ + Wrong command in response. + - Expected: ${expectedStep.command} + - Actual: ${recordedStep.command} + """.trimIndent() + } + + val mismatch = findMismatch(expectedStep.body, recordedStep.body) + if (mismatch != null) { + fail { + buildString { + appendLine("Wrong LLDB output.") + append("- Command: ").appendLine(expectedStep.command) + append("- Expected (pattern): ").appendLine(mismatch) + appendLine("- Actual:") + recordedStep.body.joinTo(this, separator = "\n") + } + } + } + } + return true + } + + private fun findMismatch(patterns: List, actualLines: List): String? { + val indices = mutableListOf() + for (pattern in patterns) { + val idx = actualLines.indexOfFirst { match(pattern, it) } + if (idx == -1) { + return pattern + } + indices += idx + } + assertTrue(indices == indices.sorted()) + return null + } + + private fun match(pattern: String, line: String): Boolean { + val chunks = pattern.split(LINE_WILDCARD) + .filter { it.isNotBlank() } + .map { it.trim() } + assertTrue(chunks.isNotEmpty()) + val trimmedLine = line.trim() + + val indices = chunks.map { trimmedLine.indexOf(it) } + if (indices.any { it == -1 } || indices != indices.sorted()) return false + if (!(trimmedLine.startsWith(chunks.first()) || pattern.startsWith("[..]"))) return false + if (!(trimmedLine.endsWith(chunks.last()) || pattern.endsWith("[..]"))) return false + return true + } + + companion object { + private const val LLDB_COMMAND_PREFIX = "(lldb)" + private const val SPEC_COMMAND_PREFIX = ">" + + private val LLDB_OUTPUT_SEPARATOR = """(?=\(lldb\))""".toRegex() + private val SPEC_BLOCK_SEPARATOR = "(?=^>)".toRegex(RegexOption.MULTILINE) + + private val LINE_WILDCARD = """\s*\[\.\.]\s*""".toRegex() + + fun parse(lldbSpec: String): LLDBSessionSpec = LLDBSessionSpec( + lldbSpec.split(SPEC_BLOCK_SEPARATOR) + .filterNot(String::isBlank) + .map { block -> Step.parse(block, SPEC_COMMAND_PREFIX) } + ) + } +} diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/LldbSessionSpecification.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/LldbSessionSpecification.kt deleted file mode 100644 index 81564e59698..00000000000 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/LldbSessionSpecification.kt +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.konan.blackboxtest.support.util - -import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail -import java.io.File - -private class Step(val command: String, val body: List) - -class LldbSessionSpecification private constructor( - private val steps: List, -) { - - - fun generateCLIArguments(pp: File): List { - val args = mutableListOf("-b", "-o", "command script import ${pp.absolutePath}") - args.addAll(steps.flatMap { listOf("-o", it.command) }) - return args - } - - // TODO: Re-introduce this when we add support for iOS simulator based testing. - fun targetIsHost() = true - - fun matchOutput(output: String): Boolean { - val blocks = output.split(lldbOutputCommand).filterNot(String::isEmpty) - if (targetIsHost()) { - check(blocks[0].startsWith("(lldb) target create")) { "Missing block \"target create\". Got: ${blocks[0]}" } - check(blocks[1].startsWith("(lldb) command script import")) { - "Missing block \"command script import\". Got: ${blocks[1]}" - } - } - val responses = if (targetIsHost()) - blocks.drop(2) - else - blocks.drop(2).dropLast(1) - - val recordedSteps = responses.map { Step(it.lines().first(), it.lines().drop(1)) } - - if (recordedSteps.size != steps.size) { - val message = """ - |Responses do not match commands. - | - |COMMANDS: |${steps.map { it.command }} (${steps.size}) - |RESPONSES: |${recordedSteps.map { it.command }} (${recordedSteps.size}) - """.trimMargin() - fail { message } - } - - steps.zip(recordedSteps).all { (step, recordedStep) -> recordedStep.command == "(lldb) ${step.command}" } - - for ((step, recordedStep) in steps.zip(recordedSteps)) { - check(recordedStep.command == "(lldb) ${step.command}") { - "Wrong command in response. Expected: (lldb) ${step.command}. Got: ${recordedStep.command}" - } - val mismatch = findMismatch(step.body, recordedStep.body) - if (mismatch != null) { - val message = """ - |Wrong LLDB output. - | - |COMMAND: ${step.command} - |PATTERN: $mismatch - |OUTPUT: - |${recordedStep.body.joinToString("\n")} - """.trimMargin() - fail { message } - } - } - return true - } - - private fun findMismatch(patterns: List, actualLines: List): String? { - val indices = mutableListOf() - for (pattern in patterns) { - val idx = actualLines.indexOfFirst { match(pattern, it) } - if (idx == -1) { - return pattern - } - indices += idx - } - check(indices == indices.sorted()) - return null - } - - private fun match(pattern: String, line: String): Boolean { - val chunks = pattern.split(wildcard) - .filter { it.isNotBlank() } - .map { it.trim() } - check(chunks.isNotEmpty()) - val trimmedLine = line.trim() - - val indices = chunks.map { trimmedLine.indexOf(it) } - if (indices.any { it == -1 } || indices != indices.sorted()) return false - if (!(trimmedLine.startsWith(chunks.first()) || pattern.startsWith("[..]"))) return false - if (!(trimmedLine.endsWith(chunks.last()) || pattern.endsWith("[..]"))) return false - return true - } - - companion object { - val lldbOutputCommand = """(?=\(lldb\))""".toRegex() - val wildcard = """\s*\[\.\.]\s*""".toRegex() - val separator = "(?=^>)".toRegex(RegexOption.MULTILINE) - - fun parse(spec: String): LldbSessionSpecification { - val blocks = spec.trimIndent() - .split(separator) - .filterNot(String::isEmpty) - for (cmd in blocks) { - check(cmd.startsWith(">")) { "Invalid lldb session specification: $cmd" } - } - val steps = blocks.map { - Step(it.lines().first().substring(1).trim(), it.lines().drop(1).filter { it.isNotBlank() }) - } - return LldbSessionSpecification(steps) - } - } -}