[K/N] Migrate LLDB test to blackboxtest

This change only introduces a single sample test together with the necessary plumbing for basic LLDB testing. Migrating the rest of the tests over and introducing more complex interop setups will be tackled as a follow-up.
2022-08-12 08:21:15 +02:00
parent d61c3d1add
commit 842a66c3de
43 changed files with 708 additions and 18 deletions
@@ -8,6 +8,8 @@ package org.jetbrains.kotlin.generators.tests
 import org.jetbrains.kotlin.generators.generateTestGroupSuiteWithJUnit5
 import org.jetbrains.kotlin.generators.model.annotation
 import org.jetbrains.kotlin.konan.blackboxtest.*
+import org.jetbrains.kotlin.konan.blackboxtest.support.ClassLevelProperty
+import org.jetbrains.kotlin.konan.blackboxtest.support.EnforcedProperty
 import org.jetbrains.kotlin.konan.blackboxtest.support.group.UseExtTestCaseGroupProvider
 import org.jetbrains.kotlin.konan.blackboxtest.support.group.UseStandardTestCaseGroupProvider
 import org.jetbrains.kotlin.test.TargetBackend
@@ -81,10 +83,22 @@ fun main() {
                model("CInterop/KT-39120/defs", pattern = "^([^_](.+))$", recursive = false)
            }
        }
+
+        // LLDB integration tests.
+        testGroup("native/native.tests/tests-gen", "native/native.tests/testData") {
+            testClass<AbstractNativeBlackBoxTest>(
+                suiteTestClassName = "LldbTestGenerated",
+                annotations = listOf(debugger(), provider<UseStandardTestCaseGroupProvider>(), debugOnly())
+            ) {
+                model("lldb")
+            }
+        }
    }
 }

 private inline fun <reified T : Annotation> provider() = annotation(T::class.java)

+private fun debugOnly() = annotation(EnforcedProperty::class.java, Pair("property", ClassLevelProperty.OPTIMIZATION_MODE), Pair("propertyValue", "DEBUG"))
 private fun codegen() = annotation(Tag::class.java, "codegen")
+private fun debugger() = annotation(Tag::class.java, "debugger")
 private fun infrastructure() = annotation(Tag::class.java, "infrastructure")
@@ -79,7 +79,8 @@ private object NativeTestSupport {
            TestProcessSettings(
                nativeHome,
                computeNativeClassLoader(),
-                computeBaseDirs()
+                computeBaseDirs(),
+                DebugUtils()
            )
        } as TestProcessSettings

@@ -179,12 +179,12 @@ internal class TestCase(
    val extras: Extras
 ) {
    sealed interface Extras
-    class NoTestRunnerExtras(val entryPoint: String, val inputDataFile: File?) : Extras
+    class NoTestRunnerExtras(val entryPoint: String, val inputDataFile: File? = null, val arguments: List<String> = emptyList()) : Extras
    class WithTestRunnerExtras(val runnerType: TestRunnerType, val ignoredTests: Set<String> = emptySet()) : Extras

    init {
        when (kind) {
-            TestKind.STANDALONE_NO_TR -> assertTrue(extras is NoTestRunnerExtras)
+            TestKind.STANDALONE_NO_TR, TestKind.STANDALONE_LLDB -> assertTrue(extras is NoTestRunnerExtras)
            TestKind.REGULAR, TestKind.STANDALONE -> assertTrue(extras is WithTestRunnerExtras)
        }
    }
@@ -12,9 +12,11 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.INPUT_DATA
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.KIND
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.OUTPUT_DATA_FILE
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.EXPECTED_TIMEOUT_FAILURE
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.LLDB_TRACE
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.TEST_RUNNER
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.OutputDataFile
+import org.jetbrains.kotlin.konan.blackboxtest.support.util.LldbSessionSpecification
 import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
 import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer
 import org.jetbrains.kotlin.test.directives.model.StringDirective
@@ -26,7 +28,7 @@ import java.io.File
 internal object TestDirectives : SimpleDirectivesContainer() {
    val KIND by enumDirective<TestKind>(
        description = """
-            Usage: // KIND: [REGULAR, STANDALONE, STANDALONE_NO_TR]
+            Usage: // KIND: [REGULAR, STANDALONE, STANDALONE_NO_TR, STANDALONE_LLDB]
            Declares the kind of the test:

            - REGULAR (the default) - include this test into the shared test binary.
@@ -37,6 +39,8 @@ internal object TestDirectives : SimpleDirectivesContainer() {

            - STANDALONE_NO_TR - compile the test to a separate binary that is supposed to have main entry point.
              The entry point can be customized Note that @kotlin.Test annotations are ignored.
+
+            - STANDALONE_LLDB - compile the test to a separate binary and debug with LLDB.
        """.trimIndent()
    )

@@ -51,7 +55,7 @@ internal object TestDirectives : SimpleDirectivesContainer() {
    val ENTRY_POINT by stringDirective(
        description = """
            Specify custom program entry point. The default entry point is `main` function in the root package.
-            Note that this directive makes sense only in combination with // KIND: STANDALONE_NO_TR
+            Note that this directive makes sense only in combination with // KIND: STANDALONE_NO_TR or // KIND: STANDALONE_LLDB
        """.trimIndent()
    )

@@ -122,12 +126,19 @@ internal object TestDirectives : SimpleDirectivesContainer() {
    val FREE_COMPILER_ARGS by stringDirective(
        description = "Specify free compiler arguments for Kotlin/Native compiler"
    )
+
+    val LLDB_TRACE by stringDirective(
+        description = """
+            Specify a filename containing the LLDB commands and the patterns that
+             the output should match""".trimIndent(),
+    )
 }

 internal enum class TestKind {
    REGULAR,
    STANDALONE,
-    STANDALONE_NO_TR;
+    STANDALONE_NO_TR,
+    STANDALONE_LLDB;
 }

 internal enum class TestRunnerType {
@@ -203,6 +214,12 @@ internal fun parseEntryPoint(registeredDirectives: RegisteredDirectives, locatio
    return entryPoint
 }

+internal fun parseLLDBSpec(baseDir:File, registeredDirectives: RegisteredDirectives, location: Location): LldbSessionSpecification {
+    val specFile = parseFileBasedDirective(baseDir, LLDB_TRACE, registeredDirectives, location)
+        ?: fail { "$location: A lldb session specification must be provided" }
+    return LldbSessionSpecification.parse(specFile.readText())
+}
+
 internal fun parseModule(parsedDirective: RegisteredDirectivesParser.ParsedDirective, location: Location): TestModule.Exclusive {
    val module = parsedDirective.values.singleOrNull()?.toString()?.let(TEST_MODULE_REGEX::matchEntire)?.let { match ->
        val name = match.groupValues[1]
@@ -48,6 +48,11 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {

            val testCases = includedTestDataFiles.map { testDataFile -> createTestCase(testDataFile, settings) }

+            if (!settings.get<DebugUtils>().lldbIsAvailable) {
+                val lldbTests = testCases.filter { it.kind == TestKind.STANDALONE_LLDB }
+                lldbTests.mapTo(disabledTestCaseIds) { it.id }
+            }
+
            TestCaseGroup.Default(disabledTestCaseIds, testCases)
        }
    }
@@ -200,6 +205,12 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
            fixPackageNames(testModules.values, nominalPackageName, testDataFile)
        }

+        val lldbSpec = if (testKind == TestKind.STANDALONE_LLDB) parseLLDBSpec(
+            baseDir = testDataFile.parentFile,
+            registeredDirectives,
+            location
+        ) else null
+
        val testCase = TestCase(
            id = TestCaseId.TestDataFile(testDataFile),
            kind = testKind,
@@ -209,15 +220,26 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
            checks = TestRunChecks(
                computeExecutionTimeoutCheck(settings, expectedTimeoutFailure),
                computeExitCodeCheck(testKind, registeredDirectives, location),
-                computeOutputDataFileCheck(testDataFile, registeredDirectives, location)
+                computeOutputDataFileCheck(testDataFile, registeredDirectives, location),
+                lldbSpec?.let { OutputMatcher(it::matchOutput) }
            ),
-            extras = if (testKind == TestKind.STANDALONE_NO_TR)
-                NoTestRunnerExtras(
-                    entryPoint = parseEntryPoint(registeredDirectives, location),
-                    inputDataFile = parseInputDataFile(baseDir = testDataFile.parentFile, registeredDirectives, location)
-                )
-            else
-                WithTestRunnerExtras(runnerType = parseTestRunner(registeredDirectives, location))
+            extras = when (testKind) {
+                TestKind.STANDALONE_NO_TR -> {
+                    NoTestRunnerExtras(
+                        entryPoint = parseEntryPoint(registeredDirectives, location),
+                        inputDataFile = parseInputDataFile(baseDir = testDataFile.parentFile, registeredDirectives, location)
+                    )
+                }
+                TestKind.REGULAR, TestKind.STANDALONE -> {
+                    WithTestRunnerExtras(runnerType = parseTestRunner(registeredDirectives, location))
+                }
+                TestKind.STANDALONE_LLDB -> {
+                    NoTestRunnerExtras(
+                        entryPoint = parseEntryPoint(registeredDirectives, location),
+                        arguments = lldbSpec!!.generateCLIArguments(settings.get<KotlinNativeHome>().lldbPrettyPrinters)
+                    )
+                }
+            }
        )
        testCase.initialize(
            givenModules = settings.get<CustomKlibs>().klibs.mapToSet(TestModule::Given),
@@ -12,6 +12,7 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.Execu
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExitCode
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.UnfilteredProcessOutput.Companion.launchReader
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter
+import org.junit.jupiter.api.Assertions.fail
 import java.io.ByteArrayOutputStream
 import kotlin.time.*

@@ -105,6 +106,22 @@ internal abstract class AbstractLocalProcessRunner<R>(private val checks: TestRu
                            "Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below."
                        }
                    }
+                    is TestRunCheck.OutputMatcher -> {
+                        try {
+                            verifyExpectation(check.match(runResult.processOutput.stdOut.filteredOutput)) {
+                                "Tested process output has not passed validation."
+                            }
+                        } catch (t: Throwable) {
+                            if (t is Exception || t is AssertionError) {
+                                fail<Nothing>(
+                                    getLoggedRun().withErrorMessage("Tested process output has not passed validation:\n\n" + t.message),
+                                    t
+                                )
+                            } else {
+                                throw t
+                            }
+                        }
+                    }
                }
            }

@@ -29,6 +29,13 @@ internal open class BaseTestRunProvider {
        addIfNotNull(testCase.checks.outputDataFile?.file?.let(TestRunParameter::WithExpectedOutputData))

        when (testCase.kind) {
+            TestKind.STANDALONE_LLDB -> {
+                assertTrue(testName == null)
+                // Note: TestRunParameter.WithLLDB adds program arguments and would therefore conflict
+                // with other TestRunParameters that do the same (such as WithTCTestLogger).
+                add(TestRunParameter.WithLLDB(testCase.extras<NoTestRunnerExtras>().arguments))
+                addIfNotNull(testCase.extras<NoTestRunnerExtras>().inputDataFile?.let(TestRunParameter::WithInputData))
+            }
            TestKind.STANDALONE_NO_TR -> {
                assertTrue(testName == null)
                addIfNotNull(testCase.extras<NoTestRunnerExtras>().inputDataFile?.let(TestRunParameter::WithInputData))
@@ -8,7 +8,10 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner
 import org.jetbrains.kotlin.konan.blackboxtest.support.*
 import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationArtifact.Executable
 import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationResult.Success
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeHome
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.DumpedTestListing
+import org.jetbrains.kotlin.konan.blackboxtest.support.util.LldbSessionSpecification
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.startsWith
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertFalse
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
@@ -96,6 +99,13 @@ internal sealed interface TestRunParameter {
        override fun applyTo(programArgs: MutableList<String>) = Unit
    }

+    class WithLLDB(val commands: List<String>) : TestRunParameter {
+        override fun applyTo(programArgs: MutableList<String>) {
+            programArgs.add(0, "lldb")
+            programArgs.addAll(commands)
+        }
+    }
+
    // Currently, used only for logging the data.
    class WithExpectedOutputData(val expectedOutputDataFile: File) : TestRunParameter {
        override fun applyTo(programArgs: MutableList<String>) = Unit
@@ -22,18 +22,22 @@ internal sealed interface TestRunCheck {
    }

    class OutputDataFile(val file: File) : TestRunCheck
+
+    class OutputMatcher(val match: (String) -> Boolean): TestRunCheck
 }

 internal class TestRunChecks(
    val executionTimeoutCheck: ExecutionTimeout,
    private val exitCodeCheck: ExitCode,
-    val outputDataFile: OutputDataFile?
+    val outputDataFile: OutputDataFile?,
+    val outputMatcher: OutputMatcher?
 ) : Iterable<TestRunCheck> {

    override fun iterator() = iterator {
        yield(executionTimeoutCheck)
        yield(exitCodeCheck)
        yieldIfNotNull(outputDataFile)
+        yieldIfNotNull(outputMatcher)
    }

    companion object {
@@ -42,7 +46,8 @@ internal class TestRunChecks(
        fun Default(timeout: Duration) = TestRunChecks(
            executionTimeoutCheck = ExecutionTimeout.ShouldNotExceed(timeout),
            exitCodeCheck = ExitCode.Expected(0),
-            outputDataFile = null
+            outputDataFile = null,
+            outputMatcher = null
        )
    }
 }
@@ -15,6 +15,7 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilati
 import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationFactory
 import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationResult.Companion.assertSuccess
 import org.jetbrains.kotlin.konan.blackboxtest.support.group.TestCaseGroupProvider
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.DebugUtils
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.ThreadSafeCache
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.TreeNode
@@ -103,6 +104,12 @@ internal class TestRunProvider(
        fun createTestRun(testRunName: String, testName: TestName?) = createTestRun(testCase, executable, testRunName, testName)

        when (testCase.kind) {
+            TestKind.STANDALONE_LLDB -> {
+                if (!settings.get<DebugUtils>().lldbIsAvailable) TreeNode.oneLevel<TestRun>()
+                val testRunName = testCase.extras<NoTestRunnerExtras>().entryPoint.substringAfterLast('.')
+                val testRun = createTestRun(testRunName, testName = null)
+                TreeNode.oneLevel(testRun)
+            }
            TestKind.STANDALONE_NO_TR -> {
                val testRunName = testCase.extras<NoTestRunnerExtras>().entryPoint.substringAfterLast('.')
                val testRun = createTestRun(testRunName, testName = null)
@@ -130,7 +137,7 @@ internal class TestRunProvider(
        val testCase = testCaseGroup.getByName(testCaseId) ?: fail { "No test case for $testCaseId" }

        val testCompilation = when (testCase.kind) {
-            TestKind.STANDALONE, TestKind.STANDALONE_NO_TR -> {
+            TestKind.STANDALONE, TestKind.STANDALONE_NO_TR, TestKind.STANDALONE_LLDB -> {
                // Create a separate compilation for each standalone test case.
                cachedCompilations.computeIfAbsent(
                    TestCompilationCacheKey.Standalone(testCaseId)
@@ -14,6 +14,7 @@ import org.jetbrains.kotlin.konan.target.Distribution
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
 import java.io.File
+import java.io.IOException
 import java.util.*
 import kotlin.time.Duration
 import kotlin.time.Duration.Companion.seconds
@@ -29,12 +30,23 @@ internal class KotlinNativeTargets(val testTarget: KonanTarget, val hostTarget:
 internal class KotlinNativeHome(val dir: File) {
    val librariesDir: File = dir.resolve("klib")
    val stdlibFile: File = librariesDir.resolve("common/stdlib")
-
+    val lldbPrettyPrinters: File = dir.resolve("tools/konan_lldb.py")
    val properties: Properties by lazy {
        dir.resolve("konan/konan.properties").inputStream().use { Properties().apply { load(it) } }
    }
 }

+internal class DebugUtils {
+    val lldbIsAvailable: Boolean by lazy {
+        try {
+            val exitCode = ProcessBuilder("lldb", "-version").start().waitFor()
+            exitCode == 0
+        } catch (e: IOException) {
+            false
+        }
+    }
+}
+
 /**
 * Lazy-initialized class loader with the Kotlin/Native embedded compiler.
 */
@@ -0,0 +1,119 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.konan.blackboxtest.support.util
+
+import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
+import java.io.File
+
+private class Step(val command: String, val body: List<String>)
+
+class LldbSessionSpecification private constructor(
+    private val steps: List<Step>,
+) {
+
+
+    fun generateCLIArguments(pp: File): List<String> {
+        val args = mutableListOf("-b", "-o", "command script import ${pp.absolutePath}")
+        args.addAll(steps.flatMap { listOf("-o", it.command) })
+        return args
+    }
+
+    // TODO: Re-introduce this when we add support for iOS simulator based testing.
+    fun targetIsHost() = true
+
+    fun matchOutput(output: String): Boolean {
+        val blocks = output.split(lldbOutputCommand).filterNot(String::isEmpty)
+        if (targetIsHost()) {
+            check(blocks[0].startsWith("(lldb) target create")) { "Missing block \"target create\". Got: ${blocks[0]}" }
+            check(blocks[1].startsWith("(lldb) command script import")) {
+                "Missing block \"command script import\". Got: ${blocks[1]}"
+            }
+        }
+        val responses = if (targetIsHost())
+            blocks.drop(2)
+        else
+            blocks.drop(2).dropLast(1)
+
+        val recordedSteps = responses.map { Step(it.lines().first(), it.lines().drop(1)) }
+
+        if (recordedSteps.size != steps.size) {
+            val message = """
+                |Responses do not match commands.
+                |
+                |COMMANDS: |${steps.map { it.command }} (${steps.size})
+                |RESPONSES: |${recordedSteps.map { it.command }} (${recordedSteps.size})
+            """.trimMargin()
+            fail { message }
+        }
+
+        steps.zip(recordedSteps).all { (step, recordedStep) -> recordedStep.command == "(lldb) ${step.command}" }
+
+        for ((step, recordedStep) in steps.zip(recordedSteps)) {
+            check(recordedStep.command == "(lldb) ${step.command}") {
+                "Wrong command in response. Expected: (lldb) ${step.command}. Got: ${recordedStep.command}"
+            }
+            val mismatch = findMismatch(step.body, recordedStep.body)
+            if (mismatch != null) {
+                val message = """
+                    |Wrong LLDB output.
+                    |
+                    |COMMAND: ${step.command}
+                    |PATTERN: $mismatch
+                    |OUTPUT:
+                    |${recordedStep.body.joinToString("\n")}
+                """.trimMargin()
+                fail { message }
+            }
+        }
+        return true
+    }
+
+    private fun findMismatch(patterns: List<String>, actualLines: List<String>): String? {
+        val indices = mutableListOf<Int>()
+        for (pattern in patterns) {
+            val idx = actualLines.indexOfFirst { match(pattern, it) }
+            if (idx == -1) {
+                return pattern
+            }
+            indices += idx
+        }
+        check(indices == indices.sorted())
+        return null
+    }
+
+    private fun match(pattern: String, line: String): Boolean {
+        val chunks = pattern.split(wildcard)
+            .filter { it.isNotBlank() }
+            .map { it.trim() }
+        check(chunks.isNotEmpty())
+        val trimmedLine = line.trim()
+
+        val indices = chunks.map { trimmedLine.indexOf(it) }
+        if (indices.any { it == -1 } || indices != indices.sorted()) return false
+        if (!(trimmedLine.startsWith(chunks.first()) || pattern.startsWith("[..]"))) return false
+        if (!(trimmedLine.endsWith(chunks.last()) || pattern.endsWith("[..]"))) return false
+        return true
+    }
+
+    companion object {
+        val lldbOutputCommand = """(?=\(lldb\))""".toRegex()
+        val wildcard = """\s*\[\.\.]\s*""".toRegex()
+        val separator = "(?=^>)".toRegex(RegexOption.MULTILINE)
+
+        fun parse(spec: String): LldbSessionSpecification {
+            val blocks = spec.trimIndent()
+                .split(separator)
+                .filterNot(String::isEmpty)
+            for (cmd in blocks) {
+                check(cmd.startsWith(">")) { "Invalid lldb session specification: $cmd" }
+            }
+            val steps = blocks.map {
+                Step(it.lines().first().substring(1).trim(), it.lines().drop(1).filter { it.isNotBlank() })
+            }
+            return LldbSessionSpecification(steps)
+        }
+    }
+}