From 2b6c284ff8695e5abd2f8b425786c1a1332ad443 Mon Sep 17 00:00:00 2001
From: Aleksey Kladov <aleksey.kladov@gmail.com>
Date: Tue, 17 Oct 2017 16:50:06 +0300
Subject: [PATCH] [compiletest]: add JUnit4-based infrastructure for testing
 compiler

---
 backend.native/compiletest/build.gradle       | 19 ++++
 .../kotlin/compiletest/DistProperties.kt      | 17 ++++
 .../jetbrains/kotlin/compiletest/Driver.kt    | 71 +++++++++++++
 .../jetbrains/kotlin/compiletest/Matchers.kt  | 99 +++++++++++++++++++
 .../jetbrains/kotlin/compiletest/LldbTests.kt | 56 +++++++++++
 settings.gradle                               |  1 +
 6 files changed, 263 insertions(+)
 create mode 100644 backend.native/compiletest/build.gradle
 create mode 100644 backend.native/compiletest/src/main/kotlin/org/jetbrains/kotlin/compiletest/DistProperties.kt
 create mode 100644 backend.native/compiletest/src/main/kotlin/org/jetbrains/kotlin/compiletest/Driver.kt
 create mode 100644 backend.native/compiletest/src/main/kotlin/org/jetbrains/kotlin/compiletest/Matchers.kt
 create mode 100644 backend.native/compiletest/src/test/kotlin/org/jetbrains/kotlin/compiletest/LldbTests.kt

diff --git a/backend.native/compiletest/build.gradle b/backend.native/compiletest/build.gradle
new file mode 100644
index 00000000000..b81357faf5c
--- /dev/null
+++ b/backend.native/compiletest/build.gradle
@@ -0,0 +1,19 @@
+apply plugin: 'java'
+apply plugin: 'kotlin'
+
+dependencies {
+    compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
+    compile project(path: ':backend.native', configuration: 'cli_bc')
+    testCompile 'junit:junit:4.12'
+}
+
+test {
+    testLogging { exceptionFormat = 'full' }
+
+    def dist = project.rootProject.file("dist").canonicalPath
+    systemProperties = [
+            'konan.home': dist,
+            'java.library.path': "$dist/konan/nativelib"
+    ]
+}
+test.dependsOn ':dist'
\ No newline at end of file
diff --git a/backend.native/compiletest/src/main/kotlin/org/jetbrains/kotlin/compiletest/DistProperties.kt b/backend.native/compiletest/src/main/kotlin/org/jetbrains/kotlin/compiletest/DistProperties.kt
new file mode 100644
index 00000000000..c3d9f1fbdbb
--- /dev/null
+++ b/backend.native/compiletest/src/main/kotlin/org/jetbrains/kotlin/compiletest/DistProperties.kt
@@ -0,0 +1,17 @@
+package org.jetbrains.kotlin.compiletest
+
+import java.nio.file.Path
+import java.nio.file.Paths
+
+
+object DistProperties {
+    private val isWindows: Boolean = requireProp("os.name").startsWith("Windows")
+
+    private val dist: Path = Paths.get(requireProp("konan.home"))
+    private val konancDriver = if (isWindows) "konanc.bat" else "konanc"
+    val konanc: Path = dist.resolve("bin/$konancDriver")
+    val lldb: Path = Paths.get("lldb")
+
+    private fun requireProp(name: String): String
+            = System.getProperty(name) ?: error("Property `$name` is not defined")
+}
\ No newline at end of file
diff --git a/backend.native/compiletest/src/main/kotlin/org/jetbrains/kotlin/compiletest/Driver.kt b/backend.native/compiletest/src/main/kotlin/org/jetbrains/kotlin/compiletest/Driver.kt
new file mode 100644
index 00000000000..e0a745b710d
--- /dev/null
+++ b/backend.native/compiletest/src/main/kotlin/org/jetbrains/kotlin/compiletest/Driver.kt
@@ -0,0 +1,71 @@
+package org.jetbrains.kotlin.compiletest
+
+import org.jetbrains.kotlin.cli.bc.K2Native
+import java.nio.file.Files
+import java.nio.file.Path
+import java.util.concurrent.TimeUnit
+
+class ToolDriver(
+        private val konancDriver: Path,
+        private val lldb: Path,
+        private val useInProcessCompiler: Boolean = false
+) {
+    fun compile(source: Path, output: Path, vararg args: String) {
+        check(!Files.exists(output))
+        val allArgs = listOf("-output", output.toString(), source.toString(), *args).toTypedArray()
+
+        if (useInProcessCompiler) {
+            K2Native.main(allArgs)
+        } else {
+            subprocess(konancDriver, *allArgs).thrownIfFailed()
+
+        }
+        check(Files.exists(output)) {
+            "Compiler has not produced an output at $output"
+        }
+    }
+
+    fun runLldb(program: Path, commands: List<String>): String {
+        val args = commands.flatMap { listOf("-o", it) }
+        return subprocess(lldb, program.toString(), "-b", *args.toTypedArray())
+                .thrownIfFailed()
+                .stdout
+    }
+}
+
+data class ProcessOutput(
+        val program: Path,
+        val process: Process,
+        val stdout: String,
+        val stderr: String,
+        val durationMs: Long
+) {
+    fun thrownIfFailed(): ProcessOutput {
+        fun renderStdStream(name: String, text: String): String =
+                if (text.isBlank()) "$name is empty" else "$name:\n$text"
+
+        check(process.exitValue() == 0) {
+            """$program exited with non-zero value: ${process.exitValue()}
+              |${renderStdStream("stdout", stdout)}
+              |${renderStdStream("stderr", stderr)}
+            """.trimMargin()
+        }
+        return this
+    }
+}
+
+fun subprocess(program: Path, vararg args: String): ProcessOutput {
+    val start = System.currentTimeMillis()
+    val process = ProcessBuilder(program.toString(), *args).start()
+    val outReader = process.inputStream.bufferedReader()
+    val errReader = process.errorStream.bufferedReader()
+
+    val timeout = 5L
+    if (!process.waitFor(timeout, TimeUnit.MINUTES)) {
+        process.destroy()
+        error("$program is running for more then $timeout minutes")
+    }
+    val stdout = outReader.readText()
+    val stderr = errReader.readText()
+    return ProcessOutput(program, process, stdout, stderr, System.currentTimeMillis() - start)
+}
\ No newline at end of file
diff --git a/backend.native/compiletest/src/main/kotlin/org/jetbrains/kotlin/compiletest/Matchers.kt b/backend.native/compiletest/src/main/kotlin/org/jetbrains/kotlin/compiletest/Matchers.kt
new file mode 100644
index 00000000000..7e48511503f
--- /dev/null
+++ b/backend.native/compiletest/src/main/kotlin/org/jetbrains/kotlin/compiletest/Matchers.kt
@@ -0,0 +1,99 @@
+package org.jetbrains.kotlin.compiletest
+
+import org.intellij.lang.annotations.Language
+import java.io.IOException
+import java.nio.file.Files
+
+
+fun multilineMatch(patterns: List<String>, actualLines: List<String>) {
+    fun indexOfLine(line: String): Int {
+        val chunks = line.split("""\[\.\.]""".toRegex()).filter { it.isNotBlank() }
+
+        for ((i, actual) in actualLines.withIndex()) {
+            val indices = chunks.map { actual.indexOf(it) }
+            if (indices.all { it != -1 }) {
+                check(indices == indices.sorted())
+                return i
+            }
+        }
+
+        error("Can't find match for `$line` in\n${actualLines.joinToString("\n")}")
+    }
+
+    val indices = patterns.map { indexOfLine(it) }
+    check(indices == indices.sorted())
+}
+
+private val haveLldb: Boolean by lazy {
+    val lldbVersion = try {
+        subprocess(DistProperties.lldb, "-version")
+                .takeIf { it.process.exitValue() == 0 }
+                ?.stdout
+    } catch (e: IOException) {
+        null
+    }
+
+    if (lldbVersion == null) {
+        println("No LLDB found")
+    } else {
+        println("Using $lldbVersion")
+    }
+
+    lldbVersion != null
+}
+
+fun lldbTest(@Language("kotlin") program: String, lldbSession: String) {
+    if (!haveLldb) {
+        println("Skipping test: no LLDB")
+        return
+    }
+
+    val lldbSessionSpec = LldbSessionSpecification.parse(lldbSession)
+
+    val tmpdir = Files.createTempDirectory("debugger_test")
+    tmpdir.toFile().deleteOnExit()
+    val source = tmpdir.resolve("main.kt")
+    val output = tmpdir.resolve("program.kexe")
+
+    val driver = ToolDriver(DistProperties.konanc, DistProperties.lldb)
+    Files.write(source, program.trimIndent().toByteArray())
+    driver.compile(source, output, "-g")
+    val result = driver.runLldb(output, lldbSessionSpec.commands)
+    lldbSessionSpec.match(result)
+}
+
+private class LldbSessionSpecification private constructor(
+        val commands: List<String>,
+        val patterns: List<List<String>>
+) {
+
+    fun match(output: String) {
+        val blocks = output.split("""(?=\(lldb\))""".toRegex())
+        check(blocks.first().startsWith("(lldb) target create"))
+        val responses = blocks.drop(1)
+        val headers = responses.map { it.lines().first() }
+        val bodies = responses.map { it.lines().drop(1) }
+        val responsesMatch = headers.size == commands.size
+                && commands.zip(headers).all { (cmd, h) -> h == "(lldb) $cmd" }
+
+        check(responsesMatch) {
+            "Responses do not match commands.\nResponses: $headers\nCommands: $commands"
+        }
+
+        for ((pattern, body) in patterns.zip(bodies)) {
+            multilineMatch(pattern, body)
+        }
+    }
+
+    companion object {
+        fun parse(spec: String): LldbSessionSpecification {
+            val blocks = spec.trimIndent().split("(?=^>)".toRegex(RegexOption.MULTILINE))
+            for (cmd in blocks) {
+                check(cmd.startsWith(">")) { "Invalid lldb session specification" }
+            }
+            val commands = blocks.map { it.lines().first().substring(1).trim() }
+            val patterns = blocks.map { it.lines().drop(1).filter { it.isNotBlank() } }
+            return LldbSessionSpecification(commands, patterns)
+        }
+    }
+}
diff --git a/backend.native/compiletest/src/test/kotlin/org/jetbrains/kotlin/compiletest/LldbTests.kt b/backend.native/compiletest/src/test/kotlin/org/jetbrains/kotlin/compiletest/LldbTests.kt
new file mode 100644
index 00000000000..5f1d193f135
--- /dev/null
+++ b/backend.native/compiletest/src/test/kotlin/org/jetbrains/kotlin/compiletest/LldbTests.kt
@@ -0,0 +1,56 @@
+import org.jetbrains.kotlin.compiletest.lldbTest
+import org.junit.Test
+
+class LldbTests {
+    //FIXME: the last one should be main.kt.5
+    @Test
+    fun `can step through code`() = lldbTest("""
+        fun main(args: Array<String>) {
+            var x = 1
+            var y = 2
+            var z = x + y
+            println(z)
+        }
+    """, """
+        > b main.kt:2
+        Breakpoint 1:
+
+        > r
+        Process [..] stopped
+        [..] at main.kt:2, [..] stop reason = breakpoint
+
+        > n
+        Process [..] stopped
+        [..] at main.kt:3, [..] stop reason = step over
+
+        > n
+        Process [..] stopped
+        [..] at main.kt:4, [..] stop reason = step over
+
+        > n
+        Process [..] stopped
+        [..] at main.kt:3, [..] stop reason = step over
+    """)
+
+    //FIXME: Boolean and Int are wrong
+    @Test
+    fun `can inspect values of primitive types`() = lldbTest("""
+        fun main(args: Array<String>) {
+            var a: Byte =  1
+            var b: Int  =  2
+            var c: Long = -3
+            var d: Char = 'c'
+            var e: Boolean = true
+            return
+        }
+    """, """
+            > b main.kt:7
+            > r
+            > fr var
+            (char) a = '\x01'
+            (int) b = 2
+            (long) c = -3
+            (unsigned char) d = 'c'
+            (void) e = <Unable to determine byte size.>
+    """)
+}
\ No newline at end of file
diff --git a/settings.gradle b/settings.gradle
index 24b2f4efa1d..693046b6ab2 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -25,6 +25,7 @@ include ':backend.native'
 include ':runtime'
 include ':common'
 include ':backend.native:tests'
+include ':backend.native:compiletest'
 include ':shared'
 include ':tools:kotlin-native-gradle-plugin'
 include ':utilities'