[coverage] Initial infrastructure for testing -Xcoverage flag (#3429)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/coverage/CoverageRegionCollector.kt

@@ -38,7 +38,7 @@ internal class CoverageRegionCollector(private val fileFilter: (IrFile) -> Boole
         }
 
         override fun visitFunction(declaration: IrFunction) {
-            if (!declaration.isInline && !declaration.isExternal) {
+            if (!declaration.isInline && !declaration.isExternal && !declaration.isGeneratedByCompiler) {
                 declaration.body?.let {
                     val regionsCollector = IrFunctionRegionsCollector(fileFilter, file)
                     regionsCollector.visitBody(it)
@@ -53,6 +53,13 @@ internal class CoverageRegionCollector(private val fileFilter: (IrFile) -> Boole
     }
 }
 
+// User doesn't bother about compiler-generated declarations.
+// So lets filter them.
+private val IrDeclaration.isGeneratedByCompiler: Boolean
+    get() {
+        return origin != IrDeclarationOrigin.DEFINED
+    }
+
 /**
  * Very similar to [org.jetbrains.kotlin.backend.konan.llvm.CodeGeneratorVisitor] but instead of bitcode generation we collect regions.
  * [fileFilter]: specify which files should be processed by code coverage. Here it is required
@@ -83,6 +90,7 @@ private class IrFunctionRegionsCollector(
     }
 
     override fun visitVariable(declaration: IrVariable) {
+        recordRegion(declaration)
         declaration.initializer?.let { collectRegions(it) }
     }
 
@@ -95,6 +103,7 @@ private class IrFunctionRegionsCollector(
     fun collectRegions(value: IrExpression): Unit = when (value) {
         is IrTypeOperatorCall -> collectTypeOperator(value)
         is IrCall -> collectCall(value)
+        is IrConstructorCall -> collectCall(value)
         is IrDelegatingConstructorCall -> collectCall(value)
         is IrInstanceInitializerCall -> collectInstanceInitializerCall(value)
         is IrGetValue -> collectGetValue(value)
@@ -130,6 +139,7 @@ private class IrFunctionRegionsCollector(
     }
 
     private fun collectSetVariable(setVariable: IrSetVariable) {
+        recordRegion(setVariable)
         setVariable.value.acceptVoid(this)
     }
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/coverage/LLVMCoverageInstrumentation.kt

@@ -27,6 +27,7 @@ internal class LLVMCoverageInstrumentation(
 
     private val functionHash = Int64(functionRegions.structuralHash).llvm
 
+    // TODO: It's a great place for some debug output.
     fun instrumentIrElement(element: IrElement) {
         functionRegions.regions[element]?.let {
             placeRegionIncrement(it)

backend.native/tests/build.gradle

@@ -112,6 +112,9 @@ allprojects {
 
     // backend.native/tests/framework
     ext.testOutputFramework = rootProject.file("$testOutputRoot/framework")
+
+    // backent.native/tests/coverage
+    ext.testOutputCoverage = rootProject.file("$testOutputRoot/coverage")
 }
 testOutputExternal.mkdirs()
 testOutputStdlib.mkdirs()
@@ -3473,7 +3476,7 @@ task interop_echo_server(type: RunInteropKonanTest) {
 */
 
 /*
-TODO: This test isn't run automatically 
+TODO: This test isn't run automatically
 standaloneTest("interop_opengl_teapot") {
     enabled = isAppleTarget(project) && project.testTarget != 'ios_x64'  // No GLUT in iOS
     dependsOnPlatformLibs(it as Task)
@@ -3750,65 +3753,61 @@ KotlinNativeTestKt.createTest(project, 'harmonyRegexTest', KonanGTest) { task ->
             extraOpts project.globalTestArgs
         }
     }
-    
+
     task.finalizedBy("resultsTask")
 }
 
-standaloneTest("coverage_basic_program") {
+if (UtilsKt.testTargetSupportsCodeCoverage(project)) {
+    task coverage_basic_program(type: CoverageTest) {
 
-    disabled = !UtilsKt.testTargetSupportsCodeCoverage(project)
+        binaryName = "CoverageBasic"
+        numberOfCoveredFunctions = 1
+        numberOfCoveredLines = 1
 
-    description = "Test that `-Xcoverage` generates correct __llvm_coverage information"
+        konanArtifacts {
-
+            program(binaryName, targets: [ target ]) {
-    def dir = buildDir.absolutePath
+                srcDir 'coverage/basic/program'
-    def coverageFile = "$dir/program.profraw"
+                baseDir "$testOutputCoverage/$binaryName"
-
+                entryPoint "coverage.basic.program.main"
-    flags = ["-Xcoverage-file=$coverageFile", "-Xcoverage", "-entry", "coverage.basic.program.main"]
+                extraOpts "-Xcoverage", "-Xcoverage-file=$profrawFile"
-    source = "$projectDir/coverage/basic/program/main.kt"
+            }
-
-    doLast {
-        execLlvm("llvm-profdata") {
-            args "merge"
-            args "$dir/program.profraw"
-            args "-o", "$dir/program.profdata"
-        }
-        // Should fail in case of corrupted __llvm_coverage.
-        execLlvm("llvm-cov") {
-            args "report"
-            args "$testOutputLocal/$name/${target.name}/$name.${target.family.exeSuffix}"
-            args "-instr-profile", "$dir/program.profdata"
         }
     }
-}
 
-standaloneTest("coverage_basic_library") {
+    task coverage_basic_library(type: CoverageTest) {
 
-    disabled = !UtilsKt.testTargetSupportsCodeCoverage(project)
+        binaryName = "CoverageBasicLibrary"
+        numberOfCoveredFunctions = 1
+        numberOfCoveredLines = 1
 
-    description = "Test that `-Xlibrary-to-cover` generates correct __llvm_coverage information"
+        konanArtifacts {
-
+            library("lib_to_cover", targets: [targetName]) {
-    def dir = buildDir.absolutePath
+                srcFiles 'coverage/basic/library/library.kt'
-
+            }
-    doBefore {
+            program(binaryName, targets: [ target ]) {
-        def konancScript = isWindows() ? "konanc.bat" : "konanc"
+                srcFiles 'coverage/basic/library/main.kt'
-        def konanc = "$dist/bin/$konancScript"
+                libraries {
-        "$konanc -target ${PlatformInfo.getTarget(project).name} $projectDir/coverage/basic/library/library.kt -p library -o $dir/lib_to_cover".execute().waitFor()
+                    artifact konanArtifacts.lib_to_cover
-    }
+                }
-    
+                baseDir "$testOutputCoverage/$binaryName"
-    flags = ["-Xcoverage-file=$dir/program.profraw", "-l", "$dir/lib_to_cover", "-Xlibrary-to-cover=$dir/lib_to_cover.klib", "-entry", "coverage.basic.library.main"]
+                entryPoint "coverage.basic.library.main"
-    source = "$projectDir/coverage/basic/library/main.kt"
+                extraOpts "-Xcoverage-file=$profrawFile", "-Xlibrary-to-cover=${konanArtifacts.lib_to_cover.getArtifactByTarget(target.name)}"
-
+            }
-    doLast {
-        execLlvm("llvm-profdata") {
-            args "merge"
-            args "$dir/program.profraw"
-            args "-o", "$dir/program.profdata"
         }
-        // Should fail in case of corrupted __llvm_coverage.
+    }
-        execLlvm("llvm-cov") {
+
-            args "report"
+    task coverage_smoke0(type: CoverageTest) {
-            args "$testOutputLocal/$name/${target.name}/$name.${target.family.exeSuffix}"
+        binaryName = "CoverageSmoke0"
-            args "-instr-profile", "$dir/program.profdata"
+        numberOfCoveredFunctions = 2
+        numberOfCoveredLines = 3
+
+        konanArtifacts {
+            program(binaryName, targets: [ target ]) {
+                srcDir 'coverage/basic/smoke0'
+                baseDir "$testOutputCoverage/$binaryName"
+                entryPoint "coverage.basic.smoke0.main"
+                extraOpts "-Xcoverage", "-Xcoverage-file=$profrawFile"
+            }
         }
     }
 }
@@ -3828,7 +3827,7 @@ KotlinNativeTestKt.createTest(project, 'stdlibTest', KonanGTest) { task ->
             srcFiles sources
             baseDir "$testOutputStdlib/stdlibTest"
             enableMultiplatform true
-            extraOpts '-tr', '-Xuse-experimental=kotlin.Experimental,kotlin.ExperimentalStdlibApi', 
+            extraOpts '-tr', '-Xuse-experimental=kotlin.Experimental,kotlin.ExperimentalStdlibApi',
                     "-friend-modules", project.rootProject.file("${project.properties['konan.home']}/klib/common/stdlib").absolutePath
             extraOpts project.globalTestArgs
         }

backend.native/tests/coverage/basic/smoke0/smoke0.kt

@@ -0,0 +1,8 @@
+package coverage.basic.smoke0
+
+data class User(val name: String)
+
+fun main() {
+    val user = User("Happy Kotlin/Native user")
+    println(user)
+}

build-tools/src/main/kotlin/org/jetbrains/kotlin/CoverageTest.kt

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+package org.jetbrains.kotlin
+
+import groovy.lang.Closure
+import org.gradle.api.DefaultTask
+import org.gradle.api.Task
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.TaskAction
+
+/**
+ * Test task for -Xcoverage and -Xlibraries-to-cover flags. Requires a binary to be built by the Konan plugin
+ * with
+ * konanArtifacts {
+ *      program([binaryName], targets: [testTarget]) {
+ *          ...
+ *          extraOpts "-Xcoverage"/"-Xlibrary-to-cover=...", "-Xcoverage-file=$[profrawFile]"
+ *      }
+ *  }
+ * and a dependency set according to a pattern "run${binaryName}".
+ *
+ * @property numberOfCoveredFunctions Expected number of covered functions
+ * @property numberOfCoveredLines Expected number of covered lines in all functions
+ * @property binaryName Name of the produced binary
+ */
+open class CoverageTest : DefaultTask() {
+
+    private val target = project.testTarget()
+    private val platform = project.platformManager().platform(target)
+    private val llvmBin = "${platform.configurables.absoluteLlvmHome}/bin"
+
+    @Input
+    lateinit var binaryName: String
+
+    // TODO: Consider better metric.
+    @Input
+    var numberOfCoveredFunctions: Int? = null
+    @Input
+    var numberOfCoveredLines: Int? = null
+
+    val profrawFile: String by lazy {
+        "${project.buildDir.absolutePath}/$binaryName.profraw"
+    }
+
+    private val profdataFile: String by lazy {
+        "${project.buildDir.absolutePath}/$binaryName.profdata"
+    }
+
+    private val outputDir: String by lazy {
+        project.file(project.property("testOutputCoverage")!!).absolutePath
+    }
+
+    override fun configure(closure: Closure<Any>): Task {
+        super.configure(closure)
+        dependsOnDist()
+        dependsOn(project.tasks.getByName("compileKonan$binaryName"))
+        return this
+    }
+
+    @TaskAction
+    fun run() {
+        val suffix = target.family.exeSuffix
+        val pathToBinary = "$outputDir/$binaryName/$target/$binaryName.$suffix"
+        runProcess(localExecutor(project), pathToBinary)
+        exec("llvm-profdata", "merge", profrawFile, "-o", profdataFile)
+        val llvmCovResult = exec("llvm-cov", "export", pathToBinary, "-instr-profile", profdataFile)
+        val jsonReport = llvmCovResult.stdOut
+        val llvmCovReport = parseLlvmCovReport(jsonReport)
+        try {
+            CoverageValidator(numberOfCoveredFunctions, numberOfCoveredLines).validateReport(llvmCovReport)
+        } catch (e: TestFailedException) {
+            // Show report in message to make debug easier.
+            val show = exec("llvm-cov", "show", pathToBinary, "-instr-profile", profdataFile).stdOut
+            // llvm-cov output contains '|' so another symbol is used as margin prefix.
+            throw TestFailedException("""
+                >${e.message}
+                >$show
+            """.trimMargin(">"))
+        }
+    }
+
+    private fun exec(llvmTool: String, vararg args: String): ProcessOutput {
+        val executable = "$llvmBin/$llvmTool"
+        val result = runProcess(localExecutor(project), executable, args.toList())
+        if (result.exitCode != 0) {
+            println("""
+                $llvmTool failed.
+                exitCode: ${result.exitCode}
+                stdout:
+                ${result.stdOut}
+                stderr:
+                ${result.stdErr}
+            """.trimIndent())
+            error("$llvmTool failed")
+        }
+        return result
+    }
+}
+
+private class CoverageValidator(
+        val numberOfCoveredFunctions: Int?,
+        val numberOfCoveredLines: Int?
+) {
+    fun validateReport(report: LlvmCovReport) {
+        val data = report.data
+        if (data.isEmpty()) {
+            failTest("Report data should not be empty!")
+        }
+        compareNumbers(numberOfCoveredFunctions, data[0].totals.functions.covered, "Number of covered functions")
+        compareNumbers(numberOfCoveredLines, data[0].totals.lines.covered, "Number of covered lines")
+    }
+
+    private fun compareNumbers(expected: Int?, actual: Int, description: String) {
+        if (expected != null && actual != expected) {
+            failTest("$description differs from expected! Expected: $expected. Got: $actual")
+        }
+    }
+
+    private fun failTest(message: String) {
+        throw TestFailedException(message)
+    }
+}

build-tools/src/main/kotlin/org/jetbrains/kotlin/ExecLlvm.kt

@@ -1,28 +0,0 @@
-package org.jetbrains.kotlin
-
-import groovy.lang.Closure
-import org.gradle.api.Action
-import org.gradle.api.Project
-import org.gradle.process.ExecResult
-import org.gradle.process.ExecSpec
-import org.gradle.util.ConfigureUtil
-
-internal class ExecLlvm(private val project: Project) {
-
-    fun execLlvm(tool: String, closure: Closure<in ExecSpec>): ExecResult =
-            this.execLlvm(tool, ConfigureUtil.configureUsing(closure))
-
-    fun execLlvm(tool: String, action: Action<in ExecSpec>): ExecResult {
-        val extendedAction = Action<ExecSpec> { execSpec ->
-            action.execute(execSpec)
-
-            execSpec.apply {
-                if (executable == null) {
-                    val llvmDir = project.findProperty("llvmDir")
-                    executable = "$llvmDir/bin/$tool"
-                }
-            }
-        }
-        return project.exec(extendedAction)
-    }
-}

build-tools/src/main/kotlin/org/jetbrains/kotlin/LlvmCovReport.kt

@@ -0,0 +1,62 @@
+package org.jetbrains.kotlin
+
+import kotlinx.serialization.*
+import kotlinx.serialization.internal.StringDescriptor
+import kotlinx.serialization.json.Json
+
+
+@Serializable
+data class LlvmCovReportFunction(
+        val name: String,
+        val count: Int,
+        val regions: List<List<Int>>,
+        val filenames: List<String>
+)
+
+@Serializable
+data class LlvmCovReportSummary(
+        val lines: LlvmCovReportStatistics,
+        val functions: LlvmCovReportStatistics,
+        val instantiations: LlvmCovReportStatistics,
+        val regions: LlvmCovReportStatistics
+
+)
+
+/**
+ * TODO: Add support for `segments` field later.
+ *  It's a bit complicated since every segment
+ *  is encoded not as dictionary, but as array of ints and bools.
+ */
+@Serializable
+data class LlvmCovReportFile(
+        val filename: String,
+        val summary: LlvmCovReportSummary
+)
+
+@Serializable
+data class LlvmCovReportStatistics(
+    val count: Int,
+    val covered: Int,
+    val percent: Double
+)
+
+@Serializable
+data class LlvmCovReportData(
+        val files: List<LlvmCovReportFile>,
+        val functions: List<LlvmCovReportFunction>,
+        val totals: LlvmCovReportSummary
+)
+
+@Serializable
+data class LlvmCovReport(
+        val version: String,
+        val type: String,
+        val data: List<LlvmCovReportData>
+)
+
+fun parseLlvmCovReport(llvmCovReport: String): LlvmCovReport =
+        Json.nonstrict.parse(LlvmCovReport.serializer(), llvmCovReport)
+
+val LlvmCovReport.isValid
+    get() = type == "llvm.coverage.json.export"
+

build.gradle

@@ -104,7 +104,6 @@ void setupClang(Project project) {
 
     project.convention.plugins.platformManager = project.rootProject.ext.platformManager
     project.convention.plugins.execClang = new org.jetbrains.kotlin.ExecClang(project)
-    project.convention.plugins.execLlvm = new org.jetbrains.kotlin.ExecLlvm(project)
 
     project.plugins.withType(NativeComponentPlugin) {
         project.model {