azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[coverage] Initial infrastructure for testing -Xcoverage flag (#3429)

Browse Source
This commit is contained in:
Sergey Bogolepov
2019-10-08 23:03:35 +03:00
committed by GitHub
parent 2ab841afd3
commit 912a889b68
8 changed files with 256 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/coverage/CoverageRegionCollector.kt
+11 -1
View File
@@ -38,7 +38,7 @@ internal class CoverageRegionCollector(private val fileFilter: (IrFile) -> Boole
}
override fun visitFunction(declaration: IrFunction) {
if (!declaration.isInline && !declaration.isExternal) {
if (!declaration.isInline && !declaration.isExternal && !declaration.isGeneratedByCompiler) {
declaration.body?.let {
val regionsCollector = IrFunctionRegionsCollector(fileFilter, file)
regionsCollector.visitBody(it)
@@ -53,6 +53,13 @@ internal class CoverageRegionCollector(private val fileFilter: (IrFile) -> Boole
}
}
// User doesn't bother about compiler-generated declarations.
// So lets filter them.
private val IrDeclaration.isGeneratedByCompiler: Boolean
get() {
return origin != IrDeclarationOrigin.DEFINED
}
/**
* Very similar to [org.jetbrains.kotlin.backend.konan.llvm.CodeGeneratorVisitor] but instead of bitcode generation we collect regions.
* [fileFilter]: specify which files should be processed by code coverage. Here it is required
@@ -83,6 +90,7 @@ private class IrFunctionRegionsCollector(
}
override fun visitVariable(declaration: IrVariable) {
recordRegion(declaration)
declaration.initializer?.let { collectRegions(it) }
}
@@ -95,6 +103,7 @@ private class IrFunctionRegionsCollector(
fun collectRegions(value: IrExpression): Unit = when (value) {
is IrTypeOperatorCall -> collectTypeOperator(value)
is IrCall -> collectCall(value)
is IrConstructorCall -> collectCall(value)
is IrDelegatingConstructorCall -> collectCall(value)
is IrInstanceInitializerCall -> collectInstanceInitializerCall(value)
is IrGetValue -> collectGetValue(value)
@@ -130,6 +139,7 @@ private class IrFunctionRegionsCollector(
}
private fun collectSetVariable(setVariable: IrSetVariable) {
recordRegion(setVariable)
setVariable.value.acceptVoid(this)
}
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/coverage/LLVMCoverageInstrumentation.kt
+1
View File
@@ -27,6 +27,7 @@ internal class LLVMCoverageInstrumentation(
private val functionHash = Int64(functionRegions.structuralHash).llvm
// TODO: It's a great place for some debug output.
fun instrumentIrElement(element: IrElement) {
functionRegions.regions[element]?.let {
placeRegionIncrement(it)
backend.native/tests/build.gradle
+49 -50
View File
@@ -112,6 +112,9 @@ allprojects {
// backend.native/tests/framework
ext.testOutputFramework = rootProject.file("$testOutputRoot/framework")
// backent.native/tests/coverage
ext.testOutputCoverage = rootProject.file("$testOutputRoot/coverage")
}
testOutputExternal.mkdirs()
testOutputStdlib.mkdirs()
@@ -3473,7 +3476,7 @@ task interop_echo_server(type: RunInteropKonanTest) {
*/
/*
TODO: This test isn't run automatically
TODO: This test isn't run automatically
standaloneTest("interop_opengl_teapot") {
enabled = isAppleTarget(project) && project.testTarget != 'ios_x64' // No GLUT in iOS
dependsOnPlatformLibs(it as Task)
@@ -3750,65 +3753,61 @@ KotlinNativeTestKt.createTest(project, 'harmonyRegexTest', KonanGTest) { task ->
extraOpts project.globalTestArgs
}
}
task.finalizedBy("resultsTask")
}
standaloneTest("coverage_basic_program") {
if (UtilsKt.testTargetSupportsCodeCoverage(project)) {
task coverage_basic_program(type: CoverageTest) {
disabled = !UtilsKt.testTargetSupportsCodeCoverage(project)
binaryName = "CoverageBasic"
numberOfCoveredFunctions = 1
numberOfCoveredLines = 1
description = "Test that `-Xcoverage` generates correct __llvm_coverage information"
def dir = buildDir.absolutePath
def coverageFile = "$dir/program.profraw"
flags = ["-Xcoverage-file=$coverageFile", "-Xcoverage", "-entry", "coverage.basic.program.main"]
source = "$projectDir/coverage/basic/program/main.kt"
doLast {
execLlvm("llvm-profdata") {
args "merge"
args "$dir/program.profraw"
args "-o", "$dir/program.profdata"
}
// Should fail in case of corrupted __llvm_coverage.
execLlvm("llvm-cov") {
args "report"
args "$testOutputLocal/$name/${target.name}/$name.${target.family.exeSuffix}"
args "-instr-profile", "$dir/program.profdata"
konanArtifacts {
program(binaryName, targets: [ target ]) {
srcDir 'coverage/basic/program'
baseDir "$testOutputCoverage/$binaryName"
entryPoint "coverage.basic.program.main"
extraOpts "-Xcoverage", "-Xcoverage-file=$profrawFile"
}
}
}
}
standaloneTest("coverage_basic_library") {
task coverage_basic_library(type: CoverageTest) {
disabled = !UtilsKt.testTargetSupportsCodeCoverage(project)
binaryName = "CoverageBasicLibrary"
numberOfCoveredFunctions = 1
numberOfCoveredLines = 1
description = "Test that `-Xlibrary-to-cover` generates correct __llvm_coverage information"
def dir = buildDir.absolutePath
doBefore {
def konancScript = isWindows() ? "konanc.bat" : "konanc"
def konanc = "$dist/bin/$konancScript"
"$konanc -target ${PlatformInfo.getTarget(project).name} $projectDir/coverage/basic/library/library.kt -p library -o $dir/lib_to_cover".execute().waitFor()
}
flags = ["-Xcoverage-file=$dir/program.profraw", "-l", "$dir/lib_to_cover", "-Xlibrary-to-cover=$dir/lib_to_cover.klib", "-entry", "coverage.basic.library.main"]
source = "$projectDir/coverage/basic/library/main.kt"
doLast {
execLlvm("llvm-profdata") {
args "merge"
args "$dir/program.profraw"
args "-o", "$dir/program.profdata"
konanArtifacts {
library("lib_to_cover", targets: [targetName]) {
srcFiles 'coverage/basic/library/library.kt'
}
program(binaryName, targets: [ target ]) {
srcFiles 'coverage/basic/library/main.kt'
libraries {
artifact konanArtifacts.lib_to_cover
}
baseDir "$testOutputCoverage/$binaryName"
entryPoint "coverage.basic.library.main"
extraOpts "-Xcoverage-file=$profrawFile", "-Xlibrary-to-cover=${konanArtifacts.lib_to_cover.getArtifactByTarget(target.name)}"
}
}
// Should fail in case of corrupted __llvm_coverage.
execLlvm("llvm-cov") {
args "report"
args "$testOutputLocal/$name/${target.name}/$name.${target.family.exeSuffix}"
args "-instr-profile", "$dir/program.profdata"
}
task coverage_smoke0(type: CoverageTest) {
binaryName = "CoverageSmoke0"
numberOfCoveredFunctions = 2
numberOfCoveredLines = 3
konanArtifacts {
program(binaryName, targets: [ target ]) {
srcDir 'coverage/basic/smoke0'
baseDir "$testOutputCoverage/$binaryName"
entryPoint "coverage.basic.smoke0.main"
extraOpts "-Xcoverage", "-Xcoverage-file=$profrawFile"
}
}
}
}
@@ -3828,7 +3827,7 @@ KotlinNativeTestKt.createTest(project, 'stdlibTest', KonanGTest) { task ->
srcFiles sources
baseDir "$testOutputStdlib/stdlibTest"
enableMultiplatform true
extraOpts '-tr', '-Xuse-experimental=kotlin.Experimental,kotlin.ExperimentalStdlibApi',
extraOpts '-tr', '-Xuse-experimental=kotlin.Experimental,kotlin.ExperimentalStdlibApi',
"-friend-modules", project.rootProject.file("${project.properties['konan.home']}/klib/common/stdlib").absolutePath
extraOpts project.globalTestArgs
}
backend.native/tests/coverage/basic/smoke0/smoke0.kt
+8
View File
@@ -0,0 +1,8 @@
package coverage.basic.smoke0
data class User(val name: String)
fun main() {
val user = User("Happy Kotlin/Native user")
println(user)
}
build-tools/src/main/kotlin/org/jetbrains/kotlin/CoverageTest.kt
+125
View File
@@ -0,0 +1,125 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin
import groovy.lang.Closure
import org.gradle.api.DefaultTask
import org.gradle.api.Task
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.TaskAction
/**
* Test task for -Xcoverage and -Xlibraries-to-cover flags. Requires a binary to be built by the Konan plugin
* with
* konanArtifacts {
* program([binaryName], targets: [testTarget]) {
* ...
* extraOpts "-Xcoverage"/"-Xlibrary-to-cover=...", "-Xcoverage-file=$[profrawFile]"
* }
* }
* and a dependency set according to a pattern "run${binaryName}".
*
* @property numberOfCoveredFunctions Expected number of covered functions
* @property numberOfCoveredLines Expected number of covered lines in all functions
* @property binaryName Name of the produced binary
*/
open class CoverageTest : DefaultTask() {
private val target = project.testTarget()
private val platform = project.platformManager().platform(target)
private val llvmBin = "${platform.configurables.absoluteLlvmHome}/bin"
@Input
lateinit var binaryName: String
// TODO: Consider better metric.
@Input
var numberOfCoveredFunctions: Int? = null
@Input
var numberOfCoveredLines: Int? = null
val profrawFile: String by lazy {
"${project.buildDir.absolutePath}/$binaryName.profraw"
}
private val profdataFile: String by lazy {
"${project.buildDir.absolutePath}/$binaryName.profdata"
}
private val outputDir: String by lazy {
project.file(project.property("testOutputCoverage")!!).absolutePath
}
override fun configure(closure: Closure<Any>): Task {
super.configure(closure)
dependsOnDist()
dependsOn(project.tasks.getByName("compileKonan$binaryName"))
return this
}
@TaskAction
fun run() {
val suffix = target.family.exeSuffix
val pathToBinary = "$outputDir/$binaryName/$target/$binaryName.$suffix"
runProcess(localExecutor(project), pathToBinary)
exec("llvm-profdata", "merge", profrawFile, "-o", profdataFile)
val llvmCovResult = exec("llvm-cov", "export", pathToBinary, "-instr-profile", profdataFile)
val jsonReport = llvmCovResult.stdOut
val llvmCovReport = parseLlvmCovReport(jsonReport)
try {
CoverageValidator(numberOfCoveredFunctions, numberOfCoveredLines).validateReport(llvmCovReport)
} catch (e: TestFailedException) {
// Show report in message to make debug easier.
val show = exec("llvm-cov", "show", pathToBinary, "-instr-profile", profdataFile).stdOut
// llvm-cov output contains '|' so another symbol is used as margin prefix.
throw TestFailedException("""
>${e.message}
>$show
""".trimMargin(">"))
}
}
private fun exec(llvmTool: String, vararg args: String): ProcessOutput {
val executable = "$llvmBin/$llvmTool"
val result = runProcess(localExecutor(project), executable, args.toList())
if (result.exitCode != 0) {
println("""
$llvmTool failed.
exitCode: ${result.exitCode}
stdout:
${result.stdOut}
stderr:
${result.stdErr}
""".trimIndent())
error("$llvmTool failed")
}
return result
}
}
private class CoverageValidator(
val numberOfCoveredFunctions: Int?,
val numberOfCoveredLines: Int?
) {
fun validateReport(report: LlvmCovReport) {
val data = report.data
if (data.isEmpty()) {
failTest("Report data should not be empty!")
}
compareNumbers(numberOfCoveredFunctions, data[0].totals.functions.covered, "Number of covered functions")
compareNumbers(numberOfCoveredLines, data[0].totals.lines.covered, "Number of covered lines")
}
private fun compareNumbers(expected: Int?, actual: Int, description: String) {
if (expected != null && actual != expected) {
failTest("$description differs from expected! Expected: $expected. Got: $actual")
}
}
private fun failTest(message: String) {
throw TestFailedException(message)
}
}
build-tools/src/main/kotlin/org/jetbrains/kotlin/ExecLlvm.kt
-28
View File
@@ -1,28 +0,0 @@
package org.jetbrains.kotlin
import groovy.lang.Closure
import org.gradle.api.Action
import org.gradle.api.Project
import org.gradle.process.ExecResult
import org.gradle.process.ExecSpec
import org.gradle.util.ConfigureUtil
internal class ExecLlvm(private val project: Project) {
fun execLlvm(tool: String, closure: Closure<in ExecSpec>): ExecResult =
this.execLlvm(tool, ConfigureUtil.configureUsing(closure))
fun execLlvm(tool: String, action: Action<in ExecSpec>): ExecResult {
val extendedAction = Action<ExecSpec> { execSpec ->
action.execute(execSpec)
execSpec.apply {
if (executable == null) {
val llvmDir = project.findProperty("llvmDir")
executable = "$llvmDir/bin/$tool"
}
}
}
return project.exec(extendedAction)
}
}
build-tools/src/main/kotlin/org/jetbrains/kotlin/LlvmCovReport.kt
+62
View File
@@ -0,0 +1,62 @@
package org.jetbrains.kotlin
import kotlinx.serialization.*
import kotlinx.serialization.internal.StringDescriptor
import kotlinx.serialization.json.Json
@Serializable
data class LlvmCovReportFunction(
val name: String,
val count: Int,
val regions: List<List<Int>>,
val filenames: List<String>
)
@Serializable
data class LlvmCovReportSummary(
val lines: LlvmCovReportStatistics,
val functions: LlvmCovReportStatistics,
val instantiations: LlvmCovReportStatistics,
val regions: LlvmCovReportStatistics
)
/**
* TODO: Add support for `segments` field later.
* It's a bit complicated since every segment
* is encoded not as dictionary, but as array of ints and bools.
*/
@Serializable
data class LlvmCovReportFile(
val filename: String,
val summary: LlvmCovReportSummary
)
@Serializable
data class LlvmCovReportStatistics(
val count: Int,
val covered: Int,
val percent: Double
)
@Serializable
data class LlvmCovReportData(
val files: List<LlvmCovReportFile>,
val functions: List<LlvmCovReportFunction>,
val totals: LlvmCovReportSummary
)
@Serializable
data class LlvmCovReport(
val version: String,
val type: String,
val data: List<LlvmCovReportData>
)
fun parseLlvmCovReport(llvmCovReport: String): LlvmCovReport =
Json.nonstrict.parse(LlvmCovReport.serializer(), llvmCovReport)
val LlvmCovReport.isValid
get() = type == "llvm.coverage.json.export"
build.gradle
-1
View File
@@ -104,7 +104,6 @@ void setupClang(Project project) {
project.convention.plugins.platformManager = project.rootProject.ext.platformManager
project.convention.plugins.execClang = new org.jetbrains.kotlin.ExecClang(project)
project.convention.plugins.execLlvm = new org.jetbrains.kotlin.ExecLlvm(project)
project.plugins.withType(NativeComponentPlugin) {
project.model {