diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/coverage/CoverageRegionCollector.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/coverage/CoverageRegionCollector.kt index 406260646ec..ff8b5b1e67a 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/coverage/CoverageRegionCollector.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/coverage/CoverageRegionCollector.kt @@ -38,7 +38,7 @@ internal class CoverageRegionCollector(private val fileFilter: (IrFile) -> Boole } override fun visitFunction(declaration: IrFunction) { - if (!declaration.isInline && !declaration.isExternal) { + if (!declaration.isInline && !declaration.isExternal && !declaration.isGeneratedByCompiler) { declaration.body?.let { val regionsCollector = IrFunctionRegionsCollector(fileFilter, file) regionsCollector.visitBody(it) @@ -53,6 +53,13 @@ internal class CoverageRegionCollector(private val fileFilter: (IrFile) -> Boole } } +// User doesn't bother about compiler-generated declarations. +// So lets filter them. +private val IrDeclaration.isGeneratedByCompiler: Boolean + get() { + return origin != IrDeclarationOrigin.DEFINED + } + /** * Very similar to [org.jetbrains.kotlin.backend.konan.llvm.CodeGeneratorVisitor] but instead of bitcode generation we collect regions. * [fileFilter]: specify which files should be processed by code coverage. Here it is required @@ -83,6 +90,7 @@ private class IrFunctionRegionsCollector( } override fun visitVariable(declaration: IrVariable) { + recordRegion(declaration) declaration.initializer?.let { collectRegions(it) } } @@ -95,6 +103,7 @@ private class IrFunctionRegionsCollector( fun collectRegions(value: IrExpression): Unit = when (value) { is IrTypeOperatorCall -> collectTypeOperator(value) is IrCall -> collectCall(value) + is IrConstructorCall -> collectCall(value) is IrDelegatingConstructorCall -> collectCall(value) is IrInstanceInitializerCall -> collectInstanceInitializerCall(value) is IrGetValue -> collectGetValue(value) @@ -130,6 +139,7 @@ private class IrFunctionRegionsCollector( } private fun collectSetVariable(setVariable: IrSetVariable) { + recordRegion(setVariable) setVariable.value.acceptVoid(this) } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/coverage/LLVMCoverageInstrumentation.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/coverage/LLVMCoverageInstrumentation.kt index 5241b6075ea..28ac78e36ed 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/coverage/LLVMCoverageInstrumentation.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/coverage/LLVMCoverageInstrumentation.kt @@ -27,6 +27,7 @@ internal class LLVMCoverageInstrumentation( private val functionHash = Int64(functionRegions.structuralHash).llvm + // TODO: It's a great place for some debug output. fun instrumentIrElement(element: IrElement) { functionRegions.regions[element]?.let { placeRegionIncrement(it) diff --git a/backend.native/tests/build.gradle b/backend.native/tests/build.gradle index 2452567d945..b296890b8a6 100644 --- a/backend.native/tests/build.gradle +++ b/backend.native/tests/build.gradle @@ -112,6 +112,9 @@ allprojects { // backend.native/tests/framework ext.testOutputFramework = rootProject.file("$testOutputRoot/framework") + + // backent.native/tests/coverage + ext.testOutputCoverage = rootProject.file("$testOutputRoot/coverage") } testOutputExternal.mkdirs() testOutputStdlib.mkdirs() @@ -3473,7 +3476,7 @@ task interop_echo_server(type: RunInteropKonanTest) { */ /* -TODO: This test isn't run automatically +TODO: This test isn't run automatically standaloneTest("interop_opengl_teapot") { enabled = isAppleTarget(project) && project.testTarget != 'ios_x64' // No GLUT in iOS dependsOnPlatformLibs(it as Task) @@ -3750,65 +3753,61 @@ KotlinNativeTestKt.createTest(project, 'harmonyRegexTest', KonanGTest) { task -> extraOpts project.globalTestArgs } } - + task.finalizedBy("resultsTask") } -standaloneTest("coverage_basic_program") { +if (UtilsKt.testTargetSupportsCodeCoverage(project)) { + task coverage_basic_program(type: CoverageTest) { - disabled = !UtilsKt.testTargetSupportsCodeCoverage(project) + binaryName = "CoverageBasic" + numberOfCoveredFunctions = 1 + numberOfCoveredLines = 1 - description = "Test that `-Xcoverage` generates correct __llvm_coverage information" - - def dir = buildDir.absolutePath - def coverageFile = "$dir/program.profraw" - - flags = ["-Xcoverage-file=$coverageFile", "-Xcoverage", "-entry", "coverage.basic.program.main"] - source = "$projectDir/coverage/basic/program/main.kt" - - doLast { - execLlvm("llvm-profdata") { - args "merge" - args "$dir/program.profraw" - args "-o", "$dir/program.profdata" - } - // Should fail in case of corrupted __llvm_coverage. - execLlvm("llvm-cov") { - args "report" - args "$testOutputLocal/$name/${target.name}/$name.${target.family.exeSuffix}" - args "-instr-profile", "$dir/program.profdata" + konanArtifacts { + program(binaryName, targets: [ target ]) { + srcDir 'coverage/basic/program' + baseDir "$testOutputCoverage/$binaryName" + entryPoint "coverage.basic.program.main" + extraOpts "-Xcoverage", "-Xcoverage-file=$profrawFile" + } } } -} -standaloneTest("coverage_basic_library") { + task coverage_basic_library(type: CoverageTest) { - disabled = !UtilsKt.testTargetSupportsCodeCoverage(project) + binaryName = "CoverageBasicLibrary" + numberOfCoveredFunctions = 1 + numberOfCoveredLines = 1 - description = "Test that `-Xlibrary-to-cover` generates correct __llvm_coverage information" - - def dir = buildDir.absolutePath - - doBefore { - def konancScript = isWindows() ? "konanc.bat" : "konanc" - def konanc = "$dist/bin/$konancScript" - "$konanc -target ${PlatformInfo.getTarget(project).name} $projectDir/coverage/basic/library/library.kt -p library -o $dir/lib_to_cover".execute().waitFor() - } - - flags = ["-Xcoverage-file=$dir/program.profraw", "-l", "$dir/lib_to_cover", "-Xlibrary-to-cover=$dir/lib_to_cover.klib", "-entry", "coverage.basic.library.main"] - source = "$projectDir/coverage/basic/library/main.kt" - - doLast { - execLlvm("llvm-profdata") { - args "merge" - args "$dir/program.profraw" - args "-o", "$dir/program.profdata" + konanArtifacts { + library("lib_to_cover", targets: [targetName]) { + srcFiles 'coverage/basic/library/library.kt' + } + program(binaryName, targets: [ target ]) { + srcFiles 'coverage/basic/library/main.kt' + libraries { + artifact konanArtifacts.lib_to_cover + } + baseDir "$testOutputCoverage/$binaryName" + entryPoint "coverage.basic.library.main" + extraOpts "-Xcoverage-file=$profrawFile", "-Xlibrary-to-cover=${konanArtifacts.lib_to_cover.getArtifactByTarget(target.name)}" + } } - // Should fail in case of corrupted __llvm_coverage. - execLlvm("llvm-cov") { - args "report" - args "$testOutputLocal/$name/${target.name}/$name.${target.family.exeSuffix}" - args "-instr-profile", "$dir/program.profdata" + } + + task coverage_smoke0(type: CoverageTest) { + binaryName = "CoverageSmoke0" + numberOfCoveredFunctions = 2 + numberOfCoveredLines = 3 + + konanArtifacts { + program(binaryName, targets: [ target ]) { + srcDir 'coverage/basic/smoke0' + baseDir "$testOutputCoverage/$binaryName" + entryPoint "coverage.basic.smoke0.main" + extraOpts "-Xcoverage", "-Xcoverage-file=$profrawFile" + } } } } @@ -3828,7 +3827,7 @@ KotlinNativeTestKt.createTest(project, 'stdlibTest', KonanGTest) { task -> srcFiles sources baseDir "$testOutputStdlib/stdlibTest" enableMultiplatform true - extraOpts '-tr', '-Xuse-experimental=kotlin.Experimental,kotlin.ExperimentalStdlibApi', + extraOpts '-tr', '-Xuse-experimental=kotlin.Experimental,kotlin.ExperimentalStdlibApi', "-friend-modules", project.rootProject.file("${project.properties['konan.home']}/klib/common/stdlib").absolutePath extraOpts project.globalTestArgs } diff --git a/backend.native/tests/coverage/basic/smoke0/smoke0.kt b/backend.native/tests/coverage/basic/smoke0/smoke0.kt new file mode 100644 index 00000000000..b32041494c9 --- /dev/null +++ b/backend.native/tests/coverage/basic/smoke0/smoke0.kt @@ -0,0 +1,8 @@ +package coverage.basic.smoke0 + +data class User(val name: String) + +fun main() { + val user = User("Happy Kotlin/Native user") + println(user) +} \ No newline at end of file diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/CoverageTest.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/CoverageTest.kt new file mode 100644 index 00000000000..e0b764d2e3b --- /dev/null +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/CoverageTest.kt @@ -0,0 +1,125 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ + +package org.jetbrains.kotlin + +import groovy.lang.Closure +import org.gradle.api.DefaultTask +import org.gradle.api.Task +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.TaskAction + +/** + * Test task for -Xcoverage and -Xlibraries-to-cover flags. Requires a binary to be built by the Konan plugin + * with + * konanArtifacts { + * program([binaryName], targets: [testTarget]) { + * ... + * extraOpts "-Xcoverage"/"-Xlibrary-to-cover=...", "-Xcoverage-file=$[profrawFile]" + * } + * } + * and a dependency set according to a pattern "run${binaryName}". + * + * @property numberOfCoveredFunctions Expected number of covered functions + * @property numberOfCoveredLines Expected number of covered lines in all functions + * @property binaryName Name of the produced binary + */ +open class CoverageTest : DefaultTask() { + + private val target = project.testTarget() + private val platform = project.platformManager().platform(target) + private val llvmBin = "${platform.configurables.absoluteLlvmHome}/bin" + + @Input + lateinit var binaryName: String + + // TODO: Consider better metric. + @Input + var numberOfCoveredFunctions: Int? = null + @Input + var numberOfCoveredLines: Int? = null + + val profrawFile: String by lazy { + "${project.buildDir.absolutePath}/$binaryName.profraw" + } + + private val profdataFile: String by lazy { + "${project.buildDir.absolutePath}/$binaryName.profdata" + } + + private val outputDir: String by lazy { + project.file(project.property("testOutputCoverage")!!).absolutePath + } + + override fun configure(closure: Closure): Task { + super.configure(closure) + dependsOnDist() + dependsOn(project.tasks.getByName("compileKonan$binaryName")) + return this + } + + @TaskAction + fun run() { + val suffix = target.family.exeSuffix + val pathToBinary = "$outputDir/$binaryName/$target/$binaryName.$suffix" + runProcess(localExecutor(project), pathToBinary) + exec("llvm-profdata", "merge", profrawFile, "-o", profdataFile) + val llvmCovResult = exec("llvm-cov", "export", pathToBinary, "-instr-profile", profdataFile) + val jsonReport = llvmCovResult.stdOut + val llvmCovReport = parseLlvmCovReport(jsonReport) + try { + CoverageValidator(numberOfCoveredFunctions, numberOfCoveredLines).validateReport(llvmCovReport) + } catch (e: TestFailedException) { + // Show report in message to make debug easier. + val show = exec("llvm-cov", "show", pathToBinary, "-instr-profile", profdataFile).stdOut + // llvm-cov output contains '|' so another symbol is used as margin prefix. + throw TestFailedException(""" + >${e.message} + >$show + """.trimMargin(">")) + } + } + + private fun exec(llvmTool: String, vararg args: String): ProcessOutput { + val executable = "$llvmBin/$llvmTool" + val result = runProcess(localExecutor(project), executable, args.toList()) + if (result.exitCode != 0) { + println(""" + $llvmTool failed. + exitCode: ${result.exitCode} + stdout: + ${result.stdOut} + stderr: + ${result.stdErr} + """.trimIndent()) + error("$llvmTool failed") + } + return result + } +} + +private class CoverageValidator( + val numberOfCoveredFunctions: Int?, + val numberOfCoveredLines: Int? +) { + fun validateReport(report: LlvmCovReport) { + val data = report.data + if (data.isEmpty()) { + failTest("Report data should not be empty!") + } + compareNumbers(numberOfCoveredFunctions, data[0].totals.functions.covered, "Number of covered functions") + compareNumbers(numberOfCoveredLines, data[0].totals.lines.covered, "Number of covered lines") + } + + private fun compareNumbers(expected: Int?, actual: Int, description: String) { + if (expected != null && actual != expected) { + failTest("$description differs from expected! Expected: $expected. Got: $actual") + } + } + + private fun failTest(message: String) { + throw TestFailedException(message) + } +} \ No newline at end of file diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/ExecLlvm.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/ExecLlvm.kt index 592ea7e9c2e..e69de29bb2d 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/ExecLlvm.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/ExecLlvm.kt @@ -1,28 +0,0 @@ -package org.jetbrains.kotlin - -import groovy.lang.Closure -import org.gradle.api.Action -import org.gradle.api.Project -import org.gradle.process.ExecResult -import org.gradle.process.ExecSpec -import org.gradle.util.ConfigureUtil - -internal class ExecLlvm(private val project: Project) { - - fun execLlvm(tool: String, closure: Closure): ExecResult = - this.execLlvm(tool, ConfigureUtil.configureUsing(closure)) - - fun execLlvm(tool: String, action: Action): ExecResult { - val extendedAction = Action { execSpec -> - action.execute(execSpec) - - execSpec.apply { - if (executable == null) { - val llvmDir = project.findProperty("llvmDir") - executable = "$llvmDir/bin/$tool" - } - } - } - return project.exec(extendedAction) - } -} \ No newline at end of file diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/LlvmCovReport.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/LlvmCovReport.kt new file mode 100644 index 00000000000..0c8b0da7648 --- /dev/null +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/LlvmCovReport.kt @@ -0,0 +1,62 @@ +package org.jetbrains.kotlin + +import kotlinx.serialization.* +import kotlinx.serialization.internal.StringDescriptor +import kotlinx.serialization.json.Json + + +@Serializable +data class LlvmCovReportFunction( + val name: String, + val count: Int, + val regions: List>, + val filenames: List +) + +@Serializable +data class LlvmCovReportSummary( + val lines: LlvmCovReportStatistics, + val functions: LlvmCovReportStatistics, + val instantiations: LlvmCovReportStatistics, + val regions: LlvmCovReportStatistics + +) + +/** + * TODO: Add support for `segments` field later. + * It's a bit complicated since every segment + * is encoded not as dictionary, but as array of ints and bools. + */ +@Serializable +data class LlvmCovReportFile( + val filename: String, + val summary: LlvmCovReportSummary +) + +@Serializable +data class LlvmCovReportStatistics( + val count: Int, + val covered: Int, + val percent: Double +) + +@Serializable +data class LlvmCovReportData( + val files: List, + val functions: List, + val totals: LlvmCovReportSummary +) + +@Serializable +data class LlvmCovReport( + val version: String, + val type: String, + val data: List +) + +fun parseLlvmCovReport(llvmCovReport: String): LlvmCovReport = + Json.nonstrict.parse(LlvmCovReport.serializer(), llvmCovReport) + +val LlvmCovReport.isValid + get() = type == "llvm.coverage.json.export" + diff --git a/build.gradle b/build.gradle index 2683fd94cb7..71ea279c3db 100644 --- a/build.gradle +++ b/build.gradle @@ -104,7 +104,6 @@ void setupClang(Project project) { project.convention.plugins.platformManager = project.rootProject.ext.platformManager project.convention.plugins.execClang = new org.jetbrains.kotlin.ExecClang(project) - project.convention.plugins.execLlvm = new org.jetbrains.kotlin.ExecLlvm(project) project.plugins.withType(NativeComponentPlugin) { project.model {