azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[FIR] Refactor collecting statistics in modularized tests

Browse Source 
Also add printing statistics of best pass at the end of build log
This commit is contained in:
Dmitriy Novozhilov
2019-09-10 16:01:49 +03:00
parent 6a75a9072c
commit 40ebe4063c
3 changed files with 123 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractModularizedTest.kt
+1
View File
@@ -89,6 +89,7 @@ abstract class AbstractModularizedTest : KtUsefulTestCase() {
protected abstract fun beforePass()
protected abstract fun afterPass(pass: Int)
protected open fun afterAllPasses() {}
protected abstract fun processModule(moduleData: ModuleData): ProcessorAction
protected fun runTestOnce(pass: Int) {
compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FirResolveModularizedTotalKotlinTest.kt
+23 -9
View File
@@ -18,8 +18,6 @@ import org.jetbrains.kotlin.config.JVMConfigurationKeys
import org.jetbrains.kotlin.fir.builder.RawFirBuilder
import org.jetbrains.kotlin.fir.declarations.FirFile
import org.jetbrains.kotlin.fir.dump.MultiModuleHtmlFirDump
import org.jetbrains.kotlin.fir.resolve.FirProvider
import org.jetbrains.kotlin.fir.resolve.impl.FirProviderImpl
import org.jetbrains.kotlin.fir.resolve.transformers.FirTotalResolveTransformer
import org.jetbrains.kotlin.fir.scopes.ProcessorAction
import org.jetbrains.kotlin.test.ConfigurationKind
@@ -30,7 +28,6 @@ import java.io.FileOutputStream
import java.io.PrintStream
import java.text.SimpleDateFormat
import java.util.*
import kotlin.system.measureNanoTime
private const val FAIL_FAST = true
@@ -44,8 +41,10 @@ internal val PASSES = System.getProperty("fir.bench.passes")?.toInt() ?: 3
class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() {
private lateinit var bench: FirResolveBench
private lateinit var dump: MultiModuleHtmlFirDump
private lateinit var bench: FirResolveBench
private var bestStatistics: FirResolveBench.TotalStatistics? = null
private var bestPass: Int = 0
private fun runAnalysis(moduleData: ModuleData, environment: KotlinCoreEnvironment) {
val project = environment.project
@@ -129,14 +128,24 @@ class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() {
}
override fun afterPass(pass: Int) {
bench.report(System.out, errorTypeReports = false)
val statistics = bench.getTotalStatistics()
statistics.report(System.out, "Pass $pass", errorTypeReports = false)
saveReport(pass)
saveReport(pass, statistics)
if (statistics.totalTime < (bestStatistics?.totalTime ?: Long.MAX_VALUE)) {
bestStatistics = statistics
bestPass = pass
}
if (FAIL_FAST) {
bench.throwFailure()
}
}
override fun afterAllPasses() {
val bestStatistics = bestStatistics ?: return
printStatistics(bestStatistics, "Best pass: $bestPass", false)
}
private val folderDateFormat = SimpleDateFormat("yyyy-MM-dd")
private lateinit var startDate: Date
@@ -155,8 +164,12 @@ class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() {
reportDateFormat.format(startDate)
}
private fun saveReport(pass: Int) {
private fun saveReport(pass: Int, statistics: FirResolveBench.TotalStatistics) {
if (DUMP_FIR) dump.finish()
printStatistics(statistics, "PASS $pass")
}
private fun printStatistics(statistics: FirResolveBench.TotalStatistics, header: String, errorTypeReports: Boolean = true) {
PrintStream(
FileOutputStream(
reportDir().resolve("report-$reportDateStr.log"),
@@ -164,8 +177,8 @@ class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() {
)
).use { stream ->
val sep = "=".repeat(10)
stream.println("$sep PASS $pass $sep")
bench.report(stream)
stream.println("$sep $header $sep")
statistics.report(stream, header)
stream.println()
stream.println()
}
@@ -178,5 +191,6 @@ class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() {
bench = FirResolveBench(withProgress = false)
runTestOnce(i)
}
afterAllPasses()
}
}
compiler/fir/resolve/tests/org/jetbrains/kotlin/fir/FirResolveBench.kt
+99 -72
View File
@@ -32,10 +32,37 @@ fun checkFirProvidersConsistency(firFiles: List<FirFile>) {
}
private data class FailureInfo(val transformer: KClass<*>, val throwable: Throwable, val file: String)
private data class ErrorTypeReport(val report: String, var count: Int = 0)
data class ErrorTypeReport(val report: String, var count: Int = 0)
class FirResolveBench(val withProgress: Boolean) {
data class TotalStatistics(
val unresolvedTypes: Int,
val resolvedTypes: Int,
val errorTypes: Int,
val implicitTypes: Int,
val errorFunctionCallTypes: Int,
val errorQualifiedAccessTypes: Int,
val fileCount: Int,
val errorTypesReports: Map<String, ErrorTypeReport>,
val timePerTransformer: Map<String, Measure>
) {
val totalTypes: Int = unresolvedTypes + resolvedTypes
val goodTypes: Int = resolvedTypes - errorTypes - implicitTypes
val uniqueErrorTypes: Int = errorTypesReports.size
val totalMeasure = Measure().apply {
with(timePerTransformer.values) {
time = sumByLong { it.time }
user = sumByLong { it.user }
cpu = sumByLong { it.cpu }
gcTime = sumByLong { it.gcTime }
gcCollections = sumBy { it.gcCollections }
files = map { it.files }.average().toInt()
}
}
val totalTime: Long get() = totalMeasure.time
}
data class Measure(
var time: Long = 0,
@@ -44,7 +71,7 @@ class FirResolveBench(val withProgress: Boolean) {
var gcTime: Long = 0,
var gcCollections: Int = 0,
var files: Int = 0
) {}
)
val timePerTransformer = mutableMapOf<KClass<*>, Measure>()
var resolvedTypes = 0
@@ -54,6 +81,7 @@ class FirResolveBench(val withProgress: Boolean) {
var errorQualifiedAccessTypes = 0
var implicitTypes = 0
var fileCount = 0
var totalTime = 0L
private val fails = mutableListOf<FailureInfo>()
@@ -76,6 +104,8 @@ class FirResolveBench(val withProgress: Boolean) {
val diff = after - before
recordTime(builder::class, diff, time)
firFile!!
}.also {
totalTime = timePerTransformer.values.sumByLong { it.time }
}
}
@@ -228,75 +258,17 @@ class FirResolveBench(val withProgress: Boolean) {
}
}
fun report(stream: PrintStream, errorTypeReports: Boolean = true) {
if (errorTypeReports)
errorTypesReports.values.sortedByDescending { it.count }.forEach {
stream.print("${it.count}:")
stream.println(it.report)
}
infix fun Int.percentOf(other: Int): String {
return String.format("%.1f%%", this * 100.0 / other)
}
val totalTypes = unresolvedTypes + resolvedTypes
stream.println("UNRESOLVED (UNTOUCHED) IMPLICIT TYPES: $unresolvedTypes (${unresolvedTypes percentOf totalTypes})")
stream.println("RESOLVED TYPES: $resolvedTypes (${resolvedTypes percentOf totalTypes})")
val goodTypes = resolvedTypes - errorTypes - implicitTypes
stream.println("CORRECTLY RESOLVED TYPES: $goodTypes (${goodTypes percentOf resolvedTypes} of resolved)")
stream.println("ERRONEOUSLY RESOLVED TYPES: $errorTypes (${errorTypes percentOf resolvedTypes} of resolved)")
stream.println(" - unresolved calls: $errorFunctionCallTypes")
stream.println(" - unresolved q.accesses: $errorQualifiedAccessTypes")
stream.println("ERRONEOUSLY RESOLVED IMPLICIT TYPES: $implicitTypes (${implicitTypes percentOf resolvedTypes} of resolved)")
stream.println("UNIQUE ERROR TYPES: ${errorTypesReports.size}")
printTable(stream) {
row {
cell("Stage", LEFT)
cells("Time", "Time per file", "Files: OK/E/T", "CPU", "User", "GC", "GC count")
}
separator()
timePerTransformer.forEach { (transformer, measure) ->
val time = measure.time
val counter = measure.files
row {
cell(transformer.simpleName.toString(), LEFT)
timeCell(time, fractionDigits = 0)
timeCell(time / counter)
cell("$counter/${fileCount - counter}/$fileCount")
timeCell(measure.cpu, fractionDigits = 0)
timeCell(measure.user)
timeCell(measure.gcTime, inputUnit = TableTimeUnit.MS)
cell(measure.gcCollections.toString())
}
}
if (timePerTransformer.keys.size > 0) {
separator()
val totalTime = timePerTransformer.values.sumByLong { it.time }
val totalCpuTime = timePerTransformer.values.sumByLong { it.cpu }
val totalUserTime = timePerTransformer.values.sumByLong { it.user }
val totalGcTime = timePerTransformer.values.sumByLong { it.gcTime }
val totalGcCollections = timePerTransformer.values.sumBy { it.gcCollections }
val averageFiles = timePerTransformer.values.map { it.files }.average().toLong()
row {
cell("Total", LEFT)
timeCell(totalTime, fractionDigits = 0)
timeCell(totalTime / averageFiles)
cell("$averageFiles")
timeCell(totalCpuTime, fractionDigits = 0)
timeCell(totalUserTime)
timeCell(totalGcTime, inputUnit = TableTimeUnit.MS)
cell(totalGcCollections.toString())
}
}
}
}
fun getTotalStatistics(): TotalStatistics = TotalStatistics(
unresolvedTypes,
resolvedTypes,
errorTypes,
implicitTypes,
errorFunctionCallTypes,
errorQualifiedAccessTypes,
fileCount,
errorTypesReports,
timePerTransformer.mapKeys { (klass, _) -> klass.simpleName!!.toString() }
)
}
fun doFirResolveTestBench(
@@ -313,7 +285,7 @@ fun doFirResolveTestBench(
val bench = FirResolveBench(withProgress)
bench.processFiles(firFiles, transformers)
if (!silent) bench.report(System.out)
if (!silent) bench.getTotalStatistics().report(System.out, "")
bench.throwFailure()
}
@@ -345,4 +317,59 @@ fun <T> Collection<T>.progress(step: Double = 0.1, computeLabel: (T) -> String):
}
progress++
}
}
fun FirResolveBench.TotalStatistics.report(stream: PrintStream, header: String, errorTypeReports: Boolean = true) {
with(stream) {
if (errorTypeReports)
errorTypesReports.values.sortedByDescending { it.count }.forEach {
print("${it.count}:")
println(it.report)
}
infix fun Int.percentOf(other: Int): String {
return String.format("%.1f%%", this * 100.0 / other)
}
println()
println("---------- $header ----------")
println("Unresolved (untouched) implicit types: $unresolvedTypes (${unresolvedTypes percentOf totalTypes})")
println("Resolved types: $resolvedTypes (${resolvedTypes percentOf totalTypes})")
println("Correctly resolved types: $goodTypes (${goodTypes percentOf resolvedTypes} of resolved)")
println("Erroneously resolved types: $errorTypes (${errorTypes percentOf resolvedTypes} of resolved)")
println(" - unresolved calls: $errorFunctionCallTypes")
println(" - unresolved q.accesses: $errorQualifiedAccessTypes")
println("Erroneously resolved implicit types: $implicitTypes (${implicitTypes percentOf resolvedTypes} of resolved)")
println("Unique error types: $uniqueErrorTypes")
printTable(stream) {
row {
cell("Stage", LEFT)
cells("Time", "Time per file", "Files: OK/E/T", "CPU", "User", "GC", "GC count")
}
separator()
timePerTransformer.forEach { (transformer, measure) ->
printMeasureAsTable(measure, this@report, transformer)
}
if (timePerTransformer.keys.isNotEmpty()) {
separator()
printMeasureAsTable(totalMeasure, this@report, "Total time")
}
}
}
}
private fun RTableContext.printMeasureAsTable(measure: FirResolveBench.Measure, statistics: FirResolveBench.TotalStatistics, label: String) {
val time = measure.time
val counter = measure.files
row {
cell(label, LEFT)
timeCell(time, fractionDigits = 0)
timeCell(time / counter)
cell("$counter/${statistics.fileCount - counter}/${statistics.fileCount}")
timeCell(measure.cpu, fractionDigits = 0)
timeCell(measure.user)
timeCell(measure.gcTime, inputUnit = TableTimeUnit.MS)
cell(measure.gcCollections.toString())
}
}