From 40ebe4063cd91cf56af88efb41d4cf315018ff94 Mon Sep 17 00:00:00 2001 From: Dmitriy Novozhilov Date: Tue, 10 Sep 2019 16:01:49 +0300 Subject: [PATCH] [FIR] Refactor collecting statistics in modularized tests Also add printing statistics of best pass at the end of build log --- .../kotlin/fir/AbstractModularizedTest.kt | 1 + .../FirResolveModularizedTotalKotlinTest.kt | 32 +++- .../jetbrains/kotlin/fir/FirResolveBench.kt | 171 ++++++++++-------- 3 files changed, 123 insertions(+), 81 deletions(-) diff --git a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractModularizedTest.kt b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractModularizedTest.kt index dce53b23b2a..7259b7d3957 100644 --- a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractModularizedTest.kt +++ b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractModularizedTest.kt @@ -89,6 +89,7 @@ abstract class AbstractModularizedTest : KtUsefulTestCase() { protected abstract fun beforePass() protected abstract fun afterPass(pass: Int) + protected open fun afterAllPasses() {} protected abstract fun processModule(moduleData: ModuleData): ProcessorAction protected fun runTestOnce(pass: Int) { diff --git a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FirResolveModularizedTotalKotlinTest.kt b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FirResolveModularizedTotalKotlinTest.kt index 0a5dcf19baf..48b88584fdd 100644 --- a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FirResolveModularizedTotalKotlinTest.kt +++ b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FirResolveModularizedTotalKotlinTest.kt @@ -18,8 +18,6 @@ import org.jetbrains.kotlin.config.JVMConfigurationKeys import org.jetbrains.kotlin.fir.builder.RawFirBuilder import org.jetbrains.kotlin.fir.declarations.FirFile import org.jetbrains.kotlin.fir.dump.MultiModuleHtmlFirDump -import org.jetbrains.kotlin.fir.resolve.FirProvider -import org.jetbrains.kotlin.fir.resolve.impl.FirProviderImpl import org.jetbrains.kotlin.fir.resolve.transformers.FirTotalResolveTransformer import org.jetbrains.kotlin.fir.scopes.ProcessorAction import org.jetbrains.kotlin.test.ConfigurationKind @@ -30,7 +28,6 @@ import java.io.FileOutputStream import java.io.PrintStream import java.text.SimpleDateFormat import java.util.* -import kotlin.system.measureNanoTime private const val FAIL_FAST = true @@ -44,8 +41,10 @@ internal val PASSES = System.getProperty("fir.bench.passes")?.toInt() ?: 3 class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() { - private lateinit var bench: FirResolveBench private lateinit var dump: MultiModuleHtmlFirDump + private lateinit var bench: FirResolveBench + private var bestStatistics: FirResolveBench.TotalStatistics? = null + private var bestPass: Int = 0 private fun runAnalysis(moduleData: ModuleData, environment: KotlinCoreEnvironment) { val project = environment.project @@ -129,14 +128,24 @@ class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() { } override fun afterPass(pass: Int) { - bench.report(System.out, errorTypeReports = false) + val statistics = bench.getTotalStatistics() + statistics.report(System.out, "Pass $pass", errorTypeReports = false) - saveReport(pass) + saveReport(pass, statistics) + if (statistics.totalTime < (bestStatistics?.totalTime ?: Long.MAX_VALUE)) { + bestStatistics = statistics + bestPass = pass + } if (FAIL_FAST) { bench.throwFailure() } } + override fun afterAllPasses() { + val bestStatistics = bestStatistics ?: return + printStatistics(bestStatistics, "Best pass: $bestPass", false) + } + private val folderDateFormat = SimpleDateFormat("yyyy-MM-dd") private lateinit var startDate: Date @@ -155,8 +164,12 @@ class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() { reportDateFormat.format(startDate) } - private fun saveReport(pass: Int) { + private fun saveReport(pass: Int, statistics: FirResolveBench.TotalStatistics) { if (DUMP_FIR) dump.finish() + printStatistics(statistics, "PASS $pass") + } + + private fun printStatistics(statistics: FirResolveBench.TotalStatistics, header: String, errorTypeReports: Boolean = true) { PrintStream( FileOutputStream( reportDir().resolve("report-$reportDateStr.log"), @@ -164,8 +177,8 @@ class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() { ) ).use { stream -> val sep = "=".repeat(10) - stream.println("$sep PASS $pass $sep") - bench.report(stream) + stream.println("$sep $header $sep") + statistics.report(stream, header) stream.println() stream.println() } @@ -178,5 +191,6 @@ class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() { bench = FirResolveBench(withProgress = false) runTestOnce(i) } + afterAllPasses() } } \ No newline at end of file diff --git a/compiler/fir/resolve/tests/org/jetbrains/kotlin/fir/FirResolveBench.kt b/compiler/fir/resolve/tests/org/jetbrains/kotlin/fir/FirResolveBench.kt index e0d743c8810..737b19dd2a3 100644 --- a/compiler/fir/resolve/tests/org/jetbrains/kotlin/fir/FirResolveBench.kt +++ b/compiler/fir/resolve/tests/org/jetbrains/kotlin/fir/FirResolveBench.kt @@ -32,10 +32,37 @@ fun checkFirProvidersConsistency(firFiles: List) { } private data class FailureInfo(val transformer: KClass<*>, val throwable: Throwable, val file: String) -private data class ErrorTypeReport(val report: String, var count: Int = 0) +data class ErrorTypeReport(val report: String, var count: Int = 0) class FirResolveBench(val withProgress: Boolean) { + data class TotalStatistics( + val unresolvedTypes: Int, + val resolvedTypes: Int, + val errorTypes: Int, + val implicitTypes: Int, + val errorFunctionCallTypes: Int, + val errorQualifiedAccessTypes: Int, + val fileCount: Int, + val errorTypesReports: Map, + val timePerTransformer: Map + ) { + val totalTypes: Int = unresolvedTypes + resolvedTypes + val goodTypes: Int = resolvedTypes - errorTypes - implicitTypes + val uniqueErrorTypes: Int = errorTypesReports.size + val totalMeasure = Measure().apply { + with(timePerTransformer.values) { + time = sumByLong { it.time } + user = sumByLong { it.user } + cpu = sumByLong { it.cpu } + gcTime = sumByLong { it.gcTime } + gcCollections = sumBy { it.gcCollections } + files = map { it.files }.average().toInt() + } + } + + val totalTime: Long get() = totalMeasure.time + } data class Measure( var time: Long = 0, @@ -44,7 +71,7 @@ class FirResolveBench(val withProgress: Boolean) { var gcTime: Long = 0, var gcCollections: Int = 0, var files: Int = 0 - ) {} + ) val timePerTransformer = mutableMapOf, Measure>() var resolvedTypes = 0 @@ -54,6 +81,7 @@ class FirResolveBench(val withProgress: Boolean) { var errorQualifiedAccessTypes = 0 var implicitTypes = 0 var fileCount = 0 + var totalTime = 0L private val fails = mutableListOf() @@ -76,6 +104,8 @@ class FirResolveBench(val withProgress: Boolean) { val diff = after - before recordTime(builder::class, diff, time) firFile!! + }.also { + totalTime = timePerTransformer.values.sumByLong { it.time } } } @@ -228,75 +258,17 @@ class FirResolveBench(val withProgress: Boolean) { } } - fun report(stream: PrintStream, errorTypeReports: Boolean = true) { - - if (errorTypeReports) - errorTypesReports.values.sortedByDescending { it.count }.forEach { - stream.print("${it.count}:") - stream.println(it.report) - } - - infix fun Int.percentOf(other: Int): String { - return String.format("%.1f%%", this * 100.0 / other) - } - - val totalTypes = unresolvedTypes + resolvedTypes - stream.println("UNRESOLVED (UNTOUCHED) IMPLICIT TYPES: $unresolvedTypes (${unresolvedTypes percentOf totalTypes})") - stream.println("RESOLVED TYPES: $resolvedTypes (${resolvedTypes percentOf totalTypes})") - val goodTypes = resolvedTypes - errorTypes - implicitTypes - stream.println("CORRECTLY RESOLVED TYPES: $goodTypes (${goodTypes percentOf resolvedTypes} of resolved)") - stream.println("ERRONEOUSLY RESOLVED TYPES: $errorTypes (${errorTypes percentOf resolvedTypes} of resolved)") - stream.println(" - unresolved calls: $errorFunctionCallTypes") - stream.println(" - unresolved q.accesses: $errorQualifiedAccessTypes") - stream.println("ERRONEOUSLY RESOLVED IMPLICIT TYPES: $implicitTypes (${implicitTypes percentOf resolvedTypes} of resolved)") - stream.println("UNIQUE ERROR TYPES: ${errorTypesReports.size}") - - printTable(stream) { - row { - cell("Stage", LEFT) - cells("Time", "Time per file", "Files: OK/E/T", "CPU", "User", "GC", "GC count") - } - separator() - timePerTransformer.forEach { (transformer, measure) -> - val time = measure.time - val counter = measure.files - row { - cell(transformer.simpleName.toString(), LEFT) - timeCell(time, fractionDigits = 0) - timeCell(time / counter) - cell("$counter/${fileCount - counter}/$fileCount") - timeCell(measure.cpu, fractionDigits = 0) - timeCell(measure.user) - timeCell(measure.gcTime, inputUnit = TableTimeUnit.MS) - cell(measure.gcCollections.toString()) - } - } - - if (timePerTransformer.keys.size > 0) { - separator() - val totalTime = timePerTransformer.values.sumByLong { it.time } - val totalCpuTime = timePerTransformer.values.sumByLong { it.cpu } - val totalUserTime = timePerTransformer.values.sumByLong { it.user } - val totalGcTime = timePerTransformer.values.sumByLong { it.gcTime } - val totalGcCollections = timePerTransformer.values.sumBy { it.gcCollections } - val averageFiles = timePerTransformer.values.map { it.files }.average().toLong() - - - row { - cell("Total", LEFT) - timeCell(totalTime, fractionDigits = 0) - timeCell(totalTime / averageFiles) - cell("$averageFiles") - timeCell(totalCpuTime, fractionDigits = 0) - timeCell(totalUserTime) - timeCell(totalGcTime, inputUnit = TableTimeUnit.MS) - cell(totalGcCollections.toString()) - } - } - - } - - } + fun getTotalStatistics(): TotalStatistics = TotalStatistics( + unresolvedTypes, + resolvedTypes, + errorTypes, + implicitTypes, + errorFunctionCallTypes, + errorQualifiedAccessTypes, + fileCount, + errorTypesReports, + timePerTransformer.mapKeys { (klass, _) -> klass.simpleName!!.toString() } + ) } fun doFirResolveTestBench( @@ -313,7 +285,7 @@ fun doFirResolveTestBench( val bench = FirResolveBench(withProgress) bench.processFiles(firFiles, transformers) - if (!silent) bench.report(System.out) + if (!silent) bench.getTotalStatistics().report(System.out, "") bench.throwFailure() } @@ -345,4 +317,59 @@ fun Collection.progress(step: Double = 0.1, computeLabel: (T) -> String): } progress++ } +} + +fun FirResolveBench.TotalStatistics.report(stream: PrintStream, header: String, errorTypeReports: Boolean = true) { + with(stream) { + if (errorTypeReports) + errorTypesReports.values.sortedByDescending { it.count }.forEach { + print("${it.count}:") + println(it.report) + } + + infix fun Int.percentOf(other: Int): String { + return String.format("%.1f%%", this * 100.0 / other) + } + println() + println("---------- $header ----------") + println("Unresolved (untouched) implicit types: $unresolvedTypes (${unresolvedTypes percentOf totalTypes})") + println("Resolved types: $resolvedTypes (${resolvedTypes percentOf totalTypes})") + println("Correctly resolved types: $goodTypes (${goodTypes percentOf resolvedTypes} of resolved)") + println("Erroneously resolved types: $errorTypes (${errorTypes percentOf resolvedTypes} of resolved)") + println(" - unresolved calls: $errorFunctionCallTypes") + println(" - unresolved q.accesses: $errorQualifiedAccessTypes") + println("Erroneously resolved implicit types: $implicitTypes (${implicitTypes percentOf resolvedTypes} of resolved)") + println("Unique error types: $uniqueErrorTypes") + + printTable(stream) { + row { + cell("Stage", LEFT) + cells("Time", "Time per file", "Files: OK/E/T", "CPU", "User", "GC", "GC count") + } + separator() + timePerTransformer.forEach { (transformer, measure) -> + printMeasureAsTable(measure, this@report, transformer) + } + + if (timePerTransformer.keys.isNotEmpty()) { + separator() + printMeasureAsTable(totalMeasure, this@report, "Total time") + } + } + } +} + +private fun RTableContext.printMeasureAsTable(measure: FirResolveBench.Measure, statistics: FirResolveBench.TotalStatistics, label: String) { + val time = measure.time + val counter = measure.files + row { + cell(label, LEFT) + timeCell(time, fractionDigits = 0) + timeCell(time / counter) + cell("$counter/${statistics.fileCount - counter}/${statistics.fileCount}") + timeCell(measure.cpu, fractionDigits = 0) + timeCell(measure.user) + timeCell(measure.gcTime, inputUnit = TableTimeUnit.MS) + cell(measure.gcCollections.toString()) + } } \ No newline at end of file