[FIR] Refactor collecting statistics in modularized tests

Also add printing statistics of best pass at the end of build log

compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractModularizedTest.kt

@@ -89,6 +89,7 @@ abstract class AbstractModularizedTest : KtUsefulTestCase() {
 
     protected abstract fun beforePass()
     protected abstract fun afterPass(pass: Int)
+    protected open fun afterAllPasses() {}
     protected abstract fun processModule(moduleData: ModuleData): ProcessorAction
 
     protected fun runTestOnce(pass: Int) {

compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FirResolveModularizedTotalKotlinTest.kt

@@ -18,8 +18,6 @@ import org.jetbrains.kotlin.config.JVMConfigurationKeys
 import org.jetbrains.kotlin.fir.builder.RawFirBuilder
 import org.jetbrains.kotlin.fir.declarations.FirFile
 import org.jetbrains.kotlin.fir.dump.MultiModuleHtmlFirDump
-import org.jetbrains.kotlin.fir.resolve.FirProvider
-import org.jetbrains.kotlin.fir.resolve.impl.FirProviderImpl
 import org.jetbrains.kotlin.fir.resolve.transformers.FirTotalResolveTransformer
 import org.jetbrains.kotlin.fir.scopes.ProcessorAction
 import org.jetbrains.kotlin.test.ConfigurationKind
@@ -30,7 +28,6 @@ import java.io.FileOutputStream
 import java.io.PrintStream
 import java.text.SimpleDateFormat
 import java.util.*
-import kotlin.system.measureNanoTime
 
 
 private const val FAIL_FAST = true
@@ -44,8 +41,10 @@ internal val PASSES = System.getProperty("fir.bench.passes")?.toInt() ?: 3
 
 class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() {
 
-    private lateinit var bench: FirResolveBench
     private lateinit var dump: MultiModuleHtmlFirDump
+    private lateinit var bench: FirResolveBench
+    private var bestStatistics: FirResolveBench.TotalStatistics? = null
+    private var bestPass: Int = 0
 
     private fun runAnalysis(moduleData: ModuleData, environment: KotlinCoreEnvironment) {
         val project = environment.project
@@ -129,14 +128,24 @@ class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() {
     }
 
     override fun afterPass(pass: Int) {
-        bench.report(System.out, errorTypeReports = false)
+        val statistics = bench.getTotalStatistics()
+        statistics.report(System.out, "Pass $pass", errorTypeReports = false)
 
-        saveReport(pass)
+        saveReport(pass, statistics)
+        if (statistics.totalTime < (bestStatistics?.totalTime ?: Long.MAX_VALUE)) {
+            bestStatistics = statistics
+            bestPass = pass
+        }
         if (FAIL_FAST) {
             bench.throwFailure()
         }
     }
 
+    override fun afterAllPasses() {
+        val bestStatistics = bestStatistics ?: return
+        printStatistics(bestStatistics, "Best pass: $bestPass", false)
+    }
+
     private val folderDateFormat = SimpleDateFormat("yyyy-MM-dd")
     private lateinit var startDate: Date
 
@@ -155,8 +164,12 @@ class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() {
         reportDateFormat.format(startDate)
     }
 
-    private fun saveReport(pass: Int) {
+    private fun saveReport(pass: Int, statistics: FirResolveBench.TotalStatistics) {
         if (DUMP_FIR) dump.finish()
+        printStatistics(statistics, "PASS $pass")
+    }
+
+    private fun printStatistics(statistics: FirResolveBench.TotalStatistics, header: String, errorTypeReports: Boolean = true) {
         PrintStream(
             FileOutputStream(
                 reportDir().resolve("report-$reportDateStr.log"),
@@ -164,8 +177,8 @@ class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() {
             )
         ).use { stream ->
             val sep = "=".repeat(10)
-            stream.println("$sep PASS $pass $sep")
+            stream.println("$sep $header $sep")
-            bench.report(stream)
+            statistics.report(stream, header)
             stream.println()
             stream.println()
         }
@@ -178,5 +191,6 @@ class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() {
             bench = FirResolveBench(withProgress = false)
             runTestOnce(i)
         }
+        afterAllPasses()
     }
 }

compiler/fir/resolve/tests/org/jetbrains/kotlin/fir/FirResolveBench.kt

@@ -32,10 +32,37 @@ fun checkFirProvidersConsistency(firFiles: List<FirFile>) {
 }
 
 private data class FailureInfo(val transformer: KClass<*>, val throwable: Throwable, val file: String)
-private data class ErrorTypeReport(val report: String, var count: Int = 0)
+data class ErrorTypeReport(val report: String, var count: Int = 0)
 
 class FirResolveBench(val withProgress: Boolean) {
+    data class TotalStatistics(
+        val unresolvedTypes: Int,
+        val resolvedTypes: Int,
+        val errorTypes: Int,
+        val implicitTypes: Int,
+        val errorFunctionCallTypes: Int,
+        val errorQualifiedAccessTypes: Int,
+        val fileCount: Int,
+        val errorTypesReports: Map<String, ErrorTypeReport>,
+        val timePerTransformer: Map<String, Measure>
+    ) {
+        val totalTypes: Int = unresolvedTypes + resolvedTypes
+        val goodTypes: Int = resolvedTypes - errorTypes - implicitTypes
+        val uniqueErrorTypes: Int = errorTypesReports.size
 
+        val totalMeasure = Measure().apply {
+            with(timePerTransformer.values) {
+                time = sumByLong { it.time }
+                user = sumByLong { it.user }
+                cpu = sumByLong { it.cpu }
+                gcTime = sumByLong { it.gcTime }
+                gcCollections = sumBy { it.gcCollections }
+                files = map { it.files }.average().toInt()
+            }
+        }
+
+        val totalTime: Long get() = totalMeasure.time
+    }
 
     data class Measure(
         var time: Long = 0,
@@ -44,7 +71,7 @@ class FirResolveBench(val withProgress: Boolean) {
         var gcTime: Long = 0,
         var gcCollections: Int = 0,
         var files: Int = 0
-    ) {}
+    )
 
     val timePerTransformer = mutableMapOf<KClass<*>, Measure>()
     var resolvedTypes = 0
@@ -54,6 +81,7 @@ class FirResolveBench(val withProgress: Boolean) {
     var errorQualifiedAccessTypes = 0
     var implicitTypes = 0
     var fileCount = 0
+    var totalTime = 0L
 
 
     private val fails = mutableListOf<FailureInfo>()
@@ -76,6 +104,8 @@ class FirResolveBench(val withProgress: Boolean) {
             val diff = after - before
             recordTime(builder::class, diff, time)
             firFile!!
+        }.also {
+            totalTime = timePerTransformer.values.sumByLong { it.time }
         }
     }
 
@@ -228,75 +258,17 @@ class FirResolveBench(val withProgress: Boolean) {
         }
     }
 
-    fun report(stream: PrintStream, errorTypeReports: Boolean = true) {
+    fun getTotalStatistics(): TotalStatistics = TotalStatistics(
-
+        unresolvedTypes,
-        if (errorTypeReports)
+        resolvedTypes,
-            errorTypesReports.values.sortedByDescending { it.count }.forEach {
+        errorTypes,
-                stream.print("${it.count}:")
+        implicitTypes,
-                stream.println(it.report)
+        errorFunctionCallTypes,
-            }
+        errorQualifiedAccessTypes,
-
+        fileCount,
-        infix fun Int.percentOf(other: Int): String {
+        errorTypesReports,
-            return String.format("%.1f%%", this * 100.0 / other)
+        timePerTransformer.mapKeys { (klass, _) -> klass.simpleName!!.toString() }
-        }
+    )
-
-        val totalTypes = unresolvedTypes + resolvedTypes
-        stream.println("UNRESOLVED (UNTOUCHED) IMPLICIT TYPES: $unresolvedTypes (${unresolvedTypes percentOf totalTypes})")
-        stream.println("RESOLVED TYPES: $resolvedTypes (${resolvedTypes percentOf totalTypes})")
-        val goodTypes = resolvedTypes - errorTypes - implicitTypes
-        stream.println("CORRECTLY RESOLVED TYPES: $goodTypes (${goodTypes percentOf resolvedTypes} of resolved)")
-        stream.println("ERRONEOUSLY RESOLVED TYPES: $errorTypes (${errorTypes percentOf resolvedTypes} of resolved)")
-        stream.println("   - unresolved calls: $errorFunctionCallTypes")
-        stream.println("   - unresolved q.accesses: $errorQualifiedAccessTypes")
-        stream.println("ERRONEOUSLY RESOLVED IMPLICIT TYPES: $implicitTypes (${implicitTypes percentOf resolvedTypes} of resolved)")
-        stream.println("UNIQUE ERROR TYPES: ${errorTypesReports.size}")
-
-        printTable(stream) {
-            row {
-                cell("Stage", LEFT)
-                cells("Time", "Time per file", "Files: OK/E/T", "CPU", "User", "GC", "GC count")
-            }
-            separator()
-            timePerTransformer.forEach { (transformer, measure) ->
-                val time = measure.time
-                val counter = measure.files
-                row {
-                    cell(transformer.simpleName.toString(), LEFT)
-                    timeCell(time, fractionDigits = 0)
-                    timeCell(time / counter)
-                    cell("$counter/${fileCount - counter}/$fileCount")
-                    timeCell(measure.cpu, fractionDigits = 0)
-                    timeCell(measure.user)
-                    timeCell(measure.gcTime, inputUnit = TableTimeUnit.MS)
-                    cell(measure.gcCollections.toString())
-                }
-            }
-
-            if (timePerTransformer.keys.size > 0) {
-                separator()
-                val totalTime = timePerTransformer.values.sumByLong { it.time }
-                val totalCpuTime = timePerTransformer.values.sumByLong { it.cpu }
-                val totalUserTime = timePerTransformer.values.sumByLong { it.user }
-                val totalGcTime = timePerTransformer.values.sumByLong { it.gcTime }
-                val totalGcCollections = timePerTransformer.values.sumBy { it.gcCollections }
-                val averageFiles = timePerTransformer.values.map { it.files }.average().toLong()
-
-
-                row {
-                    cell("Total", LEFT)
-                    timeCell(totalTime, fractionDigits = 0)
-                    timeCell(totalTime / averageFiles)
-                    cell("$averageFiles")
-                    timeCell(totalCpuTime, fractionDigits = 0)
-                    timeCell(totalUserTime)
-                    timeCell(totalGcTime, inputUnit = TableTimeUnit.MS)
-                    cell(totalGcCollections.toString())
-                }
-            }
-
-        }
-
-    }
 }
 
 fun doFirResolveTestBench(
@@ -313,7 +285,7 @@ fun doFirResolveTestBench(
 
     val bench = FirResolveBench(withProgress)
     bench.processFiles(firFiles, transformers)
-    if (!silent) bench.report(System.out)
+    if (!silent) bench.getTotalStatistics().report(System.out, "")
     bench.throwFailure()
 }
 
@@ -346,3 +318,58 @@ fun <T> Collection<T>.progress(step: Double = 0.1, computeLabel: (T) -> String):
         progress++
     }
 }
+
+fun FirResolveBench.TotalStatistics.report(stream: PrintStream, header: String, errorTypeReports: Boolean = true) {
+    with(stream) {
+        if (errorTypeReports)
+            errorTypesReports.values.sortedByDescending { it.count }.forEach {
+                print("${it.count}:")
+                println(it.report)
+            }
+
+        infix fun Int.percentOf(other: Int): String {
+            return String.format("%.1f%%", this * 100.0 / other)
+        }
+        println()
+        println("---------- $header ----------")
+        println("Unresolved (untouched) implicit types: $unresolvedTypes (${unresolvedTypes percentOf totalTypes})")
+        println("Resolved types: $resolvedTypes (${resolvedTypes percentOf totalTypes})")
+        println("Correctly resolved types: $goodTypes (${goodTypes percentOf resolvedTypes} of resolved)")
+        println("Erroneously resolved types: $errorTypes (${errorTypes percentOf resolvedTypes} of resolved)")
+        println("   - unresolved calls: $errorFunctionCallTypes")
+        println("   - unresolved q.accesses: $errorQualifiedAccessTypes")
+        println("Erroneously resolved implicit types: $implicitTypes (${implicitTypes percentOf resolvedTypes} of resolved)")
+        println("Unique error types: $uniqueErrorTypes")
+
+        printTable(stream) {
+            row {
+                cell("Stage", LEFT)
+                cells("Time", "Time per file", "Files: OK/E/T", "CPU", "User", "GC", "GC count")
+            }
+            separator()
+            timePerTransformer.forEach { (transformer, measure) ->
+                printMeasureAsTable(measure, this@report, transformer)
+            }
+
+            if (timePerTransformer.keys.isNotEmpty()) {
+                separator()
+                printMeasureAsTable(totalMeasure, this@report, "Total time")
+            }
+        }
+    }
+}
+
+private fun RTableContext.printMeasureAsTable(measure: FirResolveBench.Measure, statistics: FirResolveBench.TotalStatistics, label: String) {
+    val time = measure.time
+    val counter = measure.files
+    row {
+        cell(label, LEFT)
+        timeCell(time, fractionDigits = 0)
+        timeCell(time / counter)
+        cell("$counter/${statistics.fileCount - counter}/${statistics.fileCount}")
+        timeCell(measure.cpu, fractionDigits = 0)
+        timeCell(measure.user)
+        timeCell(measure.gcTime, inputUnit = TableTimeUnit.MS)
+        cell(measure.gcCollections.toString())
+    }
+}