FIR full pipeline test: output build status per-module

compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractModularizedTest.kt

@@ -36,6 +36,10 @@ data class ModuleData(
     val classpath = rawClasspath.map { File(ROOT_PATH_PREFIX, it.removePrefix("/")) }
     val sources = rawSources.map { File(ROOT_PATH_PREFIX, it.removePrefix("/")) }
     val javaSourceRoots = rawJavaSourceRoots.map { File(ROOT_PATH_PREFIX, it.removePrefix("/")) }
+    lateinit var targetInfo: String
+    var compilationError: String? = null
+    var jvmInternalError: String? = null
+    var exceptionMessage: String = "NO MESSAGE"
 }
 
 private fun NodeList.toList(): List<Node> {

compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FullPipelineModularizedTest.kt

@@ -7,8 +7,7 @@ package org.jetbrains.kotlin.fir
 
 import com.intellij.util.io.delete
 import org.jetbrains.kotlin.cli.common.*
-import org.jetbrains.kotlin.cli.common.messages.MessageRenderer
+import org.jetbrains.kotlin.cli.common.messages.*
-import org.jetbrains.kotlin.cli.common.messages.PrintingMessageCollector
 import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
 import org.jetbrains.kotlin.config.Services
 import org.jetbrains.kotlin.fir.TableTimeUnit.MS
@@ -21,8 +20,10 @@ import java.nio.file.Files
 class FullPipelineModularizedTest : AbstractModularizedTest() {
     override fun beforePass(pass: Int) {
         totalPassResult = CumulativeTime()
-        totalModules = 0
+        totalModules.clear()
-        okModules = 0
+        okModules.clear()
+        errorModules.clear()
+        crashedModules.clear()
     }
 
     private data class CumulativeTime(
@@ -52,17 +53,19 @@ class FullPipelineModularizedTest : AbstractModularizedTest() {
     }
 
     private lateinit var totalPassResult: CumulativeTime
-    private var totalModules = 0
+    private val totalModules = mutableListOf<ModuleData>()
-    private var okModules = 0
+    private val okModules = mutableListOf<ModuleData>()
+    private val errorModules = mutableListOf<ModuleData>()
+    private val crashedModules = mutableListOf<ModuleData>()
 
     override fun afterPass(pass: Int) {
-        createReport()
+        createReport(finalReport = pass == PASSES - 1)
-        require(totalModules > 0) { "No modules were analyzed" }
+        require(totalModules.isNotEmpty()) { "No modules were analyzed" }
-        require(okModules > 0) { "All of $totalModules is failed" }
+        require(okModules.isNotEmpty()) { "All of $totalModules is failed" }
     }
 
-    private fun createReport() {
+    private fun createReport(finalReport: Boolean) {
-        formatReport(System.out)
+        formatReport(System.out, finalReport)
 
         PrintStream(
             FileOutputStream(
@@ -70,15 +73,22 @@ class FullPipelineModularizedTest : AbstractModularizedTest() {
                 true
             )
         ).use { stream ->
-            formatReport(stream)
+            formatReport(stream, finalReport)
             stream.println()
             stream.println()
         }
     }
 
-    private fun formatReport(stream: PrintStream) {
+    private fun String.shorten(): String {
-        stream.println("TOTAL MODULES: $totalModules")
+        val split = split("\n")
-        stream.println("OK MODULES: $okModules")
+        return split.mapIndexedNotNull { index, s ->
+            if (index < 4 || index >= split.size - 6) s else null
+        }.joinToString("\n")
+    }
+
+    private fun formatReport(stream: PrintStream, finalReport: Boolean) {
+        stream.println("TOTAL MODULES: ${totalModules.size}")
+        stream.println("OK MODULES: ${okModules.size}")
         val total = totalPassResult
         var totalGcTimeMs = 0L
         var totalGcCount = 0L
@@ -121,8 +131,70 @@ class FullPipelineModularizedTest : AbstractModularizedTest() {
             separator()
             phase("Total", total.totalTime(), total.files, total.lines)
         }
+
+        if (finalReport) {
+            with(stream) {
+                println()
+                println("SUCCESSFUL MODULES")
+                println("------------------")
+                println()
+                for (okModule in okModules) {
+                    println("${okModule.qualifiedName}: ${okModule.targetInfo}")
+                }
+                println()
+                println("COMPILATION ERRORS")
+                println("------------------")
+                println()
+                for (errorModule in errorModules.filter { it.jvmInternalError == null }) {
+                    println("${errorModule.qualifiedName}: ${errorModule.targetInfo}")
+                    println("        1st error: ${errorModule.compilationError}")
+                }
+                println()
+                println("JVM INTERNAL ERRORS")
+                println("------------------")
+                println()
+                for (errorModule in errorModules.filter { it.jvmInternalError != null }) {
+                    println("${errorModule.qualifiedName}: ${errorModule.targetInfo}")
+                    println("        1st error: ${errorModule.jvmInternalError?.shorten()}")
+                }
+                val crashedModuleGroups = crashedModules.groupBy { it.exceptionMessage.take(60) }
+                for (modules in crashedModuleGroups.values) {
+                    println()
+                    println(modules.first().exceptionMessage)
+                    println("--------------------------------------------------------")
+                    println()
+                    for (module in modules) {
+                        println("${module.qualifiedName}: ${module.targetInfo}")
+                        println("        ${module.exceptionMessage}")
+                    }
+                }
+            }
+        }
     }
 
+    private class TestMessageCollector : MessageCollector {
+
+        data class Message(val severity: CompilerMessageSeverity, val message: String, val location: CompilerMessageSourceLocation?)
+
+        val messages = arrayListOf<Message>()
+
+        override fun clear() {
+            messages.clear()
+        }
+
+        override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageSourceLocation?) {
+            messages.add(Message(severity, message, location))
+            if (severity in CompilerMessageSeverity.VERBOSE) return
+            println(MessageRenderer.GRADLE_STYLE.render(severity, message, location))
+        }
+
+        override fun hasErrors(): Boolean = messages.any {
+            it.severity == CompilerMessageSeverity.EXCEPTION || it.severity == CompilerMessageSeverity.ERROR
+        }
+    }
+
+    private fun Throwable.deepestCause(): Throwable = cause?.deepestCause() ?: this
+
     override fun processModule(moduleData: ModuleData): ProcessorAction {
         val compiler = K2JVMCompiler()
         val args = compiler.createArguments()
@@ -137,8 +209,9 @@ class FullPipelineModularizedTest : AbstractModularizedTest() {
         args.destination = tmp.toAbsolutePath().toFile().toString()
         val manager = CompilerPerformanceManager()
         val services = Services.Builder().register(CommonCompilerPerformanceManager::class.java, manager).build()
+        val collector = TestMessageCollector()
         val result = try {
-            compiler.exec(PrintingMessageCollector(System.out, MessageRenderer.GRADLE_STYLE, false), services, args)
+            compiler.exec(collector, services, args)
         } catch (e: Exception) {
             e.printStackTrace()
             ExitCode.INTERNAL_ERROR
@@ -148,12 +221,32 @@ class FullPipelineModularizedTest : AbstractModularizedTest() {
         PerformanceCounter.resetAllCounters()
 
         tmp.delete(recursively = true)
-        totalModules++
+        totalModules += moduleData
+        moduleData.targetInfo = manager.getTargetInfo()
 
         return when (result) {
             ExitCode.OK -> {
                 totalPassResult += resultTime
-                okModules++
+                okModules += moduleData
+                ProcessorAction.NEXT
+            }
+            ExitCode.COMPILATION_ERROR -> {
+                errorModules += moduleData
+                moduleData.compilationError = collector.messages.firstOrNull {
+                    it.severity == CompilerMessageSeverity.ERROR
+                }?.message
+                moduleData.jvmInternalError = collector.messages.firstOrNull {
+                    it.severity == CompilerMessageSeverity.EXCEPTION
+                }?.message
+                ProcessorAction.NEXT
+            }
+            ExitCode.INTERNAL_ERROR -> {
+                crashedModules += moduleData
+                moduleData.exceptionMessage = collector.messages.firstOrNull() {
+                    it.severity == CompilerMessageSeverity.EXCEPTION
+                }?.message?.split("\n")?.let {
+                    it.lastOrNull { it.startsWith("Caused by: ") } ?: it.firstOrNull()
+                } ?: "NO MESSAGE"
                 ProcessorAction.NEXT
             }
             else -> ProcessorAction.NEXT