[Test] Add wrapping of failures from different parts of test pipeline

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/ExceptionFromTestError.kt

@@ -1,14 +0,0 @@
-/*
- * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.test
-
-class ExceptionFromTestError(cause: Throwable) : AssertionError(cause) {
-    override val message: String
-        get() = "Exception was thrown"
-
-    override val cause: Throwable
-        get() = super.cause!!
-}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/TestRunner.kt

@@ -13,7 +13,7 @@ import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
 import java.io.IOException
 
 class TestRunner(private val testConfiguration: TestConfiguration) {
-    private val failedAssertions = mutableListOf<Throwable>()
+    private val failedAssertions = mutableListOf<WrappedException>()
 
     fun runTest(@TestDataFile testDataFileName: String, beforeDispose: (TestConfiguration) -> Unit = {}) {
         try {
@@ -46,7 +46,9 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
             }
         } catch (e: ExceptionFromModuleStructureTransformer) {
             services.register(TestModuleStructure::class, e.alreadyParsedModuleStructure)
-            val exception = filterFailedExceptions(listOf(e.cause)).singleOrNull() ?: return
+            val exception = filterFailedExceptions(
+                listOf(WrappedException.FromModuleStructureTransformer(e.cause))
+            ).singleOrNull() ?: return
             throw exception
         }
 
@@ -60,18 +62,26 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
         val modules = moduleStructure.modules
         val dependencyProvider = DependencyProviderImpl(services, modules)
         services.registerDependencyProvider(dependencyProvider)
-        var failedException: Throwable? = null
+        var failedException: WrappedException? = null
         try {
             for (module in modules) {
                 val shouldProcessNextModules = processModule(services, module, dependencyProvider, moduleStructure)
                 if (!shouldProcessNextModules) break
             }
-        } catch (e: Throwable) {
+        } catch (e: WrappedException) {
             failedException = e
+        } catch (e: Exception) {
+            throw IllegalStateException("Unexpected exception type. Only WrappedException are expected here", e)
         }
 
         for (handler in testConfiguration.getAllHandlers()) {
-            withAssertionCatching {
+            val wrapperFactory = when (handler) {
+                is FrontendOutputHandler -> WrappedException::FromFrontendHandler
+                is BackendInputHandler -> WrappedException::FromBackendHandler
+                is BinaryArtifactHandler -> WrappedException::FromBinaryHandler
+                else -> WrappedException::FromUnknownHandler
+            }
+            withAssertionCatching(wrapperFactory) {
                 val thereWasAnException = failedException != null || failedAssertions.isNotEmpty()
                 if (handler.shouldRun(thereWasAnException)) {
                     handler.processAfterAllModules(thereWasAnException)
@@ -79,32 +89,35 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
             }
         }
         if (testConfiguration.metaInfoHandlerEnabled) {
-            withAssertionCatching(insertExceptionInStart = true) {
+            withAssertionCatching(WrappedException::FromMetaInfoHandler) {
                 globalMetadataInfoHandler.compareAllMetaDataInfos()
             }
         }
         if (failedException != null) {
-            failedAssertions.add(0, ExceptionFromTestError(failedException))
+            failedAssertions.add(0, failedException)
         }
 
         testConfiguration.afterAnalysisCheckers.forEach {
-            withAssertionCatching {
+            withAssertionCatching(WrappedException::FromAfterAnalysisChecker) {
                 it.check(failedAssertions)
             }
         }
 
         val filteredFailedAssertions = filterFailedExceptions(failedAssertions)
-        filteredFailedAssertions.firstIsInstanceOrNull<ExceptionFromTestError>()?.let {
+        filteredFailedAssertions.firstIsInstanceOrNull<WrappedException.FromFacade>()?.let {
             throw it
         }
         services.assertions.assertAll(filteredFailedAssertions)
     }
 
-    private fun filterFailedExceptions(failedExceptions: List<Throwable>): List<Throwable> = testConfiguration.afterAnalysisCheckers
+    private fun filterFailedExceptions(failedExceptions: List<WrappedException>): List<Throwable> {
-        .fold(failedExceptions) { assertions, checker ->
+        return testConfiguration.afterAnalysisCheckers
-            checker.suppressIfNeeded(assertions)
+            .fold(failedExceptions) { assertions, checker ->
-        }
+                checker.suppressIfNeeded(assertions)
-        .map { if (it is ExceptionFromTestError) it.cause else it }
+            }
+            .sorted()
+            .map { it.cause }
+    }
 
     /*
      * If there was failure from handler with `failureDisablesNextSteps=true` then `processModule`
@@ -121,11 +134,14 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
         val frontendKind = module.frontendKind
         if (!frontendKind.shouldRunAnalysis) return true
 
-        val frontendArtifacts: ResultingArtifact.FrontendOutput<*> = testConfiguration.getFacade(SourcesKind, frontendKind)
+        val frontendArtifacts: ResultingArtifact.FrontendOutput<*> = withExceptionWrapping(WrappedException.FromFacade::Frontend) {
-            .transform(module, sourcesArtifact)?.also { dependencyProvider.registerArtifact(module, it) } ?: return true
+            testConfiguration.getFacade(SourcesKind, frontendKind)
+                .transform(module, sourcesArtifact)?.also { dependencyProvider.registerArtifact(module, it) }
+                ?: return true
+        }
         val frontendHandlers: List<AnalysisHandler<*>> = testConfiguration.getHandlers(frontendKind)
         for (frontendHandler in frontendHandlers) {
-            val thereWasAnException = withAssertionCatching {
+            val thereWasAnException = withAssertionCatching(WrappedException::FromFrontendHandler) {
                 if (frontendHandler.shouldRun(failedAssertions.isNotEmpty())) {
                     frontendHandler.hackyProcess(module, frontendArtifacts)
                 }
@@ -136,12 +152,14 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
         val backendKind = services.backendKindExtractor.backendKind(module.targetBackend)
         if (!backendKind.shouldRunAnalysis) return true
 
-        val backendInputInfo = testConfiguration.getFacade(frontendKind, backendKind)
+        val backendInputInfo = withExceptionWrapping(WrappedException.FromFacade::Converter) {
-            .hackyTransform(module, frontendArtifacts)?.also { dependencyProvider.registerArtifact(module, it) } ?: return true
+            testConfiguration.getFacade(frontendKind, backendKind)
+                .hackyTransform(module, frontendArtifacts)?.also { dependencyProvider.registerArtifact(module, it) } ?: return true
+        }
 
         val backendHandlers: List<AnalysisHandler<*>> = testConfiguration.getHandlers(backendKind)
         for (backendHandler in backendHandlers) {
-            val thereWasAnException = withAssertionCatching {
+            val thereWasAnException = withAssertionCatching(WrappedException::FromBackendHandler) {
                 if (backendHandler.shouldRun(failedAssertions.isNotEmpty())) {
                     backendHandler.hackyProcess(module, backendInputInfo)
                 }
@@ -151,14 +169,16 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
 
         for (artifactKind in moduleStructure.getTargetArtifactKinds(module)) {
             if (!artifactKind.shouldRunAnalysis) continue
-            val binaryArtifact = testConfiguration.getFacade(backendKind, artifactKind)
+            val binaryArtifact = withExceptionWrapping(WrappedException.FromFacade::Backend) {
-                .hackyTransform(module, backendInputInfo)?.also {
+                testConfiguration.getFacade(backendKind, artifactKind)
-                    dependencyProvider.registerArtifact(module, it)
+                    .hackyTransform(module, backendInputInfo)?.also {
-                } ?: return true
+                        dependencyProvider.registerArtifact(module, it)
+                    } ?: return true
+            }
 
             val binaryHandlers: List<AnalysisHandler<*>> = testConfiguration.getHandlers(artifactKind)
             for (binaryHandler in binaryHandlers) {
-                val thereWasAnException = withAssertionCatching {
+                val thereWasAnException = withAssertionCatching(WrappedException::FromBinaryHandler) {
                     if (binaryHandler.shouldRun(failedAssertions.isNotEmpty())) {
                         binaryHandler.hackyProcess(module, binaryArtifact)
                     }
@@ -173,20 +193,24 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
     /*
      * Returns true if there was an exception in block
      */
-    private inline fun withAssertionCatching(insertExceptionInStart: Boolean = false, block: () -> Unit): Boolean {
+    private inline fun withAssertionCatching(exceptionWrapper: (Throwable) -> WrappedException, block: () -> Unit): Boolean {
         return try {
             block()
             false
         } catch (e: Throwable) {
-            if (insertExceptionInStart) {
+            failedAssertions += exceptionWrapper(e)
-                failedAssertions.add(0, e)
-            } else {
-                failedAssertions += e
-            }
             true
         }
     }
 
+    private inline fun <R> withExceptionWrapping(exceptionWrapper: (Throwable) -> WrappedException, block: () -> R): R {
+        return try {
+            block()
+        } catch (e: Throwable) {
+            throw exceptionWrapper(e)
+        }
+    }
+
     private fun AnalysisHandler<*>.shouldRun(thereWasAnException: Boolean): Boolean {
         return !(doNotRunIfThereWerePreviousFailures && thereWasAnException)
     }

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/WrappedException.kt

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test
+
+sealed class WrappedException(
+    cause: Throwable,
+    val priority: Int,
+    val additionalPriority: Int
+) : Exception(cause), Comparable<WrappedException> {
+    sealed class FromFacade(cause: Throwable, additionalPriority: Int) : WrappedException(cause, 0, additionalPriority) {
+        class Frontend(cause: Throwable) : FromFacade(cause, 1)
+        class Converter(cause: Throwable) : FromFacade(cause, 2)
+        class Backend(cause: Throwable) : FromFacade(cause, 3)
+
+        override val message: String
+            get() = "Exception was thrown"
+    }
+
+    class FromMetaInfoHandler(cause: Throwable) : WrappedException(cause, 1, 1)
+
+    class FromFrontendHandler(cause: Throwable) : WrappedException(cause, 1, 1)
+    class FromBackendHandler(cause: Throwable) : WrappedException(cause, 1, 2)
+    class FromBinaryHandler(cause: Throwable) : WrappedException(cause, 1, 3)
+    class FromUnknownHandler(cause: Throwable) : WrappedException(cause, 1, 4)
+
+    class FromAfterAnalysisChecker(cause: Throwable) : WrappedException(cause, 2, 1)
+
+    class FromModuleStructureTransformer(cause: Throwable) : WrappedException(cause, 2, 1)
+
+    override val cause: Throwable
+        get() = super.cause!!
+
+    override fun compareTo(other: WrappedException): Int {
+        if (priority == other.priority) {
+            return additionalPriority - other.additionalPriority
+        }
+        return priority - other.priority
+    }
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/model/AfterAnalysisChecker.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.kotlin.test.model
 
+import org.jetbrains.kotlin.test.WrappedException
 import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
 import org.jetbrains.kotlin.test.services.TestServices
 
@@ -12,7 +13,9 @@ abstract class AfterAnalysisChecker(protected val testServices: TestServices) {
     open val directives: List<DirectivesContainer>
         get() = emptyList()
 
-    open fun check(failedAssertions: List<Throwable>) {}
+    open fun check(failedAssertions: List<WrappedException>) {}
 
-    open fun suppressIfNeeded(failedAssertions: List<Throwable>): List<Throwable> = failedAssertions
+    open fun suppressIfNeeded(failedAssertions: List<WrappedException>): List<WrappedException> = failedAssertions
+
+    protected fun Throwable.wrap(): WrappedException = WrappedException.FromAfterAnalysisChecker(this)
 }

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/BlackBoxCodegenSuppressor.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.test.backend
 
 import org.jetbrains.kotlin.test.TargetBackend
+import org.jetbrains.kotlin.test.WrappedException
 import org.jetbrains.kotlin.test.directives.CodegenTestDirectives
 import org.jetbrains.kotlin.test.directives.CodegenTestDirectives.IGNORE_BACKEND
 import org.jetbrains.kotlin.test.directives.CodegenTestDirectives.IGNORE_BACKEND_FIR
@@ -24,7 +25,7 @@ class BlackBoxCodegenSuppressor(
         get() = listOf(CodegenTestDirectives)
 
     @OptIn(ExperimentalStdlibApi::class)
-    override fun suppressIfNeeded(failedAssertions: List<Throwable>): List<Throwable> {
+    override fun suppressIfNeeded(failedAssertions: List<WrappedException>): List<WrappedException> {
         val moduleStructure = testServices.moduleStructure
         val targetBackends = buildList {
             testServices.defaultsProvider.defaultTargetBackend?.let {
@@ -45,8 +46,8 @@ class BlackBoxCodegenSuppressor(
         moduleStructure: TestModuleStructure,
         directive: ValueDirective<TargetBackend>,
         targetBackends: List<TargetBackend>,
-        failedAssertions: List<Throwable>
+        failedAssertions: List<WrappedException>
-    ): List<Throwable> {
+    ): List<WrappedException> {
         val ignoredBackends = moduleStructure.allDirectives[directive]
         if (ignoredBackends.isEmpty()) return failedAssertions
         val matchedBackend = ignoredBackends.intersect(targetBackends)
@@ -61,10 +62,10 @@ class BlackBoxCodegenSuppressor(
 
 
     private fun processAssertions(
-        failedAssertions: List<Throwable>,
+        failedAssertions: List<WrappedException>,
         directive: ValueDirective<TargetBackend>,
         additionalMessage: String = ""
-    ): List<Throwable> {
+    ): List<WrappedException> {
         return if (failedAssertions.isNotEmpty()) emptyList()
         else {
             val message = buildString {
@@ -74,7 +75,7 @@ class BlackBoxCodegenSuppressor(
                     append(additionalMessage)
                 }
             }
-            listOf(AssertionError(message))
+            listOf(AssertionError(message).wrap())
         }
     }
 }

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/FirIrDumpIdenticalCheckers.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.kotlin.test.backend.handlers
 
+import org.jetbrains.kotlin.test.WrappedException
 import org.jetbrains.kotlin.test.directives.FirDiagnosticsDirectives
 import org.jetbrains.kotlin.test.directives.FirDiagnosticsDirectives.FIR_IDENTICAL
 import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
@@ -39,7 +40,7 @@ class FirIrDumpIdenticalChecker(testServices: TestServices) : AfterAnalysisCheck
         }
     }
 
-    override fun check(failedAssertions: List<Throwable>) {
+    override fun check(failedAssertions: List<WrappedException>) {
         if (failedAssertions.isNotEmpty()) return
         val testDataFile = testServices.moduleStructure.originalTestDataFiles.first()
         if (FIR_IDENTICAL in testServices.moduleStructure.allDirectives) {

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/handlers/FirTestDataConsistencyHandler.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.test.frontend.classic.handlers
 
 import org.jetbrains.kotlin.codeMetaInfo.clearTextFromDiagnosticMarkup
+import org.jetbrains.kotlin.test.WrappedException
 import org.jetbrains.kotlin.test.directives.FirDiagnosticsDirectives
 import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
 import org.jetbrains.kotlin.test.model.AfterAnalysisChecker
@@ -21,7 +22,7 @@ class FirTestDataConsistencyHandler(testServices: TestServices) : AfterAnalysisC
     override val directives: List<DirectivesContainer>
         get() = listOf(FirDiagnosticsDirectives)
 
-    override fun check(failedAssertions: List<Throwable>) {
+    override fun check(failedAssertions: List<WrappedException>) {
         val moduleStructure = testServices.moduleStructure
         val testData = moduleStructure.originalTestDataFiles.first()
         if (testData.extension == "kts") return

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirFailingTestSuppressor.kt

@@ -5,20 +5,20 @@
 
 package org.jetbrains.kotlin.test.frontend.fir
 
-import org.jetbrains.kotlin.test.ExceptionFromTestError
+import org.jetbrains.kotlin.test.WrappedException
 import org.jetbrains.kotlin.test.model.AfterAnalysisChecker
 import org.jetbrains.kotlin.test.services.TestServices
 import org.jetbrains.kotlin.test.services.moduleStructure
 
 class FirFailingTestSuppressor(testServices: TestServices) : AfterAnalysisChecker(testServices) {
-    override fun suppressIfNeeded(failedAssertions: List<Throwable>): List<Throwable> {
+    override fun suppressIfNeeded(failedAssertions: List<WrappedException>): List<WrappedException> {
         val testFile = testServices.moduleStructure.originalTestDataFiles.first()
         val failFile = testFile.parentFile.resolve("${testFile.nameWithoutExtension}.fail")
-        val exceptionFromFir = failedAssertions.firstOrNull { it is ExceptionFromTestError }
+        val exceptionFromFir = failedAssertions.firstOrNull { it is WrappedException.FromFacade.Frontend }
         return when {
             failFile.exists() && exceptionFromFir != null -> emptyList()
             failFile.exists() && exceptionFromFir == null -> {
-                failedAssertions + AssertionError("Fail file exists but no exception was throw. Please remove ${failFile.name}")
+                failedAssertions + AssertionError("Fail file exists but no exception was throw. Please remove ${failFile.name}").wrap()
             }
             else -> failedAssertions
         }

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/handlers/FirIdenticalChecker.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.kotlin.test.frontend.fir.handlers
 
+import org.jetbrains.kotlin.test.WrappedException
 import org.jetbrains.kotlin.test.model.AfterAnalysisChecker
 import org.jetbrains.kotlin.test.services.TestServices
 import org.jetbrains.kotlin.test.services.moduleStructure
@@ -25,7 +26,7 @@ class FirIdenticalChecker(testServices: TestServices) : AfterAnalysisChecker(tes
         }
     }
 
-    override fun check(failedAssertions: List<Throwable>) {
+    override fun check(failedAssertions: List<WrappedException>) {
         if (failedAssertions.isNotEmpty()) return
         val testDataFile = testServices.moduleStructure.originalTestDataFiles.first()
         if (testDataFile.isFirTestData) {

idea/idea-frontend-fir/idea-fir-low-level-api/tests/org/jetbrains/kotlin/idea/fir/low/level/api/compiler/based/IdeTestIgnoreHandler.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.kotlin.idea.fir.low.level.api.compiler.based
 
+import org.jetbrains.kotlin.test.WrappedException
 import org.jetbrains.kotlin.test.model.AfterAnalysisChecker
 import org.jetbrains.kotlin.test.directives.FirDiagnosticsDirectives
 import org.jetbrains.kotlin.test.services.TestServices
@@ -20,7 +21,7 @@ class IdeTestIgnoreHandler(testServices: TestServices) : AfterAnalysisChecker(te
     override val directives: List<DirectivesContainer>
         get() = listOf(FirIdeDirectives)
 
-    override fun check(failedAssertions: List<Throwable>) {
+    override fun check(failedAssertions: List<WrappedException>) {
         if (!isFirIdeIgnoreDirectivePresent()) return
         if (failedAssertions.isNotEmpty()) return
         val moduleStructure = testServices.moduleStructure
@@ -45,9 +46,9 @@ class IdeTestIgnoreHandler(testServices: TestServices) : AfterAnalysisChecker(te
     }
 
 
-    override fun suppressIfNeeded(failedAssertions: List<Throwable>): List<Throwable> {
+    override fun suppressIfNeeded(failedAssertions: List<WrappedException>): List<WrappedException> {
         return if (isFirIdeIgnoreDirectivePresent())
-            failedAssertions.filterIsInstance<TestWithFirIdeIgnoreAnnotationPassesException>()
+            failedAssertions.filter{ it.cause is TestWithFirIdeIgnoreAnnotationPassesException }
         else failedAssertions
     }
 
@@ -57,4 +58,4 @@ class IdeTestIgnoreHandler(testServices: TestServices) : AfterAnalysisChecker(te
     }
 }
 
-class TestWithFirIdeIgnoreAnnotationPassesException(override val message: String) : IllegalStateException()
+class TestWithFirIdeIgnoreAnnotationPassesException(override val message: String) : IllegalStateException()