[Test] Add ability to stop test pipeline if there was an exception in handler

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/TestRunner.kt

@@ -61,7 +61,8 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
         var failedException: Throwable? = null
         try {
             for (module in modules) {
-                processModule(services, module, dependencyProvider, moduleStructure)
+                val shouldProcessNextModules = processModule(services, module, dependencyProvider, moduleStructure)
+                if (!shouldProcessNextModules) break
             }
         } catch (e: Throwable) {
             failedException = e
@@ -101,41 +102,47 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
         }
         .map { if (it is ExceptionFromTestError) it.cause else it }
 
+    /*
+     * If there was failure from handler with `failureDisablesNextSteps=true` then `processModule`
+     *   returns false which indicates that other modules should not be processed
+     */
     private fun processModule(
         services: TestServices,
         module: TestModule,
         dependencyProvider: DependencyProviderImpl,
         moduleStructure: TestModuleStructure
-    ) {
+    ): Boolean {
         val sourcesArtifact = ResultingArtifact.Source()
 
         val frontendKind = module.frontendKind
-        if (!frontendKind.shouldRunAnalysis) return
+        if (!frontendKind.shouldRunAnalysis) return true
 
         val frontendArtifacts: ResultingArtifact.FrontendOutput<*> = testConfiguration.getFacade(SourcesKind, frontendKind)
-            .transform(module, sourcesArtifact)?.also { dependencyProvider.registerArtifact(module, it) } ?: return
+            .transform(module, sourcesArtifact)?.also { dependencyProvider.registerArtifact(module, it) } ?: return true
         val frontendHandlers: List<AnalysisHandler<*>> = testConfiguration.getHandlers(frontendKind)
         for (frontendHandler in frontendHandlers) {
-            withAssertionCatching {
+            val thereWasAnException = withAssertionCatching {
                 if (frontendHandler.shouldRun(failedAssertions.isNotEmpty())) {
                     frontendHandler.hackyProcess(module, frontendArtifacts)
                 }
             }
+            if (thereWasAnException && frontendHandler.failureDisablesNextSteps) return false
         }
 
         val backendKind = services.backendKindExtractor.backendKind(module.targetBackend)
-        if (!backendKind.shouldRunAnalysis) return
+        if (!backendKind.shouldRunAnalysis) return true
 
         val backendInputInfo = testConfiguration.getFacade(frontendKind, backendKind)
-            .hackyTransform(module, frontendArtifacts)?.also { dependencyProvider.registerArtifact(module, it) } ?: return
+            .hackyTransform(module, frontendArtifacts)?.also { dependencyProvider.registerArtifact(module, it) } ?: return true
 
         val backendHandlers: List<AnalysisHandler<*>> = testConfiguration.getHandlers(backendKind)
         for (backendHandler in backendHandlers) {
-            withAssertionCatching {
+            val thereWasAnException = withAssertionCatching {
                 if (backendHandler.shouldRun(failedAssertions.isNotEmpty())) {
                     backendHandler.hackyProcess(module, backendInputInfo)
                 }
             }
+            if (thereWasAnException && backendHandler.failureDisablesNextSteps) return false
         }
 
         for (artifactKind in moduleStructure.getTargetArtifactKinds(module)) {
@@ -143,28 +150,36 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
             val binaryArtifact = testConfiguration.getFacade(backendKind, artifactKind)
                 .hackyTransform(module, backendInputInfo)?.also {
                     dependencyProvider.registerArtifact(module, it)
-                } ?: return
+                } ?: return true
 
             val binaryHandlers: List<AnalysisHandler<*>> = testConfiguration.getHandlers(artifactKind)
             for (binaryHandler in binaryHandlers) {
-                withAssertionCatching {
+                val thereWasAnException = withAssertionCatching {
                     if (binaryHandler.shouldRun(failedAssertions.isNotEmpty())) {
                         binaryHandler.hackyProcess(module, binaryArtifact)
                     }
                 }
+                if (thereWasAnException && binaryHandler.failureDisablesNextSteps) return false
             }
         }
+
+        return true
     }
 
-    private inline fun withAssertionCatching(insertExceptionInStart: Boolean = false, block: () -> Unit) {
-        try {
+    /*
+     * Returns true if there was an exception in block
+     */
+    private inline fun withAssertionCatching(insertExceptionInStart: Boolean = false, block: () -> Unit): Boolean {
+        return try {
             block()
+            false
         } catch (e: Throwable) {
             if (insertExceptionInStart) {
                 failedAssertions.add(0, e)
             } else {
                 failedAssertions += e
             }
+            true
         }
     }
 

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/model/AnalysisHandler.kt

@@ -13,6 +13,7 @@ import org.jetbrains.kotlin.test.services.assertions
 
 abstract class AnalysisHandler<A : ResultingArtifact<A>>(
     val testServices: TestServices,
+    val failureDisablesNextSteps: Boolean,
     val doNotRunIfThereWerePreviousFailures: Boolean
 ) {
     protected val assertions: Assertions
@@ -34,17 +35,20 @@ abstract class AnalysisHandler<A : ResultingArtifact<A>>(
 abstract class FrontendOutputHandler<R : ResultingArtifact.FrontendOutput<R>>(
     testServices: TestServices,
     override val artifactKind: FrontendKind<R>,
+    failureDisablesNextSteps: Boolean,
     doNotRunIfThereWerePreviousFailures: Boolean
-) : AnalysisHandler<R>(testServices, doNotRunIfThereWerePreviousFailures)
+) : AnalysisHandler<R>(testServices, failureDisablesNextSteps, doNotRunIfThereWerePreviousFailures)
 
 abstract class BackendInputHandler<I : ResultingArtifact.BackendInput<I>>(
     testServices: TestServices,
     override val artifactKind: BackendKind<I>,
+    failureDisablesNextSteps: Boolean,
     doNotRunIfThereWerePreviousFailures: Boolean
-) : AnalysisHandler<I>(testServices, doNotRunIfThereWerePreviousFailures)
+) : AnalysisHandler<I>(testServices, failureDisablesNextSteps, doNotRunIfThereWerePreviousFailures)
 
 abstract class BinaryArtifactHandler<A : ResultingArtifact.Binary<A>>(
     testServices: TestServices,
     override val artifactKind: BinaryKind<A>,
+    failureDisablesNextSteps: Boolean,
     doNotRunIfThereWerePreviousFailures: Boolean
-) : AnalysisHandler<A>(testServices, doNotRunIfThereWerePreviousFailures)
+) : AnalysisHandler<A>(testServices, failureDisablesNextSteps, doNotRunIfThereWerePreviousFailures)

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/AbstractIrHandler.kt

@@ -12,5 +12,6 @@ import org.jetbrains.kotlin.test.services.TestServices
 
 abstract class AbstractIrHandler(
     testServices: TestServices,
+    failureDisablesNextSteps: Boolean = false,
     doNotRunIfThereWerePreviousFailures: Boolean = false
-) : BackendInputHandler<IrBackendInput>(testServices, BackendKinds.IrBackend, doNotRunIfThereWerePreviousFailures)
+) : BackendInputHandler<IrBackendInput>(testServices, BackendKinds.IrBackend, failureDisablesNextSteps, doNotRunIfThereWerePreviousFailures)

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/BinaryArtifactHandlers.kt

@@ -12,27 +12,33 @@ import org.jetbrains.kotlin.test.services.TestServices
 
 abstract class JvmBinaryArtifactHandler(
     testServices: TestServices,
+    failureDisablesNextSteps: Boolean = false,
     doNotRunIfThereWerePreviousFailures: Boolean = false
 ) : BinaryArtifactHandler<BinaryArtifacts.Jvm>(
     testServices,
     ArtifactKinds.Jvm,
+    failureDisablesNextSteps,
     doNotRunIfThereWerePreviousFailures
 )
 
 abstract class JsBinaryArtifactHandler(
     testServices: TestServices,
+    failureDisablesNextSteps: Boolean = false,
     doNotRunIfThereWerePreviousFailures: Boolean = false
 ) : BinaryArtifactHandler<BinaryArtifacts.Js>(
     testServices,
     ArtifactKinds.Js,
+    failureDisablesNextSteps,
     doNotRunIfThereWerePreviousFailures
 )
 
 abstract class NativeBinaryArtifactHandler(
     testServices: TestServices,
+    failureDisablesNextSteps: Boolean = false,
     doNotRunIfThereWerePreviousFailures: Boolean = false
 ) : BinaryArtifactHandler<BinaryArtifacts.Native>(
     testServices,
     ArtifactKinds.Native,
+    failureDisablesNextSteps,
     doNotRunIfThereWerePreviousFailures
 )

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/NoCompilationErrorsHandler.kt

@@ -14,7 +14,10 @@ import org.jetbrains.kotlin.test.frontend.classic.handlers.ClassicFrontendAnalys
 import org.jetbrains.kotlin.test.model.TestModule
 import org.jetbrains.kotlin.test.services.TestServices
 
-class NoCompilationErrorsHandler(testServices: TestServices) : ClassicFrontendAnalysisHandler(testServices) {
+class NoCompilationErrorsHandler(testServices: TestServices) : ClassicFrontendAnalysisHandler(
+    testServices,
+    failureDisablesNextSteps = true
+) {
     override val directivesContainers: List<DirectivesContainer>
         get() = listOf(CodegenTestDirectives)
 

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/NoFirCompilationErrorsHandler.kt

@@ -21,7 +21,7 @@ import org.jetbrains.kotlin.test.frontend.fir.handlers.FirAnalysisHandler
 import org.jetbrains.kotlin.test.model.TestModule
 import org.jetbrains.kotlin.test.services.TestServices
 
-class NoFirCompilationErrorsHandler(testServices: TestServices) : FirAnalysisHandler(testServices) {
+class NoFirCompilationErrorsHandler(testServices: TestServices) : FirAnalysisHandler(testServices, failureDisablesNextSteps = true) {
     override val directivesContainers: List<DirectivesContainer>
         get() = listOf(CodegenTestDirectives)
 

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/handlers/ClassicFrontendAnalysisHandler.kt

@@ -12,10 +12,12 @@ import org.jetbrains.kotlin.test.services.TestServices
 
 abstract class ClassicFrontendAnalysisHandler(
     testServices: TestServices,
+    failureDisablesNextSteps: Boolean = false,
     doNotRunIfThereWerePreviousFailures: Boolean = false
 ) : FrontendOutputHandler<ClassicFrontendOutputArtifact>(
     testServices,
     FrontendKinds.ClassicFrontend,
+    failureDisablesNextSteps,
     doNotRunIfThereWerePreviousFailures
 )
 

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/handlers/FirAnalysisHandler.kt

@@ -13,8 +13,14 @@ import java.io.File
 
 abstract class FirAnalysisHandler(
     testServices: TestServices,
+    failureDisablesNextSteps: Boolean = false,
     doNotRunIfThereWerePreviousFailures: Boolean = false
-) : FrontendOutputHandler<FirOutputArtifact>(testServices, FrontendKinds.FIR, doNotRunIfThereWerePreviousFailures) {
+) : FrontendOutputHandler<FirOutputArtifact>(
+    testServices,
+    FrontendKinds.FIR,
+    failureDisablesNextSteps,
+    doNotRunIfThereWerePreviousFailures
+) {
     protected val File.nameWithoutFirExtension: String
         get() = nameWithoutExtension.removeSuffix(".fir")
 }