azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Test] Add wrapping of failures from different parts of test pipeline

Browse Source
This commit is contained in:
Dmitriy Novozhilov
2021-05-14 14:53:36 +03:00
committed by TeamCityServer
parent 6aca0bb374
commit 27f0d938c9
10 changed files with 124 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/ExceptionFromTestError.kt
-14
View File
@@ -1,14 +0,0 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.test
class ExceptionFromTestError(cause: Throwable) : AssertionError(cause) {
override val message: String
get() = "Exception was thrown"
override val cause: Throwable
get() = super.cause!!
}
compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/TestRunner.kt
+55 -31
View File
@@ -13,7 +13,7 @@ import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
import java.io.IOException
class TestRunner(private val testConfiguration: TestConfiguration) {
private val failedAssertions = mutableListOf<Throwable>()
private val failedAssertions = mutableListOf<WrappedException>()
fun runTest(@TestDataFile testDataFileName: String, beforeDispose: (TestConfiguration) -> Unit = {}) {
try {
@@ -46,7 +46,9 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
}
} catch (e: ExceptionFromModuleStructureTransformer) {
services.register(TestModuleStructure::class, e.alreadyParsedModuleStructure)
val exception = filterFailedExceptions(listOf(e.cause)).singleOrNull() ?: return
val exception = filterFailedExceptions(
listOf(WrappedException.FromModuleStructureTransformer(e.cause))
).singleOrNull() ?: return
throw exception
}
@@ -60,18 +62,26 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
val modules = moduleStructure.modules
val dependencyProvider = DependencyProviderImpl(services, modules)
services.registerDependencyProvider(dependencyProvider)
var failedException: Throwable? = null
var failedException: WrappedException? = null
try {
for (module in modules) {
val shouldProcessNextModules = processModule(services, module, dependencyProvider, moduleStructure)
if (!shouldProcessNextModules) break
}
} catch (e: Throwable) {
} catch (e: WrappedException) {
failedException = e
} catch (e: Exception) {
throw IllegalStateException("Unexpected exception type. Only WrappedException are expected here", e)
}
for (handler in testConfiguration.getAllHandlers()) {
withAssertionCatching {
val wrapperFactory = when (handler) {
is FrontendOutputHandler -> WrappedException::FromFrontendHandler
is BackendInputHandler -> WrappedException::FromBackendHandler
is BinaryArtifactHandler -> WrappedException::FromBinaryHandler
else -> WrappedException::FromUnknownHandler
}
withAssertionCatching(wrapperFactory) {
val thereWasAnException = failedException != null || failedAssertions.isNotEmpty()
if (handler.shouldRun(thereWasAnException)) {
handler.processAfterAllModules(thereWasAnException)
@@ -79,32 +89,35 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
}
}
if (testConfiguration.metaInfoHandlerEnabled) {
withAssertionCatching(insertExceptionInStart = true) {
withAssertionCatching(WrappedException::FromMetaInfoHandler) {
globalMetadataInfoHandler.compareAllMetaDataInfos()
}
}
if (failedException != null) {
failedAssertions.add(0, ExceptionFromTestError(failedException))
failedAssertions.add(0, failedException)
}
testConfiguration.afterAnalysisCheckers.forEach {
withAssertionCatching {
withAssertionCatching(WrappedException::FromAfterAnalysisChecker) {
it.check(failedAssertions)
}
}
val filteredFailedAssertions = filterFailedExceptions(failedAssertions)
filteredFailedAssertions.firstIsInstanceOrNull<ExceptionFromTestError>()?.let {
filteredFailedAssertions.firstIsInstanceOrNull<WrappedException.FromFacade>()?.let {
throw it
}
services.assertions.assertAll(filteredFailedAssertions)
}
private fun filterFailedExceptions(failedExceptions: List<Throwable>): List<Throwable> = testConfiguration.afterAnalysisCheckers
.fold(failedExceptions) { assertions, checker ->
checker.suppressIfNeeded(assertions)
}
.map { if (it is ExceptionFromTestError) it.cause else it }
private fun filterFailedExceptions(failedExceptions: List<WrappedException>): List<Throwable> {
return testConfiguration.afterAnalysisCheckers
.fold(failedExceptions) { assertions, checker ->
checker.suppressIfNeeded(assertions)
}
.sorted()
.map { it.cause }
}
/*
* If there was failure from handler with `failureDisablesNextSteps=true` then `processModule`
@@ -121,11 +134,14 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
val frontendKind = module.frontendKind
if (!frontendKind.shouldRunAnalysis) return true
val frontendArtifacts: ResultingArtifact.FrontendOutput<*> = testConfiguration.getFacade(SourcesKind, frontendKind)
.transform(module, sourcesArtifact)?.also { dependencyProvider.registerArtifact(module, it) } ?: return true
val frontendArtifacts: ResultingArtifact.FrontendOutput<*> = withExceptionWrapping(WrappedException.FromFacade::Frontend) {
testConfiguration.getFacade(SourcesKind, frontendKind)
.transform(module, sourcesArtifact)?.also { dependencyProvider.registerArtifact(module, it) }
?: return true
}
val frontendHandlers: List<AnalysisHandler<*>> = testConfiguration.getHandlers(frontendKind)
for (frontendHandler in frontendHandlers) {
val thereWasAnException = withAssertionCatching {
val thereWasAnException = withAssertionCatching(WrappedException::FromFrontendHandler) {
if (frontendHandler.shouldRun(failedAssertions.isNotEmpty())) {
frontendHandler.hackyProcess(module, frontendArtifacts)
}
@@ -136,12 +152,14 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
val backendKind = services.backendKindExtractor.backendKind(module.targetBackend)
if (!backendKind.shouldRunAnalysis) return true
val backendInputInfo = testConfiguration.getFacade(frontendKind, backendKind)
.hackyTransform(module, frontendArtifacts)?.also { dependencyProvider.registerArtifact(module, it) } ?: return true
val backendInputInfo = withExceptionWrapping(WrappedException.FromFacade::Converter) {
testConfiguration.getFacade(frontendKind, backendKind)
.hackyTransform(module, frontendArtifacts)?.also { dependencyProvider.registerArtifact(module, it) } ?: return true
}
val backendHandlers: List<AnalysisHandler<*>> = testConfiguration.getHandlers(backendKind)
for (backendHandler in backendHandlers) {
val thereWasAnException = withAssertionCatching {
val thereWasAnException = withAssertionCatching(WrappedException::FromBackendHandler) {
if (backendHandler.shouldRun(failedAssertions.isNotEmpty())) {
backendHandler.hackyProcess(module, backendInputInfo)
}
@@ -151,14 +169,16 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
for (artifactKind in moduleStructure.getTargetArtifactKinds(module)) {
if (!artifactKind.shouldRunAnalysis) continue
val binaryArtifact = testConfiguration.getFacade(backendKind, artifactKind)
.hackyTransform(module, backendInputInfo)?.also {
dependencyProvider.registerArtifact(module, it)
} ?: return true
val binaryArtifact = withExceptionWrapping(WrappedException.FromFacade::Backend) {
testConfiguration.getFacade(backendKind, artifactKind)
.hackyTransform(module, backendInputInfo)?.also {
dependencyProvider.registerArtifact(module, it)
} ?: return true
}
val binaryHandlers: List<AnalysisHandler<*>> = testConfiguration.getHandlers(artifactKind)
for (binaryHandler in binaryHandlers) {
val thereWasAnException = withAssertionCatching {
val thereWasAnException = withAssertionCatching(WrappedException::FromBinaryHandler) {
if (binaryHandler.shouldRun(failedAssertions.isNotEmpty())) {
binaryHandler.hackyProcess(module, binaryArtifact)
}
@@ -173,20 +193,24 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
/*
* Returns true if there was an exception in block
*/
private inline fun withAssertionCatching(insertExceptionInStart: Boolean = false, block: () -> Unit): Boolean {
private inline fun withAssertionCatching(exceptionWrapper: (Throwable) -> WrappedException, block: () -> Unit): Boolean {
return try {
block()
false
} catch (e: Throwable) {
if (insertExceptionInStart) {
failedAssertions.add(0, e)
} else {
failedAssertions += e
}
failedAssertions += exceptionWrapper(e)
true
}
}
private inline fun <R> withExceptionWrapping(exceptionWrapper: (Throwable) -> WrappedException, block: () -> R): R {
return try {
block()
} catch (e: Throwable) {
throw exceptionWrapper(e)
}
}
private fun AnalysisHandler<*>.shouldRun(thereWasAnException: Boolean): Boolean {
return !(doNotRunIfThereWerePreviousFailures && thereWasAnException)
}
compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/WrappedException.kt
+42
View File
@@ -0,0 +1,42 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.test
sealed class WrappedException(
cause: Throwable,
val priority: Int,
val additionalPriority: Int
) : Exception(cause), Comparable<WrappedException> {
sealed class FromFacade(cause: Throwable, additionalPriority: Int) : WrappedException(cause, 0, additionalPriority) {
class Frontend(cause: Throwable) : FromFacade(cause, 1)
class Converter(cause: Throwable) : FromFacade(cause, 2)
class Backend(cause: Throwable) : FromFacade(cause, 3)
override val message: String
get() = "Exception was thrown"
}
class FromMetaInfoHandler(cause: Throwable) : WrappedException(cause, 1, 1)
class FromFrontendHandler(cause: Throwable) : WrappedException(cause, 1, 1)
class FromBackendHandler(cause: Throwable) : WrappedException(cause, 1, 2)
class FromBinaryHandler(cause: Throwable) : WrappedException(cause, 1, 3)
class FromUnknownHandler(cause: Throwable) : WrappedException(cause, 1, 4)
class FromAfterAnalysisChecker(cause: Throwable) : WrappedException(cause, 2, 1)
class FromModuleStructureTransformer(cause: Throwable) : WrappedException(cause, 2, 1)
override val cause: Throwable
get() = super.cause!!
override fun compareTo(other: WrappedException): Int {
if (priority == other.priority) {
return additionalPriority - other.additionalPriority
}
return priority - other.priority
}
}
compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/model/AfterAnalysisChecker.kt
+5 -2
View File
@@ -5,6 +5,7 @@
package org.jetbrains.kotlin.test.model
import org.jetbrains.kotlin.test.WrappedException
import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
import org.jetbrains.kotlin.test.services.TestServices
@@ -12,7 +13,9 @@ abstract class AfterAnalysisChecker(protected val testServices: TestServices) {
open val directives: List<DirectivesContainer>
get() = emptyList()
open fun check(failedAssertions: List<Throwable>) {}
open fun check(failedAssertions: List<WrappedException>) {}
open fun suppressIfNeeded(failedAssertions: List<Throwable>): List<Throwable> = failedAssertions
open fun suppressIfNeeded(failedAssertions: List<WrappedException>): List<WrappedException> = failedAssertions
protected fun Throwable.wrap(): WrappedException = WrappedException.FromAfterAnalysisChecker(this)
}
compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/BlackBoxCodegenSuppressor.kt
+7 -6
View File
@@ -6,6 +6,7 @@
package org.jetbrains.kotlin.test.backend
import org.jetbrains.kotlin.test.TargetBackend
import org.jetbrains.kotlin.test.WrappedException
import org.jetbrains.kotlin.test.directives.CodegenTestDirectives
import org.jetbrains.kotlin.test.directives.CodegenTestDirectives.IGNORE_BACKEND
import org.jetbrains.kotlin.test.directives.CodegenTestDirectives.IGNORE_BACKEND_FIR
@@ -24,7 +25,7 @@ class BlackBoxCodegenSuppressor(
get() = listOf(CodegenTestDirectives)
@OptIn(ExperimentalStdlibApi::class)
override fun suppressIfNeeded(failedAssertions: List<Throwable>): List<Throwable> {
override fun suppressIfNeeded(failedAssertions: List<WrappedException>): List<WrappedException> {
val moduleStructure = testServices.moduleStructure
val targetBackends = buildList {
testServices.defaultsProvider.defaultTargetBackend?.let {
@@ -45,8 +46,8 @@ class BlackBoxCodegenSuppressor(
moduleStructure: TestModuleStructure,
directive: ValueDirective<TargetBackend>,
targetBackends: List<TargetBackend>,
failedAssertions: List<Throwable>
): List<Throwable> {
failedAssertions: List<WrappedException>
): List<WrappedException> {
val ignoredBackends = moduleStructure.allDirectives[directive]
if (ignoredBackends.isEmpty()) return failedAssertions
val matchedBackend = ignoredBackends.intersect(targetBackends)
@@ -61,10 +62,10 @@ class BlackBoxCodegenSuppressor(
private fun processAssertions(
failedAssertions: List<Throwable>,
failedAssertions: List<WrappedException>,
directive: ValueDirective<TargetBackend>,
additionalMessage: String = ""
): List<Throwable> {
): List<WrappedException> {
return if (failedAssertions.isNotEmpty()) emptyList()
else {
val message = buildString {
@@ -74,7 +75,7 @@ class BlackBoxCodegenSuppressor(
append(additionalMessage)
}
}
listOf(AssertionError(message))
listOf(AssertionError(message).wrap())
}
}
}
compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/FirIrDumpIdenticalCheckers.kt
+2 -1
View File
@@ -5,6 +5,7 @@
package org.jetbrains.kotlin.test.backend.handlers
import org.jetbrains.kotlin.test.WrappedException
import org.jetbrains.kotlin.test.directives.FirDiagnosticsDirectives
import org.jetbrains.kotlin.test.directives.FirDiagnosticsDirectives.FIR_IDENTICAL
import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
@@ -39,7 +40,7 @@ class FirIrDumpIdenticalChecker(testServices: TestServices) : AfterAnalysisCheck
}
}
override fun check(failedAssertions: List<Throwable>) {
override fun check(failedAssertions: List<WrappedException>) {
if (failedAssertions.isNotEmpty()) return
val testDataFile = testServices.moduleStructure.originalTestDataFiles.first()
if (FIR_IDENTICAL in testServices.moduleStructure.allDirectives) {
compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/handlers/FirTestDataConsistencyHandler.kt
+2 -1
View File
@@ -6,6 +6,7 @@
package org.jetbrains.kotlin.test.frontend.classic.handlers
import org.jetbrains.kotlin.codeMetaInfo.clearTextFromDiagnosticMarkup
import org.jetbrains.kotlin.test.WrappedException
import org.jetbrains.kotlin.test.directives.FirDiagnosticsDirectives
import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
import org.jetbrains.kotlin.test.model.AfterAnalysisChecker
@@ -21,7 +22,7 @@ class FirTestDataConsistencyHandler(testServices: TestServices) : AfterAnalysisC
override val directives: List<DirectivesContainer>
get() = listOf(FirDiagnosticsDirectives)
override fun check(failedAssertions: List<Throwable>) {
override fun check(failedAssertions: List<WrappedException>) {
val moduleStructure = testServices.moduleStructure
val testData = moduleStructure.originalTestDataFiles.first()
if (testData.extension == "kts") return
compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirFailingTestSuppressor.kt
+4 -4
View File
@@ -5,20 +5,20 @@
package org.jetbrains.kotlin.test.frontend.fir
import org.jetbrains.kotlin.test.ExceptionFromTestError
import org.jetbrains.kotlin.test.WrappedException
import org.jetbrains.kotlin.test.model.AfterAnalysisChecker
import org.jetbrains.kotlin.test.services.TestServices
import org.jetbrains.kotlin.test.services.moduleStructure
class FirFailingTestSuppressor(testServices: TestServices) : AfterAnalysisChecker(testServices) {
override fun suppressIfNeeded(failedAssertions: List<Throwable>): List<Throwable> {
override fun suppressIfNeeded(failedAssertions: List<WrappedException>): List<WrappedException> {
val testFile = testServices.moduleStructure.originalTestDataFiles.first()
val failFile = testFile.parentFile.resolve("${testFile.nameWithoutExtension}.fail")
val exceptionFromFir = failedAssertions.firstOrNull { it is ExceptionFromTestError }
val exceptionFromFir = failedAssertions.firstOrNull { it is WrappedException.FromFacade.Frontend }
return when {
failFile.exists() && exceptionFromFir != null -> emptyList()
failFile.exists() && exceptionFromFir == null -> {
failedAssertions + AssertionError("Fail file exists but no exception was throw. Please remove ${failFile.name}")
failedAssertions + AssertionError("Fail file exists but no exception was throw. Please remove ${failFile.name}").wrap()
}
else -> failedAssertions
}
compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/handlers/FirIdenticalChecker.kt
+2 -1
View File
@@ -5,6 +5,7 @@
package org.jetbrains.kotlin.test.frontend.fir.handlers
import org.jetbrains.kotlin.test.WrappedException
import org.jetbrains.kotlin.test.model.AfterAnalysisChecker
import org.jetbrains.kotlin.test.services.TestServices
import org.jetbrains.kotlin.test.services.moduleStructure
@@ -25,7 +26,7 @@ class FirIdenticalChecker(testServices: TestServices) : AfterAnalysisChecker(tes
}
}
override fun check(failedAssertions: List<Throwable>) {
override fun check(failedAssertions: List<WrappedException>) {
if (failedAssertions.isNotEmpty()) return
val testDataFile = testServices.moduleStructure.originalTestDataFiles.first()
if (testDataFile.isFirTestData) {
idea/idea-frontend-fir/idea-fir-low-level-api/tests/org/jetbrains/kotlin/idea/fir/low/level/api/compiler/based/IdeTestIgnoreHandler.kt
+5 -4
View File
@@ -5,6 +5,7 @@
package org.jetbrains.kotlin.idea.fir.low.level.api.compiler.based
import org.jetbrains.kotlin.test.WrappedException
import org.jetbrains.kotlin.test.model.AfterAnalysisChecker
import org.jetbrains.kotlin.test.directives.FirDiagnosticsDirectives
import org.jetbrains.kotlin.test.services.TestServices
@@ -20,7 +21,7 @@ class IdeTestIgnoreHandler(testServices: TestServices) : AfterAnalysisChecker(te
override val directives: List<DirectivesContainer>
get() = listOf(FirIdeDirectives)
override fun check(failedAssertions: List<Throwable>) {
override fun check(failedAssertions: List<WrappedException>) {
if (!isFirIdeIgnoreDirectivePresent()) return
if (failedAssertions.isNotEmpty()) return
val moduleStructure = testServices.moduleStructure
@@ -45,9 +46,9 @@ class IdeTestIgnoreHandler(testServices: TestServices) : AfterAnalysisChecker(te
}
override fun suppressIfNeeded(failedAssertions: List<Throwable>): List<Throwable> {
override fun suppressIfNeeded(failedAssertions: List<WrappedException>): List<WrappedException> {
return if (isFirIdeIgnoreDirectivePresent())
failedAssertions.filterIsInstance<TestWithFirIdeIgnoreAnnotationPassesException>()
failedAssertions.filter{ it.cause is TestWithFirIdeIgnoreAnnotationPassesException }
else failedAssertions
}
@@ -57,4 +58,4 @@ class IdeTestIgnoreHandler(testServices: TestServices) : AfterAnalysisChecker(te
}
}
class TestWithFirIdeIgnoreAnnotationPassesException(override val message: String) : IllegalStateException()
class TestWithFirIdeIgnoreAnnotationPassesException(override val message: String) : IllegalStateException()