[Test] Replace three fixed phases of tests with step system

Now each test is just a sequence of any number of different steps. There
  are two kinds of steps:
1. Run some facade
2. Run handlers of specific artifact kind

Through the test each module passed to each step (if it is compatible
  by kinds of artifacts)
This commit is contained in:
Dmitriy Novozhilov
2021-07-20 16:54:37 +03:00
committed by teamcityserver
parent ba48f80e53
commit a66f3d26fd
37 changed files with 900 additions and 559 deletions
@@ -1,5 +1,5 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
@@ -8,7 +8,9 @@ package org.jetbrains.kotlin.test
import com.intellij.openapi.Disposable
import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
import org.jetbrains.kotlin.test.model.*
import org.jetbrains.kotlin.test.model.AfterAnalysisChecker
import org.jetbrains.kotlin.test.model.ResultingArtifact
import org.jetbrains.kotlin.test.model.TestModule
import org.jetbrains.kotlin.test.services.MetaTestConfigurator
import org.jetbrains.kotlin.test.services.ModuleStructureExtractor
import org.jetbrains.kotlin.test.services.PreAnalysisHandler
@@ -33,16 +35,11 @@ abstract class TestConfiguration {
abstract val afterAnalysisCheckers: List<AfterAnalysisChecker>
abstract val startingArtifactFactory: (TestModule) -> ResultingArtifact<*>
abstract val steps: List<TestStep<*, *>>
abstract val metaInfoHandlerEnabled: Boolean
abstract fun <I : ResultingArtifact<I>, O : ResultingArtifact<O>> getFacade(
inputKind: TestArtifactKind<I>,
outputKind: TestArtifactKind<O>
): AbstractTestFacade<I, O>
abstract fun <A : ResultingArtifact<A>> getHandlers(artifactKind: TestArtifactKind<A>): List<AnalysisHandler<A>>
abstract fun getAllHandlers(): List<AnalysisHandler<*>>
}
// ---------------------------- Utils ----------------------------
@@ -1,5 +1,5 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
@@ -7,13 +7,22 @@ package org.jetbrains.kotlin.test
import com.intellij.openapi.util.Disposer
import com.intellij.testFramework.TestDataFile
import org.jetbrains.kotlin.test.model.*
import org.jetbrains.kotlin.test.model.AnalysisHandler
import org.jetbrains.kotlin.test.model.ResultingArtifact
import org.jetbrains.kotlin.test.model.TestModule
import org.jetbrains.kotlin.test.services.*
import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
import java.io.IOException
class TestRunner(private val testConfiguration: TestConfiguration) {
private val failedAssertions = mutableListOf<WrappedException>()
companion object {
fun AnalysisHandler<*>.shouldRun(thereWasAnException: Boolean): Boolean {
return !(doNotRunIfThereWerePreviousFailures && thereWasAnException)
}
}
private val allFailedExceptions = mutableListOf<WrappedException>()
private val allRanHandlers = mutableSetOf<AnalysisHandler<*>>()
fun runTest(@TestDataFile testDataFileName: String, beforeDispose: (TestConfiguration) -> Unit = {}) {
try {
@@ -68,27 +77,15 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
preprocessor.preprocessModuleStructure(moduleStructure)
}
var failedException: WrappedException? = null
try {
for (module in modules) {
val shouldProcessNextModules = processModule(services, module, dependencyProvider, moduleStructure)
if (!shouldProcessNextModules) break
}
} catch (e: WrappedException) {
failedException = e
} catch (e: Exception) {
throw IllegalStateException("Unexpected exception type. Only WrappedException are expected here", e)
for (module in modules) {
val shouldProcessNextModules = processModule(module, dependencyProvider)
if (!shouldProcessNextModules) break
}
for (handler in testConfiguration.getAllHandlers()) {
val wrapperFactory = when (handler) {
is FrontendOutputHandler -> WrappedException::FromFrontendHandler
is BackendInputHandler -> WrappedException::FromBackendHandler
is BinaryArtifactHandler -> WrappedException::FromBinaryHandler
else -> WrappedException::FromUnknownHandler
}
for (handler in allRanHandlers) {
val wrapperFactory: (Throwable) -> WrappedException = { WrappedException.FromHandler(it, handler) }
withAssertionCatching(wrapperFactory) {
val thereWasAnException = failedException != null || failedAssertions.isNotEmpty()
val thereWasAnException = allFailedExceptions.isNotEmpty()
if (handler.shouldRun(thereWasAnException)) {
handler.processAfterAllModules(thereWasAnException)
}
@@ -99,100 +96,56 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
globalMetadataInfoHandler.compareAllMetaDataInfos()
}
}
if (failedException != null) {
failedAssertions.add(0, failedException)
}
testConfiguration.afterAnalysisCheckers.forEach {
withAssertionCatching(WrappedException::FromAfterAnalysisChecker) {
it.check(failedAssertions)
it.check(allFailedExceptions)
}
}
val filteredFailedAssertions = filterFailedExceptions(failedAssertions)
val filteredFailedAssertions = filterFailedExceptions(allFailedExceptions)
filteredFailedAssertions.firstIsInstanceOrNull<WrappedException.FromFacade>()?.let {
throw it
}
services.assertions.assertAll(filteredFailedAssertions)
}
private fun filterFailedExceptions(failedExceptions: List<WrappedException>): List<Throwable> {
return testConfiguration.afterAnalysisCheckers
.fold(failedExceptions) { assertions, checker ->
checker.suppressIfNeeded(assertions)
}
.sorted()
.map { it.cause }
}
/*
* If there was failure from handler with `failureDisablesNextSteps=true` then `processModule`
* returns false which indicates that other modules should not be processed
* Returns false if next modules should be not processed
*/
private fun processModule(
services: TestServices,
module: TestModule,
dependencyProvider: DependencyProviderImpl,
moduleStructure: TestModuleStructure
dependencyProvider: DependencyProviderImpl
): Boolean {
val sourcesArtifact = ResultingArtifact.Source()
var inputArtifact = testConfiguration.startingArtifactFactory.invoke(module)
val frontendKind = module.frontendKind
if (!frontendKind.shouldRunAnalysis) return true
for (step in testConfiguration.steps) {
if (!step.shouldProcessModule(module, inputArtifact)) continue
val frontendArtifacts: ResultingArtifact.FrontendOutput<*> = withExceptionWrapping(WrappedException.FromFacade::Frontend) {
testConfiguration.getFacade(SourcesKind, frontendKind)
.transform(module, sourcesArtifact)?.also { dependencyProvider.registerArtifact(module, it) }
?: return true
}
val frontendHandlers: List<AnalysisHandler<*>> = testConfiguration.getHandlers(frontendKind)
for (frontendHandler in frontendHandlers) {
val thereWasAnException = withAssertionCatching(WrappedException::FromFrontendHandler) {
if (frontendHandler.shouldRun(failedAssertions.isNotEmpty())) {
frontendHandler.hackyProcess(module, frontendArtifacts)
when (val result = step.hackyProcessModule(module, inputArtifact, allFailedExceptions.isNotEmpty())) {
is TestStep.StepResult.Artifact<*> -> {
require(step is TestStep.FacadeStep<*, *>)
if (step.inputArtifactKind != step.outputArtifactKind) {
dependencyProvider.registerArtifact(module, result.outputArtifact)
}
inputArtifact = result.outputArtifact
}
}
if (thereWasAnException && frontendHandler.failureDisablesNextSteps) return false
}
val backendKind = services.backendKindExtractor.backendKind(module.targetBackend)
if (!backendKind.shouldRunAnalysis) return true
val backendInputInfo = withExceptionWrapping(WrappedException.FromFacade::Converter) {
testConfiguration.getFacade(frontendKind, backendKind)
.hackyTransform(module, frontendArtifacts)?.also { dependencyProvider.registerArtifact(module, it) } ?: return true
}
val backendHandlers: List<AnalysisHandler<*>> = testConfiguration.getHandlers(backendKind)
for (backendHandler in backendHandlers) {
val thereWasAnException = withAssertionCatching(WrappedException::FromBackendHandler) {
if (backendHandler.shouldRun(failedAssertions.isNotEmpty())) {
backendHandler.hackyProcess(module, backendInputInfo)
is TestStep.StepResult.ErrorFromFacade -> {
allFailedExceptions += result.exception
return false
}
}
if (thereWasAnException && backendHandler.failureDisablesNextSteps) return false
}
for (artifactKind in moduleStructure.getTargetArtifactKinds(module)) {
if (!artifactKind.shouldRunAnalysis) continue
val binaryArtifact = withExceptionWrapping(WrappedException.FromFacade::Backend) {
testConfiguration.getFacade(backendKind, artifactKind)
.hackyTransform(module, backendInputInfo)?.also {
dependencyProvider.registerArtifact(module, it)
} ?: return true
}
val binaryHandlers: List<AnalysisHandler<*>> = testConfiguration.getHandlers(artifactKind)
for (binaryHandler in binaryHandlers) {
val thereWasAnException = withAssertionCatching(WrappedException::FromBinaryHandler) {
if (binaryHandler.shouldRun(failedAssertions.isNotEmpty())) {
binaryHandler.hackyProcess(module, binaryArtifact)
is TestStep.StepResult.HandlersResult -> {
val (exceptionsFromHandlers, shouldRunNextSteps) = result
require(step is TestStep.HandlersStep<*>)
allRanHandlers += step.handlers
allFailedExceptions += exceptionsFromHandlers
if (!shouldRunNextSteps) {
return false
}
}
if (thereWasAnException && binaryHandler.failureDisablesNextSteps) return false
is TestStep.StepResult.NoArtifactFromFacade -> return false
}
}
return true
}
@@ -204,57 +157,38 @@ class TestRunner(private val testConfiguration: TestConfiguration) {
block()
false
} catch (e: Throwable) {
failedAssertions += exceptionWrapper(e)
allFailedExceptions += exceptionWrapper(e)
true
}
}
private inline fun <R> withExceptionWrapping(exceptionWrapper: (Throwable) -> WrappedException, block: () -> R): R {
return try {
block()
} catch (e: Throwable) {
throw exceptionWrapper(e)
}
private fun filterFailedExceptions(failedExceptions: List<WrappedException>): List<Throwable> {
return testConfiguration.afterAnalysisCheckers
.fold(failedExceptions) { assertions, checker ->
checker.suppressIfNeeded(assertions)
}
.sorted()
.map { it.cause }
}
private fun AnalysisHandler<*>.shouldRun(thereWasAnException: Boolean): Boolean {
return !(doNotRunIfThereWerePreviousFailures && thereWasAnException)
// -------------------------------------- hacks --------------------------------------
private fun TestStep<*, *>.hackyProcessModule(
module: TestModule,
inputArtifact: ResultingArtifact<*>,
thereWereExceptionsOnPreviousSteps: Boolean
): TestStep.StepResult<*> {
@Suppress("UNCHECKED_CAST")
return (this as TestStep<ResultingArtifact.Source, *>)
.processModule(module, inputArtifact as ResultingArtifact<ResultingArtifact.Source>, thereWereExceptionsOnPreviousSteps)
}
private fun <I : ResultingArtifact<I>> TestStep<I, *>.processModule(
module: TestModule,
artifact: ResultingArtifact<I>,
thereWereExceptionsOnPreviousSteps: Boolean
): TestStep.StepResult<*> {
@Suppress("UNCHECKED_CAST")
return processModule(module, artifact as I, thereWereExceptionsOnPreviousSteps)
}
}
// ----------------------------------------------------------------------------------------------------------------
/*
* Those `hackyProcess` methods are needed to hack kotlin type system. In common test case
* we have artifact of type ResultingArtifact<*> and handler of type AnalysisHandler<*> and actually
* at runtime types under `*` are same (that achieved by grouping handlers and facades by
* frontend/backend/artifact kind). But there is no way to tell that to compiler, so I unsafely cast types with `*`
* to types with Empty artifacts to make it compile. Since unsafe cast has no effort at runtime, it's safe to use it
*/
private fun AnalysisHandler<*>.hackyProcess(module: TestModule, artifact: ResultingArtifact<*>) {
@Suppress("UNCHECKED_CAST")
(this as AnalysisHandler<ResultingArtifact.Source>)
.processModule(module, artifact as ResultingArtifact<ResultingArtifact.Source>)
}
private fun <A : ResultingArtifact<A>> AnalysisHandler<A>.processModule(module: TestModule, artifact: ResultingArtifact<A>) {
@Suppress("UNCHECKED_CAST")
processModule(module, artifact as A)
}
private fun AbstractTestFacade<*, *>.hackyTransform(
module: TestModule,
artifact: ResultingArtifact<*>
): ResultingArtifact<*>? {
@Suppress("UNCHECKED_CAST")
return (this as AbstractTestFacade<ResultingArtifact.Source, ResultingArtifact.Source>)
.transform(module, artifact as ResultingArtifact<ResultingArtifact.Source>)
}
private fun <I : ResultingArtifact<I>, O : ResultingArtifact<O>> AbstractTestFacade<I, O>.transform(
module: TestModule,
inputArtifact: ResultingArtifact<I>
): O? {
@Suppress("UNCHECKED_CAST")
return transform(module, inputArtifact as I)
}
@@ -0,0 +1,84 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.test
import org.jetbrains.kotlin.test.TestRunner.Companion.shouldRun
import org.jetbrains.kotlin.test.model.*
sealed class TestStep<I : ResultingArtifact<I>, out O : ResultingArtifact<out O>> {
abstract val inputArtifactKind: TestArtifactKind<I>
open fun shouldProcessModule(module: TestModule, inputArtifact: ResultingArtifact<*>): Boolean {
return inputArtifact.kind == inputArtifactKind
}
abstract fun processModule(module: TestModule, inputArtifact: I, thereWereExceptionsOnPreviousSteps: Boolean): StepResult<O>
class FacadeStep<I : ResultingArtifact<I>, O : ResultingArtifact<O>>(val facade: AbstractTestFacade<I, O>) : TestStep<I, O>() {
override val inputArtifactKind: TestArtifactKind<I>
get() = facade.inputKind
val outputArtifactKind: TestArtifactKind<O>
get() = facade.outputKind
override fun shouldProcessModule(module: TestModule, inputArtifact: ResultingArtifact<*>): Boolean {
return super.shouldProcessModule(module, inputArtifact) && facade.shouldRunAnalysis(module)
}
override fun processModule(module: TestModule, inputArtifact: I, thereWereExceptionsOnPreviousSteps: Boolean): StepResult<O> {
val outputArtifact = try {
facade.transform(module, inputArtifact) ?: return StepResult.NoArtifactFromFacade
} catch (e: Throwable) {
// TODO: remove inheritors of WrappedException.FromFacade
return StepResult.ErrorFromFacade(WrappedException.FromFacade(e, facade))
}
return StepResult.Artifact(outputArtifact)
}
}
class HandlersStep<I : ResultingArtifact<I>>(
override val inputArtifactKind: TestArtifactKind<I>,
val handlers: List<AnalysisHandler<I>>
) : TestStep<I, Nothing>() {
init {
require(handlers.all { it.artifactKind == inputArtifactKind })
}
override fun processModule(
module: TestModule,
inputArtifact: I,
thereWereExceptionsOnPreviousSteps: Boolean
): StepResult.HandlersResult {
val exceptions = mutableListOf<WrappedException>()
for (outputHandler in handlers) {
if (outputHandler.shouldRun(thereWasAnException = thereWereExceptionsOnPreviousSteps || exceptions.isNotEmpty())) {
try {
outputHandler.processModule(module, inputArtifact)
} catch (e: Throwable) {
exceptions += WrappedException.FromHandler(e, outputHandler)
if (outputHandler.failureDisablesNextSteps) {
return StepResult.HandlersResult(exceptions, shouldRunNextSteps = false)
}
}
}
}
return StepResult.HandlersResult(exceptions, shouldRunNextSteps = true)
}
}
sealed class StepResult<out O : ResultingArtifact<out O>> {
class Artifact<out O : ResultingArtifact<out O>>(val outputArtifact: O, ) : StepResult<O>()
class ErrorFromFacade<O : ResultingArtifact<O>>(val exception: WrappedException) : StepResult<O>()
data class HandlersResult(
val exceptionsFromHandlers: Collection<WrappedException>,
val shouldRunNextSteps: Boolean
) : StepResult<Nothing>()
object NoArtifactFromFacade : StepResult<Nothing>()
}
}
@@ -0,0 +1,41 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.test
import org.jetbrains.kotlin.test.model.*
import org.jetbrains.kotlin.test.services.TestServices
sealed class TestStepBuilder<I : ResultingArtifact<I>, out O : ResultingArtifact<out O>> {
@TestInfrastructureInternals
abstract fun createTestStep(testServices: TestServices): TestStep<I, O>
}
class FacadeStepBuilder<I : ResultingArtifact<I>, O : ResultingArtifact<O>>(
val facade: Constructor<AbstractTestFacade<I, O>>
) : TestStepBuilder<I, O>() {
@TestInfrastructureInternals
override fun createTestStep(testServices: TestServices): TestStep.FacadeStep<I, O> {
return TestStep.FacadeStep(facade.invoke(testServices))
}
}
class HandlersStepBuilder<I : ResultingArtifact<I>>(val artifactKind: TestArtifactKind<I>) : TestStepBuilder<I, Nothing>() {
private val handlers: MutableList<Constructor<AnalysisHandler<I>>> = mutableListOf()
fun useHandlers(vararg constructor: Constructor<AnalysisHandler<I>>) {
handlers += constructor
}
fun useHandlers(constructors: List<Constructor<AnalysisHandler<I>>>) {
handlers += constructors
}
@TestInfrastructureInternals
override fun createTestStep(testServices: TestServices): TestStep.HandlersStep<I> {
return TestStep.HandlersStep(artifactKind, handlers.map { it.invoke(testServices) })
}
}
@@ -5,26 +5,22 @@
package org.jetbrains.kotlin.test
import org.jetbrains.kotlin.test.model.AbstractTestFacade
import org.jetbrains.kotlin.test.model.AnalysisHandler
sealed class WrappedException(
cause: Throwable,
val priority: Int,
val additionalPriority: Int
) : Exception(cause), Comparable<WrappedException> {
sealed class FromFacade(cause: Throwable, additionalPriority: Int) : WrappedException(cause, 0, additionalPriority) {
class Frontend(cause: Throwable) : FromFacade(cause, 1)
class Converter(cause: Throwable) : FromFacade(cause, 2)
class Backend(cause: Throwable) : FromFacade(cause, 3)
class FromFacade(cause: Throwable, val facade: AbstractTestFacade<*, *>) : WrappedException(cause, 0, 1) {
override val message: String
get() = "Exception was thrown"
}
class FromMetaInfoHandler(cause: Throwable) : WrappedException(cause, 1, 1)
class FromFrontendHandler(cause: Throwable) : WrappedException(cause, 1, 1)
class FromBackendHandler(cause: Throwable) : WrappedException(cause, 1, 2)
class FromBinaryHandler(cause: Throwable) : WrappedException(cause, 1, 3)
class FromUnknownHandler(cause: Throwable) : WrappedException(cause, 1, 4)
class FromHandler(cause: Throwable, val handler: AnalysisHandler<*>) : WrappedException(cause, 1, 2)
class FromAfterAnalysisChecker(cause: Throwable) : WrappedException(cause, 2, 1)
@@ -8,6 +8,7 @@ package org.jetbrains.kotlin.test.model
import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
import org.jetbrains.kotlin.test.services.ServiceRegistrationData
import org.jetbrains.kotlin.test.services.TestServices
import org.jetbrains.kotlin.test.services.backendKindExtractor
interface ServicesAndDirectivesContainer {
val additionalServices: List<ServiceRegistrationData>
@@ -22,6 +23,7 @@ abstract class AbstractTestFacade<I : ResultingArtifact<I>, O : ResultingArtifac
abstract val outputKind: TestArtifactKind<O>
abstract fun transform(module: TestModule, inputArtifact: I): O?
abstract fun shouldRunAnalysis(module: TestModule): Boolean
}
abstract class FrontendFacade<R : ResultingArtifact.FrontendOutput<R>>(
@@ -31,6 +33,10 @@ abstract class FrontendFacade<R : ResultingArtifact.FrontendOutput<R>>(
final override val inputKind: TestArtifactKind<ResultingArtifact.Source>
get() = SourcesKind
override fun shouldRunAnalysis(module: TestModule): Boolean {
return module.frontendKind == outputKind
}
abstract fun analyze(module: TestModule): R
final override fun transform(module: TestModule, inputArtifact: ResultingArtifact.Source): R {
@@ -43,10 +49,18 @@ abstract class Frontend2BackendConverter<R : ResultingArtifact.FrontendOutput<R>
val testServices: TestServices,
final override val inputKind: FrontendKind<R>,
final override val outputKind: BackendKind<I>
) : AbstractTestFacade<R, I>()
) : AbstractTestFacade<R, I>() {
override fun shouldRunAnalysis(module: TestModule): Boolean {
return testServices.backendKindExtractor.backendKind(module.targetBackend) == outputKind
}
}
abstract class BackendFacade<I : ResultingArtifact.BackendInput<I>, A : ResultingArtifact.Binary<A>>(
val testServices: TestServices,
final override val inputKind: BackendKind<I>,
final override val outputKind: BinaryKind<A>
) : AbstractTestFacade<I, A>()
) : AbstractTestFacade<I, A>() {
override fun shouldRunAnalysis(module: TestModule): Boolean {
return testServices.backendKindExtractor.backendKind(module.targetBackend) == inputKind && module.binaryKind == outputKind
}
}