[TEST] Add core of new tests infrastructure

It contains different abstractions which represents parts of compiler
  pipeline and artifacts produced by them, service structure, handlers
  for analysis of artifacts

compiler/test-infrastructure-utils/tests/org/jetbrains/kotlin/test/util/StringUtils.kt

@@ -0,0 +1,9 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.util
+
+fun Iterable<*>.joinToArrayString(): String = joinToString(separator = ", ", prefix = "[", postfix = "]")
+fun Array<*>.joinToArrayString(): String = joinToString(separator = ", ", prefix = "[", postfix = "]")

compiler/test-infrastructure/build.gradle.kts

@@ -0,0 +1,27 @@
+plugins {
+    kotlin("jvm")
+    id("jps-compatible")
+}
+
+dependencies {
+    testApi(project(":compiler:fir:entrypoint"))
+    testApi(project(":compiler:cli"))
+    testApi(intellijCoreDep()) { includeJars("intellij-core") }
+
+    testCompileOnly(project(":kotlin-reflect-api"))
+    testRuntimeOnly(project(":kotlin-reflect"))
+    testRuntimeOnly(project(":core:descriptors.runtime"))
+
+    testImplementation(projectTests(":compiler:test-infrastructure-utils"))
+
+    testRuntimeOnly(intellijDep()) {
+        includeJars("jna", rootProject = rootProject)
+    }
+}
+
+sourceSets {
+    "main" { none() }
+    "test" { projectDefault() }
+}
+
+testsJar()

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/ExceptionFromTestError.kt

@@ -0,0 +1,11 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test
+
+class ExceptionFromTestError(cause: Throwable) : AssertionError(cause) {
+    override val message: String
+        get() = "Exception was thrown"
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/TestConfiguration.kt

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test
+
+import com.intellij.openapi.Disposable
+import org.jetbrains.kotlin.test.directives.ConfigurationDirectives
+import org.jetbrains.kotlin.test.directives.LanguageSettingsDirectives
+import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
+import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
+import org.jetbrains.kotlin.test.model.*
+import org.jetbrains.kotlin.test.services.MetaTestConfigurator
+import org.jetbrains.kotlin.test.services.ModuleStructureExtractor
+import org.jetbrains.kotlin.test.services.SourceFilePreprocessor
+import org.jetbrains.kotlin.test.services.TestServices
+
+typealias Constructor<T> = (TestServices) -> T
+
+abstract class TestConfiguration {
+    abstract val rootDisposable: Disposable
+
+    abstract val testServices: TestServices
+
+    abstract val directives: DirectivesContainer
+
+    abstract val defaultRegisteredDirectives: RegisteredDirectives
+
+    abstract val moduleStructureExtractor: ModuleStructureExtractor
+
+    abstract val metaTestConfigurators: List<MetaTestConfigurator>
+
+    abstract val afterAnalysisCheckers: List<AfterAnalysisChecker>
+
+    abstract val metaInfoHandlerEnabled: Boolean
+
+    abstract fun <I : ResultingArtifact<I>, O : ResultingArtifact<O>> getFacade(
+        inputKind: TestArtifactKind<I>,
+        outputKind: TestArtifactKind<O>
+    ): AbstractTestFacade<I, O>
+
+    abstract fun <A : ResultingArtifact<A>> getHandlers(artifactKind: TestArtifactKind<A>): List<AnalysisHandler<A>>
+
+    abstract fun getAllHandlers(): List<AnalysisHandler<*>>
+}
+

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/TestRunner.kt

@@ -0,0 +1,171 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test
+
+import com.intellij.openapi.util.Disposer
+import com.intellij.testFramework.TestDataFile
+import org.jetbrains.kotlin.test.model.*
+import org.jetbrains.kotlin.test.services.*
+
+class TestRunner(private val testConfiguration: TestConfiguration) {
+    private val failedAssertions = mutableListOf<AssertionError>()
+
+    fun runTest(@TestDataFile testDataFileName: String) {
+        try {
+            runTestImpl(testDataFileName)
+        } finally {
+            Disposer.dispose(testConfiguration.rootDisposable)
+        }
+    }
+
+    private fun runTestImpl(@TestDataFile testDataFileName: String) {
+        val services = testConfiguration.testServices
+
+        @Suppress("NAME_SHADOWING")
+        val testDataFileName = testConfiguration.metaTestConfigurators.fold(testDataFileName) { fileName, configurator ->
+            configurator.transformTestDataPath(fileName)
+        }
+
+        val moduleStructure = testConfiguration.moduleStructureExtractor.splitTestDataByModules(
+            testDataFileName,
+            testConfiguration.directives,
+        ).also {
+            services.register(TestModuleStructure::class, it)
+        }
+
+        testConfiguration.metaTestConfigurators.forEach {
+            if (it.shouldSkipTest()) return
+        }
+
+        val globalMetadataInfoHandler = testConfiguration.testServices.globalMetadataInfoHandler
+        globalMetadataInfoHandler.parseExistingMetadataInfosFromAllSources()
+
+        val modules = moduleStructure.modules
+        val dependencyProvider = DependencyProviderImpl(services, modules)
+        services.registerDependencyProvider(dependencyProvider)
+        var failedException: Throwable? = null
+        try {
+            for (module in modules) {
+                processModule(module, dependencyProvider, moduleStructure)
+            }
+        } catch (e: Throwable) {
+            failedException = e
+        }
+        for (handler in testConfiguration.getAllHandlers()) {
+            withAssertionCatching { handler.processAfterAllModules(failedAssertions.isNotEmpty()) }
+        }
+        if (testConfiguration.metaInfoHandlerEnabled) {
+            withAssertionCatching {
+                globalMetadataInfoHandler.compareAllMetaDataInfos()
+            }
+        }
+        if (failedException != null) {
+            failedAssertions.add(0, ExceptionFromTestError(failedException))
+        }
+
+        testConfiguration.afterAnalysisCheckers.forEach {
+            withAssertionCatching {
+                it.check(failedAssertions)
+            }
+        }
+
+        val filteredFailedAssertions = testConfiguration.afterAnalysisCheckers
+            .fold<AfterAnalysisChecker, List<AssertionError>>(failedAssertions) { assertions, checker ->
+                checker.suppressIfNeeded(assertions)
+            }
+
+        services.assertions.assertAll(filteredFailedAssertions)
+    }
+
+    private fun processModule(
+        module: TestModule,
+        dependencyProvider: DependencyProviderImpl,
+        moduleStructure: TestModuleStructure
+    ) {
+        val sourcesArtifact = ResultingArtifact.Source()
+
+        val frontendKind = module.frontendKind
+        if (!frontendKind.shouldRunAnalysis) return
+
+        val frontendArtifacts: ResultingArtifact.FrontendOutput<*> = testConfiguration.getFacade(SourcesKind, frontendKind)
+            .transform(module, sourcesArtifact).also { dependencyProvider.registerArtifact(module, it) }
+        val frontendHandlers: List<AnalysisHandler<*>> = testConfiguration.getHandlers(frontendKind)
+        for (frontendHandler in frontendHandlers) {
+            withAssertionCatching {
+                frontendHandler.hackyProcess(module, frontendArtifacts)
+            }
+        }
+
+        val backendKind = module.backendKind
+        if (!backendKind.shouldRunAnalysis) return
+
+        val backendInputInfo = testConfiguration.getFacade(frontendKind, backendKind)
+            .hackyTransform(module, frontendArtifacts).also { dependencyProvider.registerArtifact(module, it) }
+
+        val backendHandlers: List<AnalysisHandler<*>> = testConfiguration.getHandlers(backendKind)
+        for (backendHandler in backendHandlers) {
+            withAssertionCatching { backendHandler.hackyProcess(module, backendInputInfo) }
+        }
+
+        for (artifactKind in moduleStructure.getTargetArtifactKinds(module)) {
+            if (!artifactKind.shouldRunAnalysis) continue
+            val binaryArtifact = testConfiguration.getFacade(backendKind, artifactKind)
+                .hackyTransform(module, backendInputInfo).also {
+                    dependencyProvider.registerArtifact(module, it)
+                }
+
+            val binaryHandlers: List<AnalysisHandler<*>> = testConfiguration.getHandlers(artifactKind)
+            for (binaryHandler in binaryHandlers) {
+                withAssertionCatching { binaryHandler.hackyProcess(module, binaryArtifact) }
+            }
+        }
+    }
+
+    private inline fun withAssertionCatching(block: () -> Unit) {
+        try {
+            block()
+        } catch (e: AssertionError) {
+            failedAssertions += e
+        }
+    }
+}
+
+// ----------------------------------------------------------------------------------------------------------------
+/*
+ * Those `hackyProcess` methods are needed to hack kotlin type system. In common test case
+ *   we have artifact of type ResultingArtifact<*> and handler of type AnalysisHandler<*> and actually
+ *   at runtime types under `*` are same (that achieved by grouping handlers and facades by
+ *   frontend/backend/artifact kind). But there is no way to tell that to compiler, so I unsafely cast types with `*`
+ *   to types with Empty artifacts to make it compile. Since unsafe cast has no effort at runtime, it's safe to use it
+ */
+
+private fun AnalysisHandler<*>.hackyProcess(module: TestModule, artifact: ResultingArtifact<*>) {
+    @Suppress("UNCHECKED_CAST")
+    (this as AnalysisHandler<ResultingArtifact.Source>)
+        .processModule(module, artifact as ResultingArtifact<ResultingArtifact.Source>)
+}
+
+private fun <A : ResultingArtifact<A>> AnalysisHandler<A>.processModule(module: TestModule, artifact: ResultingArtifact<A>) {
+    @Suppress("UNCHECKED_CAST")
+    processModule(module, artifact as A)
+}
+
+private fun AbstractTestFacade<*, *>.hackyTransform(
+    module: TestModule,
+    artifact: ResultingArtifact<*>
+): ResultingArtifact<*> {
+    @Suppress("UNCHECKED_CAST")
+    return (this as AbstractTestFacade<ResultingArtifact.Source, ResultingArtifact.Source>)
+        .transform(module, artifact as ResultingArtifact<ResultingArtifact.Source>)
+}
+
+private fun <I : ResultingArtifact<I>, O : ResultingArtifact<O>> AbstractTestFacade<I, O>.transform(
+    module: TestModule,
+    inputArtifact: ResultingArtifact<I>
+): O {
+    @Suppress("UNCHECKED_CAST")
+    return transform(module, inputArtifact as I)
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/builders/LanguageVersionSettingsBuilder.kt

@@ -0,0 +1,112 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.builders
+
+import org.jetbrains.kotlin.config.*
+import org.jetbrains.kotlin.test.directives.LanguageSettingsDirectives
+import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
+import org.jetbrains.kotlin.test.directives.model.singleOrZeroValue
+import org.jetbrains.kotlin.test.services.DefaultsDsl
+import org.jetbrains.kotlin.utils.addToStdlib.runIf
+import java.util.regex.Pattern
+
+@DefaultsDsl
+class LanguageVersionSettingsBuilder {
+    companion object {
+        private val languageFeaturePattern = Pattern.compile("""(\+|-|warn:)(\w+)\s*""")
+
+        fun fromExistingSettings(builder: LanguageVersionSettingsBuilder): LanguageVersionSettingsBuilder {
+            return LanguageVersionSettingsBuilder().apply {
+                languageVersion = builder.languageVersion
+                apiVersion = builder.apiVersion
+                specificFeatures += builder.specificFeatures
+                analysisFlags += builder.analysisFlags
+            }
+        }
+    }
+
+    var languageVersion: LanguageVersion = LanguageVersion.LATEST_STABLE
+    var apiVersion: ApiVersion = ApiVersion.LATEST_STABLE
+
+    private val specificFeatures: MutableMap<LanguageFeature, LanguageFeature.State> = mutableMapOf()
+    private val analysisFlags: MutableMap<AnalysisFlag<*>, Any?> = mutableMapOf()
+
+    fun enable(feature: LanguageFeature) {
+        specificFeatures[feature] = LanguageFeature.State.ENABLED
+    }
+
+    fun enableWithWarning(feature: LanguageFeature) {
+        specificFeatures[feature] = LanguageFeature.State.ENABLED_WITH_WARNING
+    }
+
+    fun disable(feature: LanguageFeature) {
+        specificFeatures[feature] = LanguageFeature.State.DISABLED
+    }
+
+    fun <T> withFlag(flag: AnalysisFlag<T>, value: T) {
+        analysisFlags[flag] = value
+    }
+
+    fun configureUsingDirectives(directives: RegisteredDirectives) {
+        val apiVersion = directives.singleOrZeroValue(LanguageSettingsDirectives.API_VERSION)
+        if (apiVersion != null) {
+            this.apiVersion = apiVersion
+            val languageVersion = maxOf(LanguageVersion.LATEST_STABLE, LanguageVersion.fromVersionString(apiVersion.versionString)!!)
+            this.languageVersion = languageVersion
+        }
+
+        val analysisFlags = listOfNotNull(
+            analysisFlag(AnalysisFlags.experimental, directives[LanguageSettingsDirectives.EXPERIMENTAL].takeIf { it.isNotEmpty() }),
+            analysisFlag(AnalysisFlags.useExperimental, directives[LanguageSettingsDirectives.USE_EXPERIMENTAL].takeIf { it.isNotEmpty() }),
+            analysisFlag(AnalysisFlags.ignoreDataFlowInAssert, trueOrNull(LanguageSettingsDirectives.IGNORE_DATA_FLOW_IN_ASSERT in directives)),
+            analysisFlag(AnalysisFlags.constraintSystemForOverloadResolution, directives.singleOrZeroValue(LanguageSettingsDirectives.CONSTRAINT_SYSTEM_FOR_OVERLOAD_RESOLUTION)),
+            analysisFlag(AnalysisFlags.allowResultReturnType, trueOrNull(LanguageSettingsDirectives.ALLOW_RESULT_RETURN_TYPE in directives)),
+
+            analysisFlag(JvmAnalysisFlags.jvmDefaultMode, directives.singleOrZeroValue(LanguageSettingsDirectives.JVM_DEFAULT_MODE)),
+            analysisFlag(JvmAnalysisFlags.inheritMultifileParts, trueOrNull(LanguageSettingsDirectives.INHERIT_MULTIFILE_PARTS in directives)),
+            analysisFlag(JvmAnalysisFlags.sanitizeParentheses, trueOrNull(LanguageSettingsDirectives.SANITIZE_PARENTHESES in directives)),
+
+            analysisFlag(AnalysisFlags.explicitApiVersion, trueOrNull(apiVersion != null)),
+        )
+
+        analysisFlags.forEach { withFlag(it.first, it.second) }
+
+        directives[LanguageSettingsDirectives.LANGUAGE].forEach { parseLanguageFeature(it) }
+    }
+
+    private fun parseLanguageFeature(featureString: String) {
+        val matcher = languageFeaturePattern.matcher(featureString)
+        if (!matcher.find()) {
+            error(
+                """Wrong syntax in the '// !${LanguageSettingsDirectives.LANGUAGE.name}: ...' directive:
+                   found: '$featureString'
+                   Must be '((+|-|warn:)LanguageFeatureName)+'
+                   where '+' means 'enable', '-' means 'disable', 'warn:' means 'enable with warning'
+                   and language feature names are names of enum entries in LanguageFeature enum class"""
+            )
+        }
+        val mode = when (val mode = matcher.group(1)) {
+            "+" -> LanguageFeature.State.ENABLED
+            "-" -> LanguageFeature.State.DISABLED
+            "warn:" -> LanguageFeature.State.ENABLED_WITH_WARNING
+            else -> error("Unknown mode for language feature: $mode")
+        }
+        val name = matcher.group(2)
+        val feature = LanguageFeature.fromString(name) ?: error("Language feature with name \"$name\" not found")
+        specificFeatures[feature] = mode
+    }
+
+    @Suppress("INVISIBLE_MEMBER", "INVISIBLE_REFERENCE")
+    private fun <T : Any> analysisFlag(flag: AnalysisFlag<T>, value: @kotlin.internal.NoInfer T?): Pair<AnalysisFlag<T>, T>? =
+        value?.let(flag::to)
+
+    private fun trueOrNull(condition: Boolean): Boolean? = runIf(condition) { true }
+
+    fun build(): LanguageVersionSettings {
+        return LanguageVersionSettingsImpl(languageVersion, apiVersion, analysisFlags, specificFeatures)
+    }
+}
+

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/directives/ConfigurationDirectives.kt

@@ -0,0 +1,14 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.directives
+
+import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer
+
+object ConfigurationDirectives : SimpleDirectivesContainer() {
+    val KOTLIN_CONFIGURATION_FLAGS by stringDirective(
+        "List of kotlin configuration flags"
+    )
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/directives/LanguageSettingsDirectives.kt

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.directives
+
+import org.jetbrains.kotlin.config.ApiVersion
+import org.jetbrains.kotlin.config.ConstraintSystemForOverloadResolutionMode
+import org.jetbrains.kotlin.config.JvmDefaultMode
+import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer
+
+object LanguageSettingsDirectives : SimpleDirectivesContainer() {
+    val LANGUAGE by stringDirective(
+        description = """
+            List of enabled and disabled language features.
+            Usage: // !LANGUAGE: +SomeFeature -OtherFeature warn:FeatureWithEarning
+        """.trimIndent()
+    )
+
+    @Suppress("RemoveExplicitTypeArguments")
+    val API_VERSION by valueDirective<ApiVersion>(
+        description = "Version of Kotlin API",
+        parser = this::parseApiVersion
+    )
+    // --------------------- Analysis Flags ---------------------
+
+    val USE_EXPERIMENTAL by stringDirective(
+        description = "List of opted in annotations (AnalysisFlags.useExperimental)"
+    )
+
+    val EXPERIMENTAL by stringDirective(
+        description = "Require opt in for specified annotations (AnalysisFlags.experimental)"
+    )
+
+    val IGNORE_DATA_FLOW_IN_ASSERT by directive(
+        description = "Enables corresponding analysis flag (AnalysisFlags.ignoreDataFlowInAssert)"
+    )
+
+    val CONSTRAINT_SYSTEM_FOR_OVERLOAD_RESOLUTION by enumDirective<ConstraintSystemForOverloadResolutionMode>(
+        description = "Configures corresponding analysis flag (AnalysisFlags.constraintSystemForOverloadResolution)",
+    )
+
+    val ALLOW_RESULT_RETURN_TYPE by directive(
+        description = "Allow using Result in return type position"
+    )
+
+    // --------------------- Jvm Analysis Flags ---------------------
+
+    @Suppress("RemoveExplicitTypeArguments")
+    val JVM_DEFAULT_MODE by enumDirective<JvmDefaultMode>(
+        description = "Configures corresponding analysis flag (JvmAnalysisFlags.jvmDefaultMode)",
+        additionalParser = JvmDefaultMode.Companion::fromStringOrNull
+    )
+
+    val INHERIT_MULTIFILE_PARTS by directive(
+        description = "Enables corresponding analysis flag (JvmAnalysisFlags.inheritMultifileParts)"
+    )
+
+    val SANITIZE_PARENTHESES by directive(
+        description = "Enables corresponding analysis flag (JvmAnalysisFlags.sanitizeParentheses)"
+    )
+
+    // --------------------- Utils ---------------------
+
+    fun parseApiVersion(versionString: String): ApiVersion = when (versionString) {
+        "LATEST" -> ApiVersion.LATEST
+        else -> ApiVersion.parse(versionString) ?: error("Unknown API version: $versionString")
+    }
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/directives/ModuleStructureDirectives.kt

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.directives
+
+import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer
+
+object ModuleStructureDirectives : SimpleDirectivesContainer() {
+    val MODULE by stringDirective(
+        """
+            Usage: // MODULE: {name}[(dependencies)]
+            Describes one module. If no targets are specified then <TODO>
+        """.trimIndent()
+    )
+
+    val DEPENDENCY by stringDirective(
+        """
+            Usage: // DEPENDENCY: {name} [SOURCE|KLIB|BINARY]
+            Declares simple dependency on other module 
+        """.trimIndent()
+    )
+
+    val DEPENDS_ON by stringDirective(
+        """
+            Usage: // DEPENDS_ON: {name} [SOURCE|KLIB|BINARY]
+            Declares dependency on other module witch may contains `expect`
+             declarations which has corresponding `expect` declarations
+             in current module
+        """.trimIndent()
+    )
+
+    val FILE by stringDirective(
+        """
+            Usage: // FILE: name.{kt|java}
+            Declares file with specified name in current module
+        """.trimIndent()
+    )
+
+    val TARGET_FRONTEND by stringDirective(
+        """
+            Usage: // TARGET_FRONTEND: {Frontend}
+            Declares frontend for analyzing current module 
+        """.trimIndent()
+    )
+
+    val TARGET_BACKEND_KIND by stringDirective(
+        """
+            Usage: // TARGET_BACKEND: {Backend}
+            Declares backend for analyzing current module 
+        """.trimIndent()
+    )
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/directives/model/Directive.kt

@@ -0,0 +1,140 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.directives.model
+
+import org.jetbrains.kotlin.test.util.joinToArrayString
+
+// --------------------------- Directive declaration ---------------------------
+
+sealed class Directive(val name: String, val description: String) {
+    override fun toString(): String {
+        return name
+    }
+}
+
+class SimpleDirective(
+    name: String,
+    description: String
+) : Directive(name, description)
+
+class StringDirective(
+    name: String,
+    description: String
+) : Directive(name, description)
+
+class ValueDirective<T : Any>(
+    name: String,
+    description: String,
+    val parser: (String) -> T?
+) : Directive(name, description)
+
+// --------------------------- Registered directive ---------------------------
+
+abstract class RegisteredDirectives {
+    companion object {
+        val Empty = RegisteredDirectivesImpl(emptyList(), emptyMap(), emptyMap())
+    }
+
+    abstract operator fun contains(directive: Directive): Boolean
+    abstract operator fun get(directive: StringDirective): List<String>
+    abstract operator fun <T : Any> get(directive: ValueDirective<T>): List<T>
+
+    abstract fun isEmpty(): Boolean
+}
+
+class RegisteredDirectivesImpl(
+    private val simpleDirectives: List<SimpleDirective>,
+    private val stringDirectives: Map<StringDirective, List<String>>,
+    private val valueDirectives: Map<ValueDirective<*>, List<Any>>
+) : RegisteredDirectives() {
+    override operator fun contains(directive: Directive): Boolean {
+        return when (directive) {
+            is SimpleDirective -> directive in simpleDirectives
+            is StringDirective -> directive in stringDirectives
+            is ValueDirective<*> -> directive in valueDirectives
+        }
+    }
+
+    override operator fun get(directive: StringDirective): List<String> {
+        return stringDirectives[directive] ?: emptyList()
+    }
+
+    override fun <T : Any> get(directive: ValueDirective<T>): List<T> {
+        @Suppress("UNCHECKED_CAST")
+        return valueDirectives[directive] as List<T>? ?: emptyList()
+    }
+
+    override fun isEmpty(): Boolean {
+        return simpleDirectives.isEmpty() && stringDirectives.isEmpty() && valueDirectives.isEmpty()
+    }
+
+    override fun toString(): String {
+        return buildString {
+            simpleDirectives.forEach { appendLine("  $it") }
+            stringDirectives.forEach { (d, v) -> appendLine("  $d: ${v.joinToArrayString()}") }
+            valueDirectives.forEach { (d, v) -> appendLine("  $d: ${v.joinToArrayString()}")}
+        }
+    }
+}
+
+class ComposedRegisteredDirectives(
+    private val containers: List<RegisteredDirectives>
+) : RegisteredDirectives() {
+    companion object {
+        operator fun invoke(vararg containers: RegisteredDirectives): RegisteredDirectives {
+            val notEmptyContainers = containers.filterNot { it.isEmpty() }
+            return when (notEmptyContainers.size) {
+                0 -> Empty
+                1 -> notEmptyContainers.single()
+                else -> ComposedRegisteredDirectives(notEmptyContainers)
+            }
+        }
+    }
+
+    override fun contains(directive: Directive): Boolean {
+        return containers.any { directive in it }
+    }
+
+    override fun get(directive: StringDirective): List<String> {
+        return containers.flatMap { it[directive] }
+    }
+
+    override fun <T : Any> get(directive: ValueDirective<T>): List<T> {
+        return containers.flatMap { it[directive] }
+    }
+
+    override fun isEmpty(): Boolean {
+        return containers.all { it.isEmpty() }
+    }
+}
+
+// --------------------------- Utils ---------------------------
+
+fun RegisteredDirectives.singleValue(directive: StringDirective): String {
+    return singleOrZeroValue(directive) ?: error("No values passed to $directive")
+}
+
+fun RegisteredDirectives.singleOrZeroValue(directive: StringDirective): String? {
+    val values = this[directive]
+    return when (values.size) {
+        0 -> null
+        1 -> values.single()
+        else -> error("Too many values passed to $directive")
+    }
+}
+
+fun <T : Any> RegisteredDirectives.singleValue(directive: ValueDirective<T>): T {
+    return singleOrZeroValue(directive) ?: error("No values passed to $directive")
+}
+
+fun <T : Any> RegisteredDirectives.singleOrZeroValue(directive: ValueDirective<T>): T? {
+    val values = this[directive]
+    return when (values.size) {
+        0 -> null
+        1 -> values.single()
+        else -> error("Too many values passed to $directive")
+    }
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/directives/model/DirectivesContainer.kt

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.directives.model
+
+import kotlin.properties.ReadOnlyProperty
+import kotlin.reflect.KProperty
+
+sealed class DirectivesContainer {
+    object Empty : SimpleDirectivesContainer()
+
+    abstract operator fun get(name: String): Directive?
+    abstract operator fun contains(directive: Directive): Boolean
+}
+
+abstract class SimpleDirectivesContainer : DirectivesContainer() {
+    private val registeredDirectives: MutableMap<String, Directive> = mutableMapOf()
+
+    override operator fun get(name: String): Directive? = registeredDirectives[name]
+
+    protected fun directive(description: String): DirectiveDelegateProvider<SimpleDirective> {
+        return DirectiveDelegateProvider { SimpleDirective(it, description) }
+    }
+
+    protected fun stringDirective(description: String): DirectiveDelegateProvider<StringDirective> {
+        return DirectiveDelegateProvider { StringDirective(it, description) }
+    }
+
+    protected inline fun <reified T : Enum<T>> enumDirective(
+        description: String,
+        noinline additionalParser: ((String) -> T?)? = null
+    ): DirectiveDelegateProvider<ValueDirective<T>> {
+        val possibleValues = enumValues<T>()
+        val parser: (String) -> T? = { value -> possibleValues.firstOrNull { it.name == value } ?: additionalParser?.invoke(value) }
+        return DirectiveDelegateProvider { ValueDirective(it, description, parser) }
+    }
+
+    protected fun <T : Any> valueDirective(
+        description: String,
+        parser: (String) -> T?
+    ): DirectiveDelegateProvider<ValueDirective<T>> {
+        return DirectiveDelegateProvider { ValueDirective(it, description, parser) }
+    }
+
+    protected fun registerDirective(directive: Directive) {
+        registeredDirectives[directive.name] = directive
+    }
+
+    override fun contains(directive: Directive): Boolean {
+        return directive in registeredDirectives.values
+    }
+
+    override fun toString(): String {
+        return buildString {
+            appendLine("Directive container:")
+            for (directive in registeredDirectives.values) {
+                append("  ")
+                appendLine(directive)
+            }
+        }
+    }
+
+    protected inner class DirectiveDelegateProvider<T : Directive>(val directiveConstructor: (String) -> T) {
+        operator fun provideDelegate(
+            thisRef: SimpleDirectivesContainer,
+            property: KProperty<*>
+        ): ReadOnlyProperty<SimpleDirectivesContainer, T> {
+            val directive = directiveConstructor(property.name).also { thisRef.registerDirective(it) }
+            return ReadOnlyProperty { _, _ -> directive }
+        }
+    }
+}
+
+class ComposedDirectivesContainer(private val containers: Collection<DirectivesContainer>) : DirectivesContainer() {
+    constructor(vararg containers: DirectivesContainer) : this(containers.toList())
+
+    override fun get(name: String): Directive? {
+        for (container in containers) {
+            container[name]?.let { return it }
+        }
+        return null
+    }
+
+    override fun contains(directive: Directive): Boolean {
+        return containers.any { directive in it }
+    }
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/model/AfterAnalysisChecker.kt

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.model
+
+import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
+import org.jetbrains.kotlin.test.services.TestServices
+
+abstract class AfterAnalysisChecker(protected val testServices: TestServices) {
+    open val directives: List<DirectivesContainer>
+        get() = emptyList()
+
+    open fun check(failedAssertions: List<AssertionError>) {}
+
+    open fun suppressIfNeeded(failedAssertions: List<AssertionError>): List<AssertionError> = failedAssertions
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/model/AnalysisHandler.kt

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.model
+
+import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
+import org.jetbrains.kotlin.test.services.Assertions
+import org.jetbrains.kotlin.test.services.ServiceRegistrationData
+import org.jetbrains.kotlin.test.services.TestServices
+import org.jetbrains.kotlin.test.services.assertions
+
+abstract class AnalysisHandler<A : ResultingArtifact<A>>(val testServices: TestServices) {
+    protected val assertions: Assertions
+        get() = testServices.assertions
+
+    open val directivesContainers: List<DirectivesContainer>
+        get() = emptyList()
+
+    open val additionalServices: List<ServiceRegistrationData>
+        get() = emptyList()
+
+    abstract val artifactKind: TestArtifactKind<A>
+
+    abstract fun processModule(module: TestModule, info: A)
+
+    abstract fun processAfterAllModules(someAssertionWasFailed: Boolean)
+}
+
+abstract class FrontendOutputHandler<R : ResultingArtifact.FrontendOutput<R>>(
+    testServices: TestServices,
+    override val artifactKind: FrontendKind<R>
+) : AnalysisHandler<R>(testServices)
+
+abstract class BackendInputHandler<I : ResultingArtifact.BackendInput<I>>(
+    testServices: TestServices,
+    override val artifactKind: BackendKind<I>
+) : AnalysisHandler<I>(testServices)
+
+abstract class BinaryArtifactHandler<A : ResultingArtifact.Binary<A>>(
+    testServices: TestServices,
+    override val artifactKind: BinaryKind<A>
+) : AnalysisHandler<A>(testServices)

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/model/Facades.kt

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.model
+
+import org.jetbrains.kotlin.test.services.ServiceRegistrationData
+import org.jetbrains.kotlin.test.services.TestServices
+
+abstract class AbstractTestFacade<I : ResultingArtifact<I>, O : ResultingArtifact<O>> {
+    abstract val inputKind: TestArtifactKind<I>
+    abstract val outputKind: TestArtifactKind<O>
+
+    abstract fun transform(module: TestModule, inputArtifact: I): O
+
+    open val additionalServices: List<ServiceRegistrationData>
+        get() = emptyList()
+}
+
+abstract class FrontendFacade<R : ResultingArtifact.FrontendOutput<R>>(
+    val testServices: TestServices,
+    final override val outputKind: FrontendKind<R>
+) : AbstractTestFacade<ResultingArtifact.Source, R>() {
+    final override val inputKind: TestArtifactKind<ResultingArtifact.Source>
+        get() = SourcesKind
+
+    abstract fun analyze(module: TestModule): R
+
+    final override fun transform(module: TestModule, inputArtifact: ResultingArtifact.Source): R {
+        // TODO: pass sources
+        return analyze(module)
+    }
+}
+
+abstract class Frontend2BackendConverter<R : ResultingArtifact.FrontendOutput<R>, I : ResultingArtifact.BackendInput<I>>(
+    val testServices: TestServices,
+    final override val inputKind: FrontendKind<R>,
+    final override val outputKind: BackendKind<I>
+) : AbstractTestFacade<R, I>()
+
+abstract class BackendFacade<I : ResultingArtifact.BackendInput<I>, A : ResultingArtifact.Binary<A>>(
+    val testServices: TestServices,
+    final override val inputKind: BackendKind<I>,
+    final override val outputKind: BinaryKind<A>
+) : AbstractTestFacade<I, A>()

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/model/Modules.kt

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.model
+
+import org.jetbrains.kotlin.config.LanguageVersionSettings
+import org.jetbrains.kotlin.platform.TargetPlatform
+import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
+import java.io.File
+
+data class TestModule(
+    val name: String,
+    val targetPlatform: TargetPlatform,
+    val frontendKind: FrontendKind<*>,
+    val backendKind: BackendKind<*>,
+    val files: List<TestFile>,
+    val dependencies: List<DependencyDescription>,
+    val directives: RegisteredDirectives,
+    val languageVersionSettings: LanguageVersionSettings
+) {
+    override fun toString(): String {
+        return buildString {
+            appendLine("Module: $name")
+            appendLine("targetPlatform = $targetPlatform")
+            appendLine("Dependencies:")
+            dependencies.forEach { appendLine("  $it") }
+            appendLine("Directives:\n  $directives")
+            files.forEach { appendLine(it) }
+        }
+    }
+}
+
+class TestFile(
+    val relativePath: String,
+    val originalContent: String,
+    val originalFile: File,
+    val startLineNumberInOriginalFile: Int, // line count starts with 0
+    /*
+     * isAdditional means that this file provided as addition to sources of testdata
+     *   and there is no need to apply any handlers or preprocessors over it
+     */
+    val isAdditional: Boolean
+) {
+    val name: String = relativePath.split("/").last()
+}
+
+enum class DependencyRelation {
+    Dependency,
+    DependsOn
+}
+
+enum class DependencyKind {
+    Source,
+    KLib,
+    Binary
+}
+
+data class DependencyDescription(
+    val moduleName: String,
+    val kind: DependencyKind,
+    val relation: DependencyRelation
+)

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/model/ResultingArtifact.kt

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.model
+
+abstract class ResultingArtifact<A : ResultingArtifact<A>> {
+    abstract val kind: TestArtifactKind<A>
+
+    class Source : ResultingArtifact<Source>() {
+        override val kind: TestArtifactKind<Source>
+            get() = SourcesKind
+    }
+
+    abstract class FrontendOutput<R : FrontendOutput<R>> : ResultingArtifact<R>() {
+        abstract override val kind: FrontendKind<R>
+
+        object Empty : FrontendOutput<Empty>() {
+            override val kind: FrontendKind<Empty>
+                get() = FrontendKind.NoFrontend
+        }
+    }
+
+    abstract class BackendInput<I : BackendInput<I>> : ResultingArtifact<I>() {
+        abstract override val kind: BackendKind<I>
+
+        object Empty : BackendInput<Empty>() {
+            override val kind: BackendKind<Empty>
+                get() = BackendKind.NoBackend
+        }
+    }
+
+    abstract class Binary<A : Binary<A>> : ResultingArtifact<A>() {
+        abstract override val kind: BinaryKind<A>
+
+        object Empty : Binary<Empty>() {
+            override val kind: BinaryKind<Empty>
+                get() = BinaryKind.NoArtifact
+        }
+    }
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/model/TestArtifactKind.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.model
+
+abstract class TestArtifactKind<R : ResultingArtifact<R>>(private val representation: String) {
+    open val shouldRunAnalysis: Boolean
+        get() = true
+
+    override fun toString(): String {
+        return representation
+    }
+}
+
+object SourcesKind : TestArtifactKind<ResultingArtifact.Source>("Sources")
+
+abstract class FrontendKind<R : ResultingArtifact.FrontendOutput<R>>(representation: String) : TestArtifactKind<R>(representation) {
+    object NoFrontend : FrontendKind<ResultingArtifact.FrontendOutput.Empty>("NoFrontend") {
+        override val shouldRunAnalysis: Boolean
+            get() = false
+    }
+}
+
+abstract class BackendKind<I : ResultingArtifact.BackendInput<I>>(representation: String) : TestArtifactKind<I>(representation) {
+    object NoBackend : BackendKind<ResultingArtifact.BackendInput.Empty>("NoBackend") {
+        override val shouldRunAnalysis: Boolean
+            get() = false
+    }
+}
+
+abstract class BinaryKind<A : ResultingArtifact.Binary<A>>(representation: String) : TestArtifactKind<A>(representation) {
+    object NoArtifact : BinaryKind<ResultingArtifact.Binary.Empty>("NoArtifact") {
+        override val shouldRunAnalysis: Boolean
+            get() = false
+    }
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/AdditionalSourceProvider.kt

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.services
+
+import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
+import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
+import org.jetbrains.kotlin.test.directives.model.SimpleDirective
+import org.jetbrains.kotlin.test.model.TestFile
+import org.jetbrains.kotlin.test.model.TestModule
+import java.io.File
+
+abstract class AdditionalSourceProvider(val testServices: TestServices) {
+    open val directives: List<DirectivesContainer>
+        get() = emptyList()
+
+    /**
+     * Note that you can not use [testServices.moduleStructure] here because it's not initialized yet
+     */
+    abstract fun produceAdditionalFiles(globalDirectives: RegisteredDirectives, module: TestModule): List<TestFile>
+
+    protected fun containsDirective(globalDirectives: RegisteredDirectives, module: TestModule, directive: SimpleDirective): Boolean {
+        return globalDirectives.contains(directive) || module.directives.contains(directive)
+    }
+
+    protected fun File.toTestFile(): TestFile {
+        return TestFile(this.name, this.readText(), this, startLineNumberInOriginalFile = 0, isAdditional = true)
+    }
+}
+

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/Assertions.kt

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.services
+
+import java.io.File
+
+abstract class Assertions : TestService {
+    fun assertEqualsToFile(expectedFile: File, actual: String, sanitizer: (String) -> String = { it }) {
+        assertEqualsToFile(expectedFile, actual, sanitizer) { "Actual data differs from file content" }
+    }
+
+    abstract fun assertEqualsToFile(
+        expectedFile: File,
+        actual: String,
+        sanitizer: (String) -> String = { it },
+        message: (() -> String)
+    )
+
+    abstract fun assertEquals(expected: Any?, actual: Any?, message: (() -> String)? = null)
+    abstract fun assertNotEquals(expected: Any?, actual: Any?, message: (() -> String)? = null)
+    abstract fun assertTrue(value: Boolean, message: (() -> String)? = null)
+    abstract fun assertFalse(value: Boolean, message: (() -> String)? = null)
+    abstract fun assertAll(exceptions: List<AssertionError>)
+
+    abstract fun fail(message: () -> String): Nothing
+}
+
+val TestServices.assertions: Assertions by TestServices.testServiceAccessor()

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/DefaultRegisteredDirectivesProvider.kt

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.services
+
+import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
+
+class DefaultRegisteredDirectivesProvider(defaultGlobalDirectives: RegisteredDirectives) : TestService {
+    val defaultDirectives: RegisteredDirectives by lazy {
+        defaultGlobalDirectives
+    }
+}
+
+private val TestServices.defaultRegisteredDirectivesProvider: DefaultRegisteredDirectivesProvider by TestServices.testServiceAccessor()
+
+val TestServices.defaultDirectives: RegisteredDirectives
+    get() = defaultRegisteredDirectivesProvider.defaultDirectives

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/DefaultsProvider.kt

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.services
+
+import org.jetbrains.kotlin.config.LanguageVersionSettings
+import org.jetbrains.kotlin.platform.TargetPlatform
+import org.jetbrains.kotlin.test.builders.LanguageVersionSettingsBuilder
+import org.jetbrains.kotlin.test.model.BackendKind
+import org.jetbrains.kotlin.test.model.DependencyKind
+import org.jetbrains.kotlin.test.model.FrontendKind
+
+/*
+ * TODO:
+ *   - default target artifact
+ *   - default libraries
+ */
+class DefaultsProvider(
+    val defaultBackend: BackendKind<*>,
+    val defaultFrontend: FrontendKind<*>,
+    val defaultLanguageSettings: LanguageVersionSettings,
+    private val defaultLanguageSettingsBuilder: LanguageVersionSettingsBuilder,
+    val defaultPlatform: TargetPlatform,
+    val defaultDependencyKind: DependencyKind
+) : TestService {
+    fun newLanguageSettingsBuilder(): LanguageVersionSettingsBuilder {
+        return LanguageVersionSettingsBuilder.fromExistingSettings(defaultLanguageSettingsBuilder)
+    }
+}
+
+val TestServices.defaultsProvider: DefaultsProvider by TestServices.testServiceAccessor()
+
+@DslMarker
+annotation class DefaultsDsl

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/DependencyProvider.kt

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.services
+
+import org.jetbrains.kotlin.test.model.*
+
+abstract class DependencyProvider : TestService {
+    abstract fun getTestModule(name: String): TestModule
+
+    abstract fun <A : ResultingArtifact<A>> getArtifact(module: TestModule, kind: TestArtifactKind<A>): A
+}
+
+val TestServices.dependencyProvider: DependencyProvider by TestServices.testServiceAccessor()
+
+class DependencyProviderImpl(
+    private val testServices: TestServices,
+    testModules: List<TestModule>
+) : DependencyProvider() {
+    private val assertions: Assertions
+        get() = testServices.assertions
+
+    private val testModulesByName = testModules.map { it.name to it }.toMap()
+
+    private val artifactsByModule: MutableMap<TestModule, MutableMap<TestArtifactKind<*>, ResultingArtifact<*>>> = mutableMapOf()
+
+    override fun getTestModule(name: String): TestModule {
+        return testModulesByName[name] ?: assertions.fail { "Module $name is not defined" }
+    }
+
+    override fun <A : ResultingArtifact<A>> getArtifact(module: TestModule, kind: TestArtifactKind<A>): A {
+        val artifact = artifactsByModule.getMap(module)[kind]
+            ?: error("Artifact with kind $kind is not registered for module ${module.name}")
+        @Suppress("UNCHECKED_CAST")
+        return artifact as A
+    }
+
+    fun <A : ResultingArtifact<A>> registerArtifact(module: TestModule, artifact: ResultingArtifact<A>) {
+        val kind = artifact.kind
+        val previousValue = artifactsByModule.getMap(module).put(kind, artifact)
+        if (previousValue != null) error("Artifact with kind $kind already registered for module ${module.name}")
+    }
+
+    private fun <K, V, R> MutableMap<K, MutableMap<V, R>>.getMap(key: K): MutableMap<V, R> {
+        return getOrPut(key) { mutableMapOf() }
+    }
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/EnvironmentConfigurator.kt

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.services
+
+import com.intellij.mock.MockProject
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.test.directives.model.ComposedRegisteredDirectives
+import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
+import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
+import org.jetbrains.kotlin.test.model.TestModule
+
+abstract class EnvironmentConfigurator(protected val testServices: TestServices) {
+    open val directivesContainers: List<DirectivesContainer>
+        get() = emptyList()
+
+    open val additionalServices: List<ServiceRegistrationData>
+        get() = emptyList()
+
+    protected val moduleStructure: TestModuleStructure
+        get() = testServices.moduleStructure
+
+    protected val TestModule.allRegisteredDirectives: RegisteredDirectives
+        get() = ComposedRegisteredDirectives(directives, testServices.defaultDirectives)
+
+    open fun configureCompilerConfiguration(configuration: CompilerConfiguration, module: TestModule, project: MockProject) {}
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/GlobalMetadataInfoHandler.kt

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.services
+
+import org.jetbrains.kotlin.codeMetaInfo.CodeMetaInfoParser
+import org.jetbrains.kotlin.codeMetaInfo.CodeMetaInfoRenderer
+import org.jetbrains.kotlin.codeMetaInfo.model.CodeMetaInfo
+import org.jetbrains.kotlin.codeMetaInfo.model.ParsedCodeMetaInfo
+import org.jetbrains.kotlin.test.model.TestFile
+import org.jetbrains.kotlin.test.model.TestModule
+
+class GlobalMetadataInfoHandler(
+    private val testServices: TestServices,
+    private val processors: List<AdditionalMetaInfoProcessor>
+) : TestService {
+    private lateinit var existingInfosPerFile: Map<TestFile, List<ParsedCodeMetaInfo>>
+
+    private val infosPerFile: MutableMap<TestFile, MutableList<CodeMetaInfo>> =
+        mutableMapOf<TestFile, MutableList<CodeMetaInfo>>().withDefault { mutableListOf() }
+
+    private val existingInfosPerFilePerInfoCache = mutableMapOf<Pair<TestFile, CodeMetaInfo>, List<ParsedCodeMetaInfo>>()
+
+    @OptIn(ExperimentalStdlibApi::class)
+    fun parseExistingMetadataInfosFromAllSources() {
+        existingInfosPerFile = buildMap {
+            for (file in testServices.moduleStructure.modules.flatMap { it.files }) {
+                put(file, CodeMetaInfoParser.getCodeMetaInfoFromText(file.originalContent))
+            }
+        }
+    }
+
+    fun getExistingMetaInfosForFile(file: TestFile): List<ParsedCodeMetaInfo> {
+        return existingInfosPerFile.getValue(file)
+    }
+
+    fun getReportedMetaInfosForFile(file: TestFile): List<CodeMetaInfo> {
+        return infosPerFile.getValue(file)
+    }
+
+    fun getExistingMetaInfosForActualMetadata(file: TestFile, metaInfo: CodeMetaInfo): List<ParsedCodeMetaInfo> {
+        return existingInfosPerFilePerInfoCache.getOrPut(file to metaInfo) {
+            getExistingMetaInfosForFile(file).filter { it == metaInfo }
+        }
+    }
+
+    fun addMetadataInfosForFile(file: TestFile, codeMetaInfos: List<CodeMetaInfo>) {
+        val infos = infosPerFile.getOrPut(file) { mutableListOf() }
+        infos += codeMetaInfos
+    }
+
+    fun compareAllMetaDataInfos() {
+        // TODO: adapt to multiple testdata files
+        val moduleStructure = testServices.moduleStructure
+        val builder = StringBuilder()
+        for (module in moduleStructure.modules) {
+            for (file in module.files) {
+                if (file.isAdditional) continue
+                processors.forEach { it.processMetaInfos(module, file) }
+                val codeMetaInfos = infosPerFile.getValue(file)
+                CodeMetaInfoRenderer.renderTagsToText(
+                    builder,
+                    codeMetaInfos,
+                    testServices.sourceFileProvider.getContentOfSourceFile(file)
+                )
+            }
+        }
+        val actualText = builder.toString()
+        testServices.assertions.assertEqualsToFile(moduleStructure.originalTestDataFiles.single(), actualText)
+    }
+}
+
+val TestServices.globalMetadataInfoHandler: GlobalMetadataInfoHandler by TestServices.testServiceAccessor()
+
+abstract class AdditionalMetaInfoProcessor(protected val testServices: TestServices) {
+    protected val globalMetadataInfoHandler: GlobalMetadataInfoHandler
+        get() = testServices.globalMetadataInfoHandler
+
+    abstract fun processMetaInfos(module: TestModule, file: TestFile)
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/MetaTestConfigurator.kt

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.services
+
+import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
+
+abstract class MetaTestConfigurator(protected val testServices: TestServices) {
+    open val directives: List<DirectivesContainer>
+        get() = emptyList()
+
+    open fun transformTestDataPath(testDataFileName: String): String = testDataFileName
+
+    open fun shouldSkipTest(): Boolean = false
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/ModuleStructureExtractor.kt

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.services
+
+import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
+
+abstract class ModuleStructureExtractor(
+    protected val testServices: TestServices,
+    protected val additionalSourceProviders: List<AdditionalSourceProvider>
+) {
+    abstract fun splitTestDataByModules(
+        testDataFileName: String,
+        directivesContainer: DirectivesContainer,
+    ): TestModuleStructure
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/SourceFileProvider.kt

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.services
+
+import com.intellij.openapi.project.Project
+import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.test.model.TestFile
+import org.jetbrains.kotlin.test.util.KtTestUtil
+import java.io.File
+
+abstract class SourceFilePreprocessor(val testServices: TestServices) {
+    abstract fun process(file: TestFile, content: String): String
+}
+
+abstract class SourceFileProvider : TestService {
+    abstract val kotlinSourceDirectory: File
+    abstract val javaSourceDirectory: File
+
+    abstract fun getContentOfSourceFile(testFile: TestFile): String
+    abstract fun getRealFileForSourceFile(testFile: TestFile): File
+}
+
+val TestServices.sourceFileProvider: SourceFileProvider by TestServices.testServiceAccessor()
+
+class SourceFileProviderImpl(val preprocessors: List<SourceFilePreprocessor>) : SourceFileProvider() {
+    override val kotlinSourceDirectory: File = KtTestUtil.tmpDir("kotlin-files")
+    override val javaSourceDirectory: File = KtTestUtil.tmpDir("java-files")
+
+    private val contentOfFiles = mutableMapOf<TestFile, String>()
+    private val realFileMap = mutableMapOf<TestFile, File>()
+
+    override fun getContentOfSourceFile(testFile: TestFile): String {
+        return contentOfFiles.getOrPut(testFile) {
+            generateFinalContent(testFile)
+        }
+    }
+
+    override fun getRealFileForSourceFile(testFile: TestFile): File {
+        return realFileMap.getOrPut(testFile) {
+            val directory = when {
+                testFile.isKtFile -> kotlinSourceDirectory
+                testFile.isJavaFile -> javaSourceDirectory
+                else -> error("Unknown file type: ${testFile.name}")
+            }
+            directory.resolve(testFile.relativePath).also {
+                it.parentFile.mkdirs()
+                it.writeText(getContentOfSourceFile(testFile))
+            }
+        }
+    }
+
+    private fun generateFinalContent(testFile: TestFile): String {
+        return preprocessors.fold(testFile.originalContent) { content, preprocessor ->
+            preprocessor.process(testFile, content)
+        }
+    }
+}
+
+fun SourceFileProvider.getKtFileForSourceFile(testFile: TestFile, project: Project): KtFile {
+// TODO
+//    return TestCheckerUtil.createCheckAndReturnPsiFile(
+    return KtTestUtil.createFile(
+        testFile.name,
+        getContentOfSourceFile(testFile),
+        project
+    )
+}
+
+fun SourceFileProvider.getKtFilesForSourceFiles(testFiles: Collection<TestFile>, project: Project): Map<TestFile, KtFile> {
+    return testFiles.mapNotNull {
+        if (!it.isKtFile) return@mapNotNull null
+        it to getKtFileForSourceFile(it, project)
+    }.toMap()
+}
+
+val TestFile.isKtFile: Boolean
+    get() = name.endsWith(".kt") || name.endsWith(".kts")
+
+val TestFile.isKtsFile: Boolean
+    get() = name.endsWith(".kts")
+
+val TestFile.isJavaFile: Boolean
+    get() = name.endsWith(".java")

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/TestModuleStructure.kt

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.services
+
+import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
+import org.jetbrains.kotlin.test.model.BinaryKind
+import org.jetbrains.kotlin.test.model.TestModule
+import java.io.File
+
+abstract class TestModuleStructure : TestService {
+    abstract val modules: List<TestModule>
+    abstract val allDirectives: RegisteredDirectives
+    abstract val originalTestDataFiles: List<File>
+
+    abstract fun getTargetArtifactKinds(module: TestModule): List<BinaryKind<*>>
+}
+
+val TestServices.moduleStructure: TestModuleStructure by TestServices.testServiceAccessor()

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/TestServices.kt

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.services
+
+import org.jetbrains.kotlin.fir.utils.ArrayMapAccessor
+import org.jetbrains.kotlin.fir.utils.ComponentArrayOwner
+import org.jetbrains.kotlin.fir.utils.TypeRegistry
+import kotlin.reflect.KClass
+
+interface TestService
+
+data class ServiceRegistrationData(
+    val kClass: KClass<out TestService>,
+    val serviceConstructor: (TestServices) -> TestService
+)
+
+inline fun <reified T : TestService> service(
+    noinline serviceConstructor: (TestServices) -> T
+): ServiceRegistrationData {
+    return ServiceRegistrationData(T::class, serviceConstructor)
+}
+
+class TestServices : ComponentArrayOwner<TestService, TestService>(){
+    override val typeRegistry: TypeRegistry<TestService, TestService>
+        get() = Companion
+
+    companion object : TypeRegistry<TestService, TestService>() {
+        inline fun <reified T : TestService> testServiceAccessor(): ArrayMapAccessor<TestService, TestService, T> {
+            return generateAccessor(T::class)
+        }
+    }
+
+    fun register(data: ServiceRegistrationData) {
+        registerComponent(data.kClass, data.serviceConstructor(this))
+    }
+
+    fun register(kClass: KClass<out TestService>, service: TestService) {
+        registerComponent(kClass, service)
+    }
+
+    fun register(data: List<ServiceRegistrationData>) {
+        data.forEach { register(it) }
+    }
+}
+
+fun TestServices.registerDependencyProvider(dependencyProvider: DependencyProvider) {
+    register(DependencyProvider::class, dependencyProvider)
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/impl/RegisteredDirectivesParser.kt

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.services.impl
+
+import org.jetbrains.kotlin.test.directives.model.*
+import org.jetbrains.kotlin.test.services.Assertions
+
+class RegisteredDirectivesParser(private val container: DirectivesContainer, private val assertions: Assertions) {
+    companion object {
+        private val DIRECTIVE_PATTERN = Regex("""^//\s*[!]?([A-Z0-9_]+)(:[ \t]*(.*))? *$""")
+        private val SPACES_PATTERN = Regex("""[,]?[ \t]+""")
+        private const val NAME_GROUP = 1
+        private const val VALUES_GROUP = 3
+
+        fun parseDirective(line: String): RawDirective? {
+            val result = DIRECTIVE_PATTERN.matchEntire(line)?.groupValues ?: return null
+            val name = result.getOrNull(NAME_GROUP) ?: return null
+            val values = result.getOrNull(VALUES_GROUP)?.split(SPACES_PATTERN)?.filter { it.isNotBlank() }?.takeIf { it.isNotEmpty() }
+            return RawDirective(name, values)
+        }
+    }
+
+    data class RawDirective(val name: String, val values: List<String>?)
+    data class ParsedDirective(val directive: Directive, val values: List<*>)
+
+    private val simpleDirectives = mutableListOf<SimpleDirective>()
+    private val stringValueDirectives = mutableMapOf<StringDirective, MutableList<String>>()
+    private val valueDirectives = mutableMapOf<ValueDirective<*>, MutableList<Any>>()
+
+    /**
+     * returns true means that line contain directive
+     */
+    fun parse(line: String): Boolean {
+        val rawDirective = parseDirective(line) ?: return false
+        val parsedDirective = convertToRegisteredDirective(rawDirective) ?: return false
+        addParsedDirective(parsedDirective)
+        return true
+    }
+
+    fun addParsedDirective(parsedDirective: ParsedDirective) {
+        val (directive, values) = parsedDirective
+        when (directive) {
+            is SimpleDirective -> simpleDirectives += directive
+            is StringDirective -> {
+                val list = stringValueDirectives.getOrPut(directive, ::mutableListOf)
+                @Suppress("UNCHECKED_CAST")
+                list += values as List<String>
+            }
+            is ValueDirective<*> -> {
+                val list = valueDirectives.getOrPut(directive, ::mutableListOf)
+                @Suppress("UNCHECKED_CAST")
+                list.addAll(values as List<Any>)
+            }
+        }
+    }
+
+    fun convertToRegisteredDirective(rawDirective: RawDirective): ParsedDirective? {
+        val (name, rawValues) = rawDirective
+        val directive = container[name] ?: return null
+
+        val values: List<*> = when (directive) {
+            is SimpleDirective -> {
+                if (rawValues != null) {
+                    assertions.fail {
+                        "Directive $directive should have no arguments, but ${rawValues.joinToString(", ")} are passed"
+                    }
+                }
+                emptyList<Any?>()
+            }
+
+            is StringDirective -> {
+                rawValues ?: emptyList<Any?>()
+            }
+
+            is ValueDirective<*> -> {
+                if (rawValues == null) {
+                    assertions.fail {
+                        "Directive $directive must have at least one value"
+                    }
+                }
+                rawValues.map { directive.extractValue(it) ?: assertions.fail { "$it is not valid value for $directive" } }
+            }
+        }
+        return ParsedDirective(directive, values)
+    }
+
+    private fun <T : Any> ValueDirective<T>.extractValue(name: String): T? {
+        return parser.invoke(name)
+    }
+
+    fun build(): RegisteredDirectives {
+        return RegisteredDirectivesImpl(simpleDirectives, stringValueDirectives, valueDirectives)
+    }
+}

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/impl/TargetPlatformParser.kt

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.services.impl
+
+import org.jetbrains.kotlin.config.JvmTarget
+import org.jetbrains.kotlin.platform.TargetPlatform
+import org.jetbrains.kotlin.platform.js.JsPlatform
+import org.jetbrains.kotlin.platform.js.JsPlatforms
+import org.jetbrains.kotlin.platform.jvm.JdkPlatform
+import org.jetbrains.kotlin.platform.konan.NativePlatform
+import org.jetbrains.kotlin.test.util.joinToArrayString
+import org.jetbrains.kotlin.utils.addToStdlib.runIf
+
+object TargetPlatformParser {
+    private const val JVM = "JVM"
+    private const val JDK = "JDK"
+    private const val JS = "JS"
+
+    fun parseTargetPlatform(declaredPlatforms: List<String>): TargetPlatform? {
+        if (declaredPlatforms.isEmpty()) return null
+        val simplePlatforms = declaredPlatforms.mapTo(mutableSetOf()) { platformString ->
+            tryParseJdkPlatform(platformString)?.let { return@mapTo it }
+            tryParseJsPlatform(platformString)?.let { return@mapTo it }
+            tryParseNativePlatform(platformString)?.let { return@mapTo it }
+            error("Unknown platform: $platformString")
+        }
+        return TargetPlatform(simplePlatforms)
+    }
+
+    private fun tryParseJdkPlatform(platformString: String): JdkPlatform? {
+        val target = when {
+            platformString == JVM -> JvmTarget.DEFAULT
+            !platformString.startsWith(JDK) -> return null
+            else -> JvmTarget.values().find { it.name == platformString }
+                ?: error("JvmTarget \"$platformString\" not found.\nAvailable targets: ${JvmTarget.values().joinToArrayString()}")
+        }
+        return JdkPlatform(target)
+    }
+
+    private fun tryParseJsPlatform(platformString: String): JsPlatform? {
+        return runIf(platformString == JS) { JsPlatforms.DefaultSimpleJsPlatform }
+    }
+
+    private fun tryParseNativePlatform(platformString: String): NativePlatform? {
+        // TODO: support native platforms
+        return null
+    }
+}

settings.gradle

@@ -326,7 +326,8 @@ include ":compiler:fir:cones",
         ":compiler:fir:entrypoint",
         ":compiler:fir:analysis-tests"
 
-include ":compiler:test-infrastructure-utils"
+include ":compiler:test-infrastructure",
+        ":compiler:test-infrastructure-utils"
 
 include ":idea:idea-frontend-fir:idea-fir-low-level-api"
 include ":idea:idea-fir-performance-tests"