[BTA tests] Add base assertions DSL

^KT-61860 In Progress

compiler/build-tools/kotlin-build-tools-api-tests/build.gradle.kts

@@ -17,6 +17,7 @@ kotlin {
         optIn.add("org.jetbrains.kotlin.buildtools.api.ExperimentalBuildToolsApi")
         optIn.add("kotlin.ExperimentalStdlibApi")
         optIn.add("kotlin.io.path.ExperimentalPathApi")
+        freeCompilerArgs.add("-Xcontext-receivers")
     }
 }
 

compiler/build-tools/kotlin-build-tools-api-tests/src/testCommon/kotlin/compilation/assertions/filesAssertions.kt

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.buildtools.api.tests.compilation.assertions
+
+import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.CompilationOutcome
+import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.LogLevel
+import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.Module
+import org.junit.jupiter.api.Assertions.assertEquals
+import java.nio.file.Path
+import kotlin.io.path.isRegularFile
+import kotlin.io.path.relativeTo
+import kotlin.io.path.walk
+
+context(Module)
+fun CompilationOutcome.assertCompiledSources(vararg expectedCompiledSources: String) {
+    assertCompiledSources(expectedCompiledSources.toSet())
+}
+
+context(Module)
+fun CompilationOutcome.assertCompiledSources(expectedCompiledSources: Set<String>) {
+    requireLogLevel(LogLevel.DEBUG)
+    val actualCompiledSources = logLines.getValue(LogLevel.DEBUG)
+        .filter { it.startsWith("compile iteration") }
+        .flatMap { it.replace("compile iteration: ", "").trim().split(", ") }
+        .toSet()
+    val normalizedPaths = expectedCompiledSources
+        .map { sourcesDirectory.resolve(it) }
+        .map { it.relativeTo(project.projectDirectory) }
+        .map(Path::toString)
+        .toSet()
+    assertEquals(normalizedPaths, actualCompiledSources) {
+        "Compiled sources do not match "
+    }
+}
+
+context(Module)
+fun CompilationOutcome.assertOutputs(vararg expectedOutputs: String) {
+    assertOutputs(expectedOutputs.toSet())
+}
+
+context(Module)
+fun CompilationOutcome.assertOutputs(expectedOutputs: Set<String>) {
+    val filesLeft = expectedOutputs.map { outputDirectory.resolve(it).relativeTo(outputDirectory) }
+        .toMutableSet()
+        .apply {
+            add(outputDirectory.resolve("META-INF/$moduleName.kotlin_module").relativeTo(outputDirectory))
+        }
+    val notDeclaredFiles = hashSetOf<Path>()
+    for (file in outputDirectory.walk()) {
+        if (!file.isRegularFile()) continue
+        val currentFile = file.relativeTo(outputDirectory)
+        filesLeft.remove(currentFile).also { wasPreviously ->
+            if (!wasPreviously) notDeclaredFiles.add(currentFile)
+        }
+    }
+    assert(filesLeft.isEmpty()) {
+        "The following files were declared as expected, but not actually produced: $filesLeft"
+    }
+    assert(notDeclaredFiles.isEmpty()) {
+        "The following files weren't declared as expected output files: $notDeclaredFiles"
+    }
+}

compiler/build-tools/kotlin-build-tools-api-tests/src/testCommon/kotlin/compilation/assertions/logAssertions.kt

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.buildtools.api.tests.compilation.assertions
+
+import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.CompilationOutcome
+import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.LogLevel
+
+fun CompilationOutcome.assertLogContainsLines(logLevel: LogLevel, vararg expectedLines: String) {
+    assertLogContainsLines(logLevel, expectedLines.toSet())
+}
+
+fun CompilationOutcome.assertLogContainsLines(logLevel: LogLevel, expectedLines: Set<String>) {
+    requireLogLevel(logLevel)
+    val absentLines = expectedLines.filter { !logLines.getValue(logLevel).contains(it) }
+    assert(absentLines.isEmpty()) {
+        """
+        |The following lines were expected to be printed on $logLevel level, however they were not:
+        |${absentLines.joinToString("\n")}
+        """.trimMargin()
+    }
+}
+
+fun CompilationOutcome.assertLogDoesNotContainLines(logLevel: LogLevel, vararg expectedLines: String) {
+    assertLogDoesNotContainLines(logLevel, expectedLines.toSet())
+}
+
+fun CompilationOutcome.assertLogDoesNotContainLines(logLevel: LogLevel, expectedLines: Set<String>) {
+    requireLogLevel(logLevel)
+    val presentLines = expectedLines.filter { logLines.getValue(logLevel).contains(it) }
+    assert(presentLines.isEmpty()) {
+        """
+        |The following lines were not expected to be printed on $logLevel level, however they were:
+        |${presentLines.joinToString("\n")}
+        """.trimMargin()
+    }
+}
+
+fun CompilationOutcome.assertLogContainsPatterns(logLevel: LogLevel, vararg expectedLines: Regex) {
+    assertLogContainsPatterns(logLevel, expectedLines.toSet())
+}
+
+fun CompilationOutcome.assertLogContainsPatterns(logLevel: LogLevel, expectedLines: Set<Regex>) {
+    requireLogLevel(logLevel)
+    val absentLines = expectedLines.filter { regex -> logLines.getValue(logLevel).none { line -> regex.matches(line) } }
+    assert(absentLines.isEmpty()) {
+        """
+        |The following lines were expected to be printed on $logLevel level, however they were not:
+        |${absentLines.joinToString("\n")}
+        """.trimMargin()
+    }
+}
+
+fun CompilationOutcome.assertLogDoesNotContainPatterns(logLevel: LogLevel, vararg expectedLines: Regex) {
+    assertLogDoesNotContainPatterns(logLevel, expectedLines.toSet())
+}
+
+fun CompilationOutcome.assertLogDoesNotContainPatterns(logLevel: LogLevel, expectedLines: Set<Regex>) {
+    requireLogLevel(logLevel)
+    val absentLines = expectedLines.filter { regex -> logLines.getValue(logLevel).any { line -> regex.matches(line) } }
+    assert(absentLines.isEmpty()) {
+        """
+        |The following lines were not expected to be printed on $logLevel level, however they were:
+        |${absentLines.joinToString("\n")}
+        """.trimMargin()
+    }
+}

compiler/build-tools/kotlin-build-tools-api-tests/src/testCommon/kotlin/compilation/model/AbstractModule.kt

@@ -5,10 +5,35 @@
 
 package org.jetbrains.kotlin.buildtools.api.tests.compilation.model
 
+import org.jetbrains.kotlin.buildtools.api.CompilationResult
+import org.jetbrains.kotlin.buildtools.api.CompilerExecutionStrategyConfiguration
+import org.jetbrains.kotlin.buildtools.api.jvm.JvmCompilationConfiguration
+import org.junit.jupiter.api.Assertions.assertEquals
 import java.nio.file.Path
 
+private class CompilationOutcomeImpl(
+    override val logLines: Map<LogLevel, Collection<String>>,
+) : CompilationOutcome {
+    var maxLogLevel: LogLevel = LogLevel.ERROR
+        private set
+    var expectedResult = CompilationResult.COMPILATION_SUCCESS
+        private set
+
+    override fun requireLogLevel(logLevel: LogLevel) {
+        maxLogLevel = maxOf(logLevel, maxLogLevel)
+    }
+
+    override fun expectFail() {
+        expectedResult = CompilationResult.COMPILATION_ERROR
+    }
+
+    override fun expectCompilationResult(compilationResult: CompilationResult) {
+        expectedResult = compilationResult
+    }
+}
+
 abstract class AbstractModule(
-    val project: Project,
+    override val project: Project,
     override val moduleName: String,
     val moduleDirectory: Path,
     val dependencies: List<Dependency>,
@@ -35,5 +60,40 @@ abstract class AbstractModule(
     override val icCachesDir: Path
         get() = icWorkingDir.resolve("caches")
 
+    override fun compile(
+        strategyConfig: CompilerExecutionStrategyConfiguration,
+        forceOutput: LogLevel?,
+        compilationConfigAction: (JvmCompilationConfiguration) -> Unit,
+        assertions: context(Module) CompilationOutcome.() -> Unit,
+    ): CompilationResult {
+        val kotlinLogger = TestKotlinLogger()
+        val result = compileImpl(strategyConfig, compilationConfigAction, kotlinLogger)
+        val outcome = CompilationOutcomeImpl(kotlinLogger.logMessagesByLevel)
+        try {
+            assertions(outcome)
+            assertEquals(outcome.expectedResult, result) {
+                "Compilation result is unexpected"
+            }
+            if (forceOutput != null) {
+                kotlinLogger.printBuildOutput(forceOutput)
+            }
+        } catch (e: AssertionError) {
+            val maxLogLevel = if (forceOutput != null) {
+                maxOf(forceOutput, outcome.maxLogLevel)
+            } else {
+                outcome.maxLogLevel
+            }
+            kotlinLogger.printBuildOutput(maxLogLevel)
+            throw e
+        }
+        return result
+    }
+
+    protected abstract fun compileImpl(
+        strategyConfig: CompilerExecutionStrategyConfiguration,
+        compilationConfigAction: (JvmCompilationConfiguration) -> Unit,
+        kotlinLogger: TestKotlinLogger,
+    ): CompilationResult
+
     override fun toString() = moduleName
 }

compiler/build-tools/kotlin-build-tools-api-tests/src/testCommon/kotlin/compilation/model/CompilationOutcome.kt

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.buildtools.api.tests.compilation.model
+
+import org.jetbrains.kotlin.buildtools.api.CompilationResult
+
+interface CompilationOutcome {
+    val logLines: Map<LogLevel, Collection<String>>
+
+    fun requireLogLevel(logLevel: LogLevel)
+
+    fun expectFail()
+
+    fun expectCompilationResult(compilationResult: CompilationResult)
+}

compiler/build-tools/kotlin-build-tools-api-tests/src/testCommon/kotlin/compilation/model/JvmModule.kt

@@ -14,6 +14,7 @@ import org.jetbrains.kotlin.buildtools.api.jvm.ClassSnapshotGranularity
 import org.jetbrains.kotlin.buildtools.api.jvm.ClasspathSnapshotBasedIncrementalCompilationApproachParameters
 import org.jetbrains.kotlin.buildtools.api.jvm.JvmCompilationConfiguration
 import org.jetbrains.kotlin.buildtools.api.tests.buildToolsVersion
+import org.jetbrains.kotlin.buildtools.api.tests.compilation.model.Module
 import org.jetbrains.kotlin.tooling.core.KotlinToolingVersion
 import java.io.File
 import java.nio.file.Path
@@ -37,15 +38,17 @@ class JvmModule(
 ) {
     private val compilationService = CompilationService.loadImplementation(this.javaClass.classLoader)
 
-    override fun compile(
+    override fun compileImpl(
         strategyConfig: CompilerExecutionStrategyConfiguration,
         compilationConfigAction: (JvmCompilationConfiguration) -> Unit,
+        kotlinLogger: TestKotlinLogger,
     ): CompilationResult {
         val stdlibLocation =
             KotlinVersion::class.java.protectionDomain.codeSource.location.toURI().toPath() // compile against the provided stdlib
         val dependencyFiles = dependencies.map { it.location }.plusElement(stdlibLocation)
         val compilationConfig = compilationService.makeJvmCompilationConfiguration()
         compilationConfigAction(compilationConfig)
+        compilationConfig.useLogger(kotlinLogger)
         val defaultCompilationArguments = listOf(
             "-no-reflect",
             "-no-stdlib",
@@ -79,10 +82,12 @@ class JvmModule(
     override fun compileIncrementally(
         strategyConfig: CompilerExecutionStrategyConfiguration,
         sourcesChanges: SourcesChanges,
+        forceOutput: LogLevel?,
         forceNonIncrementalCompilation: Boolean,
         compilationConfigAction: (JvmCompilationConfiguration) -> Unit,
+        assertions: context(Module) CompilationOutcome.() -> Unit,
     ): CompilationResult {
-        return compile(strategyConfig) { compilationConfig ->
+        return compile(strategyConfig, forceOutput, { compilationConfig ->
             val snapshots = dependencies.map {
                 generateClasspathSnapshot(it).toFile()
             }
@@ -111,6 +116,6 @@ class JvmModule(
                 params,
                 options,
             )
-        }
+        }, assertions)
     }
 }

compiler/build-tools/kotlin-build-tools-api-tests/src/testCommon/kotlin/compilation/model/LogLevel.kt

@@ -0,0 +1,14 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.buildtools.api.tests.compilation.model
+
+enum class LogLevel {
+    ERROR,
+    WARN,
+    LIFECYCLE,
+    INFO,
+    DEBUG,
+}

compiler/build-tools/kotlin-build-tools-api-tests/src/testCommon/kotlin/compilation/model/Module.kt

@@ -12,6 +12,7 @@ import org.jetbrains.kotlin.buildtools.api.jvm.JvmCompilationConfiguration
 import java.nio.file.Path
 
 interface Module : Dependency {
+    val project: Project
     val moduleName: String
     val additionalCompilationArguments: List<String>
     val sourcesDirectory: Path
@@ -22,13 +23,17 @@ interface Module : Dependency {
 
     fun compile(
         strategyConfig: CompilerExecutionStrategyConfiguration,
+        forceOutput: LogLevel? = null,
         compilationConfigAction: (JvmCompilationConfiguration) -> Unit = {},
+        assertions: context(Module) CompilationOutcome.() -> Unit = {},
     ): CompilationResult
 
     fun compileIncrementally(
         strategyConfig: CompilerExecutionStrategyConfiguration,
         sourcesChanges: SourcesChanges,
+        forceOutput: LogLevel? = null,
         forceNonIncrementalCompilation: Boolean = false,
         compilationConfigAction: (JvmCompilationConfiguration) -> Unit = {},
+        assertions: context(Module) CompilationOutcome.() -> Unit = {},
     ): CompilationResult
 }

compiler/build-tools/kotlin-build-tools-api-tests/src/testCommon/kotlin/compilation/model/TestKotlinLogger.kt

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.buildtools.api.tests.compilation.model
+
+import org.jetbrains.kotlin.buildtools.api.KotlinLogger
+import java.util.EnumMap
+import java.util.concurrent.ConcurrentLinkedQueue
+
+class LogEntry(
+    val logLevel: LogLevel,
+    val message: String,
+    val exception: Throwable? = null,
+) {
+    override fun toString() = "$logLevel: $message ${exception?.stackTraceToString() ?: ""}"
+}
+
+class TestKotlinLogger : KotlinLogger {
+    override val isDebugEnabled = true
+
+    private val logMessages = ConcurrentLinkedQueue<LogEntry>()
+    private val logMessagesByLevelImpl = EnumMap<LogLevel, ConcurrentLinkedQueue<String>>(LogLevel::class.java).apply {
+        for (logLevel in LogLevel.entries) {
+            put(logLevel, ConcurrentLinkedQueue())
+        }
+    }
+
+    val logMessagesByLevel: Map<LogLevel, Collection<String>> = logMessagesByLevelImpl
+
+    override fun debug(msg: String) {
+        saveLogEntry(LogLevel.DEBUG, msg)
+    }
+
+    override fun error(msg: String, throwable: Throwable?) {
+        saveLogEntry(LogLevel.ERROR, msg, throwable)
+    }
+
+    override fun info(msg: String) {
+        saveLogEntry(LogLevel.INFO, msg)
+    }
+
+    override fun lifecycle(msg: String) {
+        saveLogEntry(LogLevel.LIFECYCLE, msg)
+    }
+
+    override fun warn(msg: String) {
+        saveLogEntry(LogLevel.WARN, msg)
+    }
+
+    private fun saveLogEntry(logLevel: LogLevel, msg: String, throwable: Throwable? = null) {
+        logMessagesByLevelImpl.getValue(logLevel).add(msg)
+        logMessages.add(LogEntry(logLevel, msg, throwable))
+    }
+
+    fun printBuildOutput(maxLogLevel: LogLevel) {
+        println("Build output (with messages up to $maxLogLevel level): ")
+        for (logMessage in logMessages.filter { it.logLevel <= maxLogLevel }) {
+            println(logMessage)
+        }
+    }
+}