[K/N] Migrate most simple filecheck tests to new infra

^KT-62157 Merge-request: KT-MR-12376 Merged-by: Vladimir Sukharev <Vladimir.Sukharev@jetbrains.com>
2023-10-23 10:07:28 +00:00
parent 3d399a8b25
commit e43b634122
113 changed files with 3245 additions and 1595 deletions
@@ -188,7 +188,8 @@ internal class TestCase(
    val freeCompilerArgs: TestCompilerArgs,
    val nominalPackageName: PackageName,
    val checks: TestRunChecks,
-    val extras: Extras
+    val extras: Extras,
+    val fileCheckStage: String? = null, // KT-62157: TODO move it to extras
 ) {
    sealed interface Extras
    class NoTestRunnerExtras(val entryPoint: String, val inputDataFile: File? = null, val arguments: List<String> = emptyList()) : Extras
@@ -144,6 +144,10 @@ internal object TestDirectives : SimpleDirectivesContainer() {
    val FIR_IDENTICAL by directive(
        description = "Test behavior should be identical for FIR testing"
    )
+
+    val FILECHECK_STAGE by stringDirective(
+        description = "Specify a LLVM stage to dump LLVM IR after, and check it with LLVM FileCheck using its directives in test file"
+    )
 }

 internal enum class TestKind {
@@ -10,6 +10,7 @@ import org.jetbrains.kotlin.konan.properties.resolvablePropertyList
 import org.jetbrains.kotlin.konan.test.blackbox.support.*
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestCase.*
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestModule.Companion.allDependsOn
+import org.jetbrains.kotlin.konan.test.blackbox.support.compilation.ExecutableCompilation.Companion.applyFileCheckArgs
 import org.jetbrains.kotlin.konan.test.blackbox.support.compilation.ExecutableCompilation.Companion.applyPartialLinkageArgs
 import org.jetbrains.kotlin.konan.test.blackbox.support.compilation.ExecutableCompilation.Companion.applyTestRunnerSpecificArgs
 import org.jetbrains.kotlin.konan.test.blackbox.support.compilation.ExecutableCompilation.Companion.assertTestDumpFileNotEmptyIfExists
@@ -362,6 +363,7 @@ internal class ExecutableCompilation(
            }
        }
        applyPartialLinkageArgs(partialLinkageConfig)
+        applyFileCheckArgs(expectedArtifact.fileCheckStage, expectedArtifact.fileCheckDump)
        super.applySpecificArgs(argsBuilder)
    }

@@ -403,6 +405,13 @@ internal class ExecutableCompilation(
                    add("-Xpartial-linkage-loglevel=${logLevel.name.lowercase()}")
            }
        }
+
+        internal fun ArgsBuilder.applyFileCheckArgs(fileCheckStage: String?, fileCheckDump: File?) =
+            fileCheckStage?.let {
+                add("-Xllvm-variant=dev")  // FileCheck utility is provided in `LLVM dev`, not `LLVM user`
+                add("-Xsave-llvm-ir-after=$it")
+                add("-Xsave-llvm-ir-directory=${fileCheckDump!!.parent}")
+            }
    }
 }

@@ -464,6 +473,7 @@ internal class StaticCacheCompilation(
            add("-Xmake-per-file-cache")

        applyPartialLinkageArgs(partialLinkageConfig)
+        applyFileCheckArgs(expectedArtifact.fileCheckStage, expectedArtifact.fileCheckDump)
    }

    override fun applyDependencies(argsBuilder: ArgsBuilder): Unit = with(argsBuilder) {
@@ -15,14 +15,22 @@ internal sealed interface TestCompilationArtifact {
        override val logFile: File get() = klibFile.resolveSibling("${klibFile.name}.log")
    }

-    data class KLIBStaticCache(val cacheDir: File, val klib: KLIB) : TestCompilationArtifact {
+    data class KLIBStaticCache(val cacheDir: File, val klib: KLIB, val fileCheckStage: String? = null) : TestCompilationArtifact {
        override val logFile: File get() = cacheDir.resolve("${klib.klibFile.nameWithoutExtension}-cache.log")
+        val fileCheckDump: File?
+            get() = fileCheckStage?.let {
+                cacheDir.resolveSibling("out.$it.ll")
+            }
    }

-    data class Executable(val executableFile: File) : TestCompilationArtifact {
+    data class Executable(val executableFile: File, val fileCheckStage: String? = null) : TestCompilationArtifact {
        val path: String get() = executableFile.path
        override val logFile: File get() = executableFile.resolveSibling("${executableFile.name}.log")
        val testDumpFile: File get() = executableFile.resolveSibling("${executableFile.name}.dump")
+        val fileCheckDump: File?
+            get() = fileCheckStage?.let {
+                executableFile.resolveSibling("out.$it.ll")
+            }
    }

    data class ObjCFramework(private val buildDir: File, val frameworkName: String) : TestCompilationArtifact {
@@ -117,7 +117,10 @@ internal class TestCompilationFactory {
        // Long pass.
        val freeCompilerArgs = rootModules.first().testCase.freeCompilerArgs // Should be identical inside the same test case group.
        val extras = testCases.first().extras // Should be identical inside the same test case group.
-        val executableArtifact = Executable(settings.artifactFileForExecutable(rootModules))
+        val fileCheckStage = testCases.map { it.fileCheckStage }.singleOrNull()
+        if (fileCheckStage != null)
+            require(testCases.size == 1) { "FILECHECK-enabled test must be standalone" }
+        val executableArtifact = Executable(settings.artifactFileForExecutable(rootModules), fileCheckStage)

        val (
            dependenciesToCompileExecutable: Iterable<CompiledDependency<*>>,
@@ -21,6 +21,8 @@ import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.LanguageFeature
 import org.jetbrains.kotlin.konan.test.blackbox.support.*
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestCase.WithTestRunnerExtras
+import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.FILECHECK_STAGE
+import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck
 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunChecks
 import org.jetbrains.kotlin.konan.test.blackbox.support.settings.*
 import org.jetbrains.kotlin.konan.test.blackbox.support.util.*
@@ -75,6 +77,8 @@ internal class ExtTestCaseGroupProvider : TestCaseGroupProvider, TestDisposable(
                    customKlibs = settings.get(),
                    pipelineType = settings.get(),
                    testMode = settings.get(),
+                    cacheMode = settings.get(),
+                    optimizationMode = settings.get(),
                    timeouts = settings.get(),
                )

@@ -99,8 +103,10 @@ private class ExtTestDataFile(
    testRoots: TestRoots,
    private val generatedSources: GeneratedSources,
    private val customKlibs: CustomKlibs,
-    private val pipelineType: PipelineType,
-    private val testMode: TestMode,
+    pipelineType: PipelineType,
+    testMode: TestMode,
+    cacheMode: CacheMode,
+    optimizationMode: OptimizationMode,
    private val timeouts: Timeouts,
 ) {
    private val structure by lazy {
@@ -144,6 +150,9 @@ private class ExtTestDataFile(
                && INCOMPATIBLE_DIRECTIVES.none { it in structure.directives }
                && structure.directives[API_VERSION_DIRECTIVE] !in INCOMPATIBLE_API_VERSIONS
                && structure.directives[LANGUAGE_VERSION_DIRECTIVE] !in INCOMPATIBLE_LANGUAGE_VERSIONS
+                && !(FILECHECK_STAGE.name in structure.directives
+                     && (cacheMode as? CacheMode.WithStaticCache)?.useStaticCacheForUserLibraries == true)
+                && !(optimizationMode != OptimizationMode.OPT && structure.directives[FILECHECK_STAGE.name] == "OptimizeTLSDataLoads")
                && !(testDataFileSettings.languageSettings.contains("+${LanguageFeature.MultiPlatformProjects.name}")
                     && pipelineType == PipelineType.K2
                     && testMode == TestMode.ONE_STAGE_MULTI_MODULE)
@@ -194,7 +203,7 @@ private class ExtTestDataFile(
        val entryPointFunctionFQN = findEntryPoint()
        generateTestLauncher(isStandaloneTest, entryPointFunctionFQN)

-        return doCreateTestCase(isStandaloneTest, sharedModules)
+        return doCreateTestCase(settings, isStandaloneTest, sharedModules)
    }

    /**
@@ -204,6 +213,7 @@ private class ExtTestDataFile(
     */
    private fun determineIfStandaloneTest(): Boolean = with(structure) {
        if (directives.contains(NATIVE_STANDALONE_DIRECTIVE)) return true
+        if (directives.contains(FILECHECK_STAGE.name)) return true

        var isStandaloneTest = false

@@ -502,6 +512,7 @@ private class ExtTestDataFile(
    }

    private fun doCreateTestCase(
+        settings: Settings,
        isStandaloneTest: Boolean,
        sharedModules: ThreadSafeCache<String, TestModule.Shared?>
    ): TestCase = with(structure) {
@@ -513,14 +524,16 @@ private class ExtTestDataFile(
                }
            }
        )
-
+        val fileCheckStage = retrieveFileCheckStage()
        val testCase = TestCase(
            id = TestCaseId.TestDataFile(testDataFile),
            kind = if (isStandaloneTest) TestKind.STANDALONE else TestKind.REGULAR,
            modules = modules,
            freeCompilerArgs = assembleFreeCompilerArgs(),
            nominalPackageName = testDataFileSettings.nominalPackageName,
-            checks = TestRunChecks.Default(timeouts.executionTimeout),
+            checks = TestRunChecks.Default(timeouts.executionTimeout)
+                .copy(fileCheckMatcher = fileCheckStage?.let { TestRunCheck.FileCheckMatcher(settings, testDataFile) }),
+            fileCheckStage = fileCheckStage,
            extras = WithTestRunnerExtras(runnerType = TestRunnerType.DEFAULT)
        )
        testCase.initialize(
@@ -531,6 +544,20 @@ private class ExtTestDataFile(
        return testCase
    }

+    private fun retrieveFileCheckStage(): String? {
+        val fileCheckStages = structure.directives.multiValues(FILECHECK_STAGE.name)
+        return when (fileCheckStages.size) {
+            0 -> {
+                require(!isDirectiveDefined(testDataFile.readText(), FILECHECK_STAGE.name)) {
+                    "In ${testDataFile.absolutePath}, one argument for FILECHECK directive is needed: LLVM stage name, to dump bitcode after"
+                }
+                null
+            }
+            1 -> fileCheckStages.single()
+            else -> fail { "In ${testDataFile.absolutePath}, only one argument for FILECHECK directive is allowed: $fileCheckStages" }
+        }
+    }
+
    companion object {
        private val INCOMPATIBLE_DIRECTIVES = setOf("FULL_JDK", "JVM_TARGET", "DIAGNOSTICS")

@@ -227,7 +227,8 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
                computeExecutionTimeoutCheck(settings, expectedTimeoutFailure),
                computeExitCodeCheck(testKind, registeredDirectives, location),
                computeOutputDataFileCheck(testDataFile, registeredDirectives, location),
-                lldbSpec?.let { OutputMatcher { output -> lldbSpec.checkLLDBOutput(output, settings.get()) } }
+                lldbSpec?.let { OutputMatcher { output -> lldbSpec.checkLLDBOutput(output, settings.get()) } },
+                fileCheckMatcher = null,
            ),
            extras = when (testKind) {
                TestKind.STANDALONE_NO_TR -> {
@@ -5,18 +5,22 @@

 package org.jetbrains.kotlin.konan.test.blackbox.support.runner

+import com.intellij.openapi.util.SystemInfo
 import com.intellij.openapi.util.text.StringUtilRt.convertLineSeparators
 import kotlinx.coroutines.*
+import org.jetbrains.kotlin.konan.target.Architecture
+import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.AbstractRunner.AbstractRun
 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExecutionTimeout
 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExitCode
 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.UnfilteredProcessOutput.Companion.launchReader
+import org.jetbrains.kotlin.konan.test.blackbox.support.settings.KotlinNativeTargets
+import org.jetbrains.kotlin.konan.test.blackbox.support.settings.OptimizationMode
+import org.jetbrains.kotlin.konan.test.blackbox.support.settings.configurables
 import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestOutputFilter
+import org.jetbrains.kotlin.util.capitalizeDecapitalize.toUpperCaseAsciiOnly
 import org.junit.jupiter.api.Assertions.fail
-import java.io.ByteArrayOutputStream
-import java.io.IOException
-import java.io.InputStream
-import java.io.OutputStream
+import java.io.*
 import java.util.concurrent.atomic.AtomicBoolean
 import kotlin.time.*

@@ -43,7 +47,7 @@ internal abstract class AbstractLocalProcessRunner<R>(protected val checks: Test
            val ignoreIOErrorsInProcessOutput = AtomicBoolean(false)

            val duration = measureTime {
-                process = ProcessBuilder(programArgs).directory(executable.executableFile.parentFile).start()
+                process = ProcessBuilder(programArgs).directory(executable.executable.executableFile.parentFile).start()
                customizeProcess(process)

                unfilteredOutputReader = launchReader(
@@ -76,6 +80,7 @@ internal abstract class AbstractLocalProcessRunner<R>(protected val checks: Test
            }

            RunResult(
+                testExecutable = executable,
                exitCode = exitCode,
                timeout = executionTimeout,
                duration = duration,
@@ -140,6 +145,53 @@ internal abstract class LocalResultHandler<R>(
                        }
                    }
                }
+                is TestRunCheck.FileCheckMatcher -> {
+                    val fileCheckExecutable = check.settings.configurables.absoluteLlvmHome + File.separator + "bin" + File.separator +
+                            if (SystemInfo.isWindows) "FileCheck.exe" else "FileCheck"
+                    require(File(fileCheckExecutable).exists()) {
+                        "$fileCheckExecutable does not exist. Make sure Distribution for `settings.configurables` " +
+                                "was created using `propertyOverrides` to specify development variant of LLVM instead of user variant."
+                    }
+                    val fileCheckDump = runResult.testExecutable.executable.fileCheckDump!!
+                    val fileCheckOut = File(fileCheckDump.absolutePath + ".out")
+                    val fileCheckErr = File(fileCheckDump.absolutePath + ".err")
+
+                    val testTarget = check.settings.get<KotlinNativeTargets>().testTarget
+                    val checkPrefixes = buildList {
+                        add("CHECK")
+                        add("CHECK-${testTarget.abiInfoString}")
+                        add("CHECK-${testTarget.name.toUpperCaseAsciiOnly()}")
+                        if (testTarget.family.isAppleFamily) {
+                            add("CHECK-APPLE")
+                        }
+                    }
+                    val optimizationMode = check.settings.get<OptimizationMode>().name
+                    val checkPrefixesWithOptMode = checkPrefixes.map { "$it-$optimizationMode" }
+                    val commaSeparatedCheckPrefixes = (checkPrefixes + checkPrefixesWithOptMode).joinToString(",")
+
+                    val result = ProcessBuilder(
+                        fileCheckExecutable,
+                        check.testDataFile.absolutePath,
+                        "--input-file",
+                        fileCheckDump.absolutePath,
+                        "--check-prefixes", commaSeparatedCheckPrefixes,
+                        "--allow-deprecated-dag-overlap" // TODO specify it via new test directive for `function_attributes_at_callsite.kt`
+                    ).redirectOutput(fileCheckOut)
+                        .redirectError(fileCheckErr)
+                        .start()
+                        .waitFor()
+                    val errText = fileCheckErr.readText()
+                    val outText = fileCheckOut.readText()
+                    verifyExpectation(result == 0 && errText.isEmpty() && outText.isEmpty()) {
+                        val shortOutText = outText.lines().take(100)
+                        val shortErrText = errText.lines().take(100)
+                        "FileCheck matching of ${fileCheckDump.absolutePath}\n" +
+                                "with '--check-prefixes $commaSeparatedCheckPrefixes'\n" +
+                                "failed with result=$result:\n" +
+                                shortOutText.joinToString("\n") + "\n" +
+                                shortErrText.joinToString("\n")
+                    }
+                }
            }
        }

@@ -183,3 +235,11 @@ private class UnfilteredProcessOutput {
        }
    }
 }
+
+// Shameless borrowing `val KonanTarget.abiInfo` from module `:kotlin-native:backend.native`, which cannot be imported here for now.
+val KonanTarget.abiInfoString: String
+    get() = when {
+        this == KonanTarget.MINGW_X64 -> "WINDOWSX64"
+        !family.isAppleFamily && architecture == Architecture.ARM64 -> "AAPCS"
+        else -> "DEFAULTABI"
+    }
@@ -16,7 +16,7 @@ internal class LocalTestNameExtractor(
    checks: TestRunChecks
 ) : AbstractLocalProcessRunner<Collection<TestName>>(checks) {
    override val visibleProcessName get() = "Test name extractor"
-    override val programArgs = listOf(executable.executableFile.path, "--ktest_list_tests")
+    override val programArgs = listOf(executable.executable.executableFile.path, "--ktest_list_tests")
    override val outputFilter get() = TestOutputFilter.NO_FILTERING

    override fun getLoggedParameters() = LoggedData.TestRunParameters(
@@ -18,7 +18,7 @@ internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProc
    override val executable get() = testRun.executable

    override val programArgs = buildList {
-        add(executable.executableFile.path)
+        add(executable.executable.executableFile.path)
        testRun.runParameters.forEach { it.applyTo(this) }
    }

@@ -9,6 +9,7 @@ import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestOutputFilter
 import kotlin.time.Duration

 internal data class RunResult(
+    val testExecutable: TestExecutable, // KT-62157: TODO extract out of RunResult and pass Pair(TestExecutable, RunResult)
    val exitCode: Int?,
    val timeout: Duration,
    val duration: Duration,
@@ -41,7 +41,7 @@ internal class RunnerWithExecutor(
        val stdout = ByteArrayOutputStream()
        val stderr = ByteArrayOutputStream()
        val request = ExecuteRequest(
-            executableAbsolutePath = executable.executableFile.absolutePath,
+            executableAbsolutePath = executable.executable.executableFile.absolutePath,
            args = programArgs,
            stdin = stdin,
            stdout = stdout,
@@ -50,6 +50,7 @@ internal class RunnerWithExecutor(
        )
        val response = executor.execute(request)
        RunResult(
+            testExecutable = executable,
            exitCode = response.exitCode,
            timeout = request.timeout,
            duration = response.executionTime,
@@ -17,7 +17,7 @@ import java.io.File
 import java.io.IOException

 internal class TestExecutable(
-    val executableFile: File,
+    val executable: Executable,
    val loggedCompilationToolCall: LoggedData.CompilerCall,
    val testNames: Collection<TestName>
 ) {
@@ -42,7 +42,7 @@ internal class TestExecutable(
            }

            return TestExecutable(
-                executableFile = compilationResult.resultingArtifact.executableFile,
+                executable = compilationResult.resultingArtifact,
                loggedCompilationToolCall = compilationResult.loggedData,
                testNames = testNames
            )
@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.konan.test.blackbox.support.runner

 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.*
+import org.jetbrains.kotlin.konan.test.blackbox.support.settings.Settings
 import org.jetbrains.kotlin.utils.yieldIfNotNull
 import java.io.File
 import kotlin.time.Duration
@@ -24,13 +25,16 @@ internal sealed interface TestRunCheck {
    class OutputDataFile(val file: File) : TestRunCheck

    class OutputMatcher(val match: (String) -> Boolean): TestRunCheck
+
+    class FileCheckMatcher(val settings: Settings, val testDataFile: File): TestRunCheck
 }

-internal class TestRunChecks(
+internal data class TestRunChecks(
    val executionTimeoutCheck: ExecutionTimeout,
    private val exitCodeCheck: ExitCode,
    val outputDataFile: OutputDataFile?,
-    val outputMatcher: OutputMatcher?
+    val outputMatcher: OutputMatcher?,
+    val fileCheckMatcher: FileCheckMatcher?,
 ) : Iterable<TestRunCheck> {

    override fun iterator() = iterator {
@@ -38,6 +42,7 @@ internal class TestRunChecks(
        yield(exitCodeCheck)
        yieldIfNotNull(outputDataFile)
        yieldIfNotNull(outputMatcher)
+        yieldIfNotNull(fileCheckMatcher)
    }

    companion object {
@@ -47,7 +52,8 @@ internal class TestRunChecks(
            executionTimeoutCheck = ExecutionTimeout.ShouldNotExceed(timeout),
            exitCodeCheck = ExitCode.Expected(0),
            outputDataFile = null,
-            outputMatcher = null
+            outputMatcher = null,
+            fileCheckMatcher = null,
        )
    }
 }
@@ -23,9 +23,7 @@ internal object TestRunners {
            if (testTarget == hostTarget) {
                LocalTestRunner(testRun)
            } else {
-                val nativeHome = get<KotlinNativeHome>()
-                val distribution = Distribution(nativeHome.dir.path)
-                val configurables = PlatformManager(distribution, true).platform(testTarget).configurables
+                val configurables = configurables

                val executor = cached(
                    when {
@@ -5,6 +5,10 @@

 package org.jetbrains.kotlin.konan.test.blackbox.support.settings

+import org.jetbrains.kotlin.konan.target.Configurables
+import org.jetbrains.kotlin.konan.target.Distribution
+import org.jetbrains.kotlin.konan.target.HostManager
+import org.jetbrains.kotlin.konan.target.PlatformManager
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
 import kotlin.reflect.KClass
@@ -59,3 +63,13 @@ internal class TestRunSettings(parent: TestClassSettings, settings: Iterable<Any
 */
 internal class SimpleTestClassSettings(parent: TestProcessSettings, settings: Iterable<Any>) : Settings(parent, settings)
 internal class SimpleTestRunSettings(parent: SimpleTestClassSettings, settings: Iterable<Any>) : Settings(parent, settings)
+
+internal val Settings.configurables: Configurables
+    get() {
+        val distribution = Distribution(
+            get<KotlinNativeHome>().dir.path,
+            // Development variant of LLVM is used to have utilities like FileCheck
+            propertyOverrides = mapOf("llvmHome.${HostManager.hostName}" to "\$llvm.${HostManager.hostName}.dev")
+        )
+        return PlatformManager(distribution, true).platform(get<KotlinNativeTargets>().testTarget).configurables
+    }