[K/N] Treat DEF files as regular test source, compiled with cinterop

^KT-61259
2023-12-05 00:36:00 +01:00
parent 2530cba82a
commit 0b0ba3160f
57 changed files with 988 additions and 934 deletions
@@ -208,20 +208,6 @@ fun main() {
            ) {
                model("CInterop/experimental/cases", pattern = "^([^_](.+))$", recursive = false)
            }
-
-            testClass<AbstractNativeCInteropExecutableTest>(
-                suiteTestClassName = "ClassicNativeCInteropExecutableTest",
-            ) {
-                model("CInterop/executable", pattern = "^([^_](.+))$", recursive = false)
-            }
-            testClass<AbstractNativeCInteropExecutableTest>(
-                suiteTestClassName = "FirNativeCInteropExecutableTest",
-                annotations = listOf(
-                    *frontendFir()
-                ),
-            ) {
-                model("CInterop/executable", pattern = "^([^_](.+))$", recursive = false)
-            }
        }

        // ObjCExport tests.
@@ -1,70 +0,0 @@
-/*
- * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.konan.test.blackbox
-
-import com.intellij.testFramework.TestDataFile
-import org.jetbrains.kotlin.konan.test.blackbox.support.ClassLevelProperty
-import org.jetbrains.kotlin.konan.test.blackbox.support.EnforcedProperty
-import org.jetbrains.kotlin.konan.test.blackbox.support.TestCase
-import org.jetbrains.kotlin.konan.test.blackbox.support.TestCompilerArgs
-import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind
-import org.jetbrains.kotlin.konan.test.blackbox.support.compilation.TestCompilationResult.Companion.assertSuccess
-import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestExecutable
-import org.jetbrains.kotlin.konan.test.blackbox.support.settings.PipelineType
-import java.io.File
-
-@EnforcedProperty(ClassLevelProperty.COMPILER_OUTPUT_INTERCEPTOR, "NONE")
-abstract class AbstractNativeCInteropExecutableTest : AbstractNativeSimpleTest() {
-
-    protected fun runTest(@TestDataFile testDataDir: String) {
-        val testName = File(testDataDir).name
-        val defFile = File(testDataDir, "$testName.def")
-        muteCInteropTestIfNecessary(defFile, targets.testTarget)
-
-        val interopSourceFileM = File(testDataDir, "$testName.m")
-        val interopSourceFileCPP = File(testDataDir, "$testName.cpp")
-        val interopSourceFile = when {
-            interopSourceFileM.exists() -> interopSourceFileM
-            interopSourceFileCPP.exists() -> interopSourceFileCPP
-            else -> null
-        }
-        val ktFile = File(testDataDir, "$testName.kt")
-
-        val compilerArgs = buildList {
-            add("-compiler-option")
-            add("-I$testDataDir")
-            val auxHeader = File(testDataDir, "$testName.aux.h")
-            if (auxHeader.exists()) {
-                add("-header")
-                add(auxHeader.absolutePath)
-            }
-            interopSourceFile?.let {
-                add("-Xcompile-source")
-                add(it.absolutePath)
-                add("-Xsource-compiler-option")
-                add("-fobjc-arc")
-                add("-Xsource-compiler-option")
-                add("-DNS_FORMAT_ARGUMENT(A)=")
-            }
-        }
-        val library = cinteropToLibrary(
-            targets = targets,
-            defFile = defFile,
-            outputDir = buildDir,
-            freeCompilerArgs = TestCompilerArgs(compilerArgs)
-        ).assertSuccess().resultingArtifact
-
-        val testCase = generateTestCaseWithSingleFile(
-            sourceFile = ktFile,
-            freeCompilerArgs = TestCompilerArgs(testRunSettings.get<PipelineType>().compilerFlags),
-            testKind = TestKind.STANDALONE_NO_TR,
-            extras = TestCase.NoTestRunnerExtras("main")
-        )
-        val compilationResult = compileToExecutable(testCase, library.asLibraryDependency()).assertSuccess()
-        val testExecutable = TestExecutable.fromCompilationResult(testCase, compilationResult)
-        runExecutableAndVerify(testCase, testExecutable)
-    }
-}
@@ -37,7 +37,7 @@ abstract class AbstractNativeCInteropKT39120Test : AbstractNativeCInteropBaseTes
        val golden1File = testPathFull.resolve("pod1.contents.gold.txt")
        val golden2File = testPathFull.resolve("pod2.contents.gold.txt")

-        val includeFrameworkArgs = TestCompilerArgs("-compiler-option", "-F${testDataDir.canonicalPath}")
+        val includeFrameworkArgs = TestCInteropArgs("-compiler-option", "-F${testDataDir.canonicalPath}")
        val klib1: KLIB = cinteropToLibrary(targets, def1File, buildDir, includeFrameworkArgs).assertSuccess().resultingArtifact
        val metadata1 = klib1.dumpMetadata(kotlinNativeClassLoader.classLoader, false, null)

@@ -45,7 +45,7 @@ abstract class AbstractNativeCInteropKT39120Test : AbstractNativeCInteropBaseTes
        val actualFiltered1Output = filterContentsOutput(metadata1, " pod.Version|POD|class Pod")
        assertEquals(StringUtilRt.convertLineSeparators(expectedFiltered1Output), StringUtilRt.convertLineSeparators(actualFiltered1Output))

-        val cinterop2ExtraArgs = TestCompilerArgs("-l", klib1.klibFile.canonicalPath, "-compiler-option", "-fmodules")
+        val cinterop2ExtraArgs = TestCInteropArgs("-l", klib1.klibFile.canonicalPath, "-compiler-option", "-fmodules")
        val klib2: KLIB = cinteropToLibrary(targets, def2File, buildDir, includeFrameworkArgs + cinterop2ExtraArgs).assertSuccess().resultingArtifact
        val metadata2 = klib2.dumpMetadata(kotlinNativeClassLoader.classLoader, false, null)

@@ -60,7 +60,7 @@ abstract class AbstractNativeCInteropKT39120Test : AbstractNativeCInteropBaseTes
                files += TestFile.createCommitted(ktFile, this)
            }
            val compilationResult = compileToExecutable(
-                createTestCaseNoTestRun(module, TestCompilerArgs(listOf())),
+                createTestCaseNoTestRun(module, TestCompilerArgs.EMPTY),
                klib1.asLibraryDependency(),
                klib2.asLibraryDependency()
            )
@@ -9,6 +9,7 @@ import com.intellij.openapi.util.text.StringUtilRt
 import com.intellij.testFramework.TestDataFile
 import org.jetbrains.kotlin.konan.target.Family
 import org.jetbrains.kotlin.konan.target.KonanTarget
+import org.jetbrains.kotlin.konan.test.blackbox.support.TestCInteropArgs
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestCompilerArgs
 import org.jetbrains.kotlin.konan.test.blackbox.support.compilation.TestCompilationResult
 import org.jetbrains.kotlin.konan.test.blackbox.support.compilation.TestCompilationResult.Companion.assertSuccess
@@ -88,11 +89,11 @@ abstract class AbstractNativeCInteropTest : AbstractNativeCInteropBaseTest() {
            getBuiltinsGoldenFile(testPathFull)
        else
            getGoldenFile(testPathFull)
-        val fmodulesArgs = if (fmodules) TestCompilerArgs("-compiler-option", "-fmodules") else TestCompilerArgs.EMPTY
+        val fmodulesArgs = if (fmodules) TestCInteropArgs("-compiler-option", "-fmodules") else TestCompilerArgs.EMPTY
        val includeArgs = if (testDataDir.name.startsWith("framework"))
-            TestCompilerArgs("-compiler-option", "-F${testDataDir.canonicalPath}")
+            TestCInteropArgs("-compiler-option", "-F${testDataDir.canonicalPath}")
        else
-            TestCompilerArgs("-compiler-option", "-I${includeFolder.canonicalPath}")
+            TestCInteropArgs("-compiler-option", "-I${includeFolder.canonicalPath}")

        val testCompilationResult = cinteropToLibrary(targets, defFile, buildDir, includeArgs + fmodulesArgs)
        // If we are running fmodules-specific test without -fmodules then we want to be sure that cinterop fails the way we want it to.
@@ -177,7 +177,7 @@ class LinkerOutputTestKT55578 : AbstractNativeLinkerOutputTest() {

    private fun compileKlib(defFile: File, sourceFile: File? = null, extraArgs: List<String> = emptyList()): KLIB {
        val sourceArguments = sourceFile?.let { listOf("-Xcompile-source", sourceFile.absolutePath) } ?: emptyList()
-        return cinteropToLibrary(targets, defFile, buildDir, TestCompilerArgs(extraArgs + sourceArguments))
+        return cinteropToLibrary(targets, defFile, buildDir, TestCInteropArgs(extraArgs + sourceArguments))
            .assertSuccess().resultingArtifact
    }
 }
@@ -8,6 +8,7 @@ package org.jetbrains.kotlin.konan.test.blackbox.support
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.ENTRY_POINT
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.EXIT_CODE
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.EXPECTED_TIMEOUT_FAILURE
+import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.FREE_CINTEROP_ARGS
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.FREE_COMPILER_ARGS
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.INPUT_DATA_FILE
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.KIND
@@ -129,6 +130,10 @@ internal object TestDirectives : SimpleDirectivesContainer() {
        description = "Specify free compiler arguments for Kotlin/Native compiler"
    )

+    val FREE_CINTEROP_ARGS by stringDirective(
+        description = "Specify free CInterop tool arguments"
+    )
+
    val LLDB_TRACE by stringDirective(
        description = """
            Specify a filename containing the LLDB commands and the patterns that
@@ -210,15 +215,23 @@ internal enum class MutedOption {
    K2
 }

-internal class TestCompilerArgs(val compilerArgs: List<String>) {
+internal class TestCInteropArgs(cinteropArgs: List<String>) : TestCompilerArgs(emptyList(), cinteropArgs) {
+    constructor(vararg cinteropArgs: String) : this(cinteropArgs.asList())
+}
+
+internal open class TestCompilerArgs(val compilerArgs: List<String>, val cinteropArgs: List<String> = emptyList()) {
    constructor(vararg compilerArgs: String) : this(compilerArgs.asList())

    private val uniqueCompilerArgs = compilerArgs.toSet()
-    override fun hashCode() = uniqueCompilerArgs.hashCode()
-    override fun equals(other: Any?) = (other as? TestCompilerArgs)?.uniqueCompilerArgs == uniqueCompilerArgs
+    private val uniqueCinteropArgs = cinteropArgs.toSet()
+    override fun hashCode() = (uniqueCompilerArgs + uniqueCinteropArgs).hashCode()
+    override fun equals(other: Any?) = (other as? TestCompilerArgs)?.uniqueCompilerArgs == uniqueCompilerArgs &&
+            other.uniqueCinteropArgs == uniqueCinteropArgs

-    operator fun plus(otherCompilerArgs: TestCompilerArgs): TestCompilerArgs = this + otherCompilerArgs.compilerArgs
-    operator fun plus(otherCompilerArgs: List<String>): TestCompilerArgs = TestCompilerArgs(compilerArgs + otherCompilerArgs)
+    operator fun plus(otherCompilerArgs: TestCompilerArgs): TestCompilerArgs = TestCompilerArgs(
+        this.compilerArgs + otherCompilerArgs.compilerArgs,
+        this.cinteropArgs + otherCompilerArgs.cinteropArgs
+    )

    companion object {
        val EMPTY = TestCompilerArgs(emptyList())
@@ -330,9 +343,15 @@ internal fun parseFileName(parsedDirective: RegisteredDirectivesParser.ParsedDir
            """.trimIndent()
        }

-    assertTrue(fileName.endsWith(".kt") && fileName.length > 3 && '/' !in fileName && '\\' !in fileName) {
-        "$location: Invalid file name in ${parsedDirective.directive} directive: $fileName"
-    }
+    val fileExtension = fileName.split(".").last()
+    if (fileExtension in setOf("kt", "def", "h", "c", "cpp", "m"))
+        assertTrue(fileName.length > 3 && '/' !in fileName && '\\' !in fileName) {
+            "$location: Invalid file name with extension $fileExtension in ${parsedDirective.directive} directive: $fileName"
+        }
+    else
+        assertTrue(false) {
+            "$location: Invalid file extension .$fileExtension in ${parsedDirective.directive} directive: $fileName"
+        }

    return fileName
 }
@@ -356,19 +375,18 @@ internal fun parseExpectedExitCode(registeredDirectives: RegisteredDirectives, l
 }

 internal fun parseFreeCompilerArgs(registeredDirectives: RegisteredDirectives, location: Location): TestCompilerArgs {
-    if (FREE_COMPILER_ARGS !in registeredDirectives)
-        return TestCompilerArgs.EMPTY
-
+    val freeCInteropArgs = registeredDirectives[FREE_CINTEROP_ARGS]
    val freeCompilerArgs = registeredDirectives[FREE_COMPILER_ARGS]
-    val forbiddenCompilerArgs = TestCompilerArgs.findForbiddenArgs(freeCompilerArgs)
-    assertTrue(forbiddenCompilerArgs.isEmpty()) {
-        """
+    if (freeCompilerArgs.isNotEmpty()) {
+        val forbiddenCompilerArgs = TestCompilerArgs.findForbiddenArgs(freeCompilerArgs)
+        assertTrue(forbiddenCompilerArgs.isEmpty()) {
+            """
            $location: Forbidden compiler arguments found in $FREE_COMPILER_ARGS directive: $forbiddenCompilerArgs
            All arguments: $freeCompilerArgs
        """.trimIndent()
+        }
    }
-
-    return TestCompilerArgs(freeCompilerArgs)
+    return TestCompilerArgs(freeCompilerArgs, freeCInteropArgs)
 }

 internal fun parseOutputDataFile(baseDir: File, registeredDirectives: RegisteredDirectives, location: Location): OutputDataFile? =
@@ -273,11 +273,25 @@ internal class CInteropCompilation(
    classLoader: KotlinNativeClassLoader,
    freeCompilerArgs: TestCompilerArgs,
    defFile: File,
+    sources: List<File> = emptyList(),
    expectedArtifact: KLIB
 ) : TestCompilation<KLIB>() {

    override val result: TestCompilationResult<out KLIB> by lazy {
-        val extraArgsArray = freeCompilerArgs.compilerArgs.toTypedArray()
+        val extraArgsArray = buildList {
+            addAll(freeCompilerArgs.cinteropArgs)
+            sources.forEach {
+                add("-Xcompile-source")
+                add(it.absolutePath)
+            }
+            add("-Xsource-compiler-option")
+            add("-fobjc-arc")
+            add("-Xsource-compiler-option")
+            add("-DNS_FORMAT_ARGUMENT(A)=")
+            add("-compiler-option")
+            add("-I${defFile.parentFile}")
+        }.toTypedArray()
+
        val loggedCInteropParameters = LoggedData.CInteropParameters(extraArgs = extraArgsArray, defFile = defFile)
        val (loggedCall: LoggedData, immediateResult: TestCompilationResult.ImmediateResult<out KLIB>) = try {
            val (exitCode, cinteropOutput, cinteropOutputHasErrors, duration) = invokeCInterop(
@@ -429,7 +443,8 @@ internal class StaticCacheCompilation(
    private val options: Options,
    private val pipelineType: PipelineType,
    dependencies: Iterable<TestCompilationDependency<*>>,
-    expectedArtifact: KLIBStaticCache
+    expectedArtifact: KLIBStaticCache,
+    makePerFileCacheOverride: Boolean? = null,
 ) : BasicCompilation<KLIBStaticCache>(
    targets = settings.get(),
    home = settings.get(),
@@ -454,7 +469,7 @@ internal class StaticCacheCompilation(
        cacheMode.staticCacheForDistributionLibrariesRootDir ?: fail { "No cache root directory found for cache mode $cacheMode" }
    }

-    private val makePerFileCache: Boolean = settings.get<CacheMode>().makePerFileCaches
+    private val makePerFileCache: Boolean = makePerFileCacheOverride ?: settings.get<CacheMode>().makePerFileCaches

    private val partialLinkageConfig: UsedPartialLinkageConfig = settings.get()

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.konan.test.blackbox.support.compilation

 import org.jetbrains.kotlin.container.topologicalSort
+import org.jetbrains.kotlin.konan.test.blackbox.muteCInteropTestIfNecessary
 import org.jetbrains.kotlin.konan.test.blackbox.support.PackageName
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestCase
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestCase.*
@@ -196,16 +197,41 @@ internal class TestCompilationFactory {
        }

        return cachedKlibCompilations.computeIfAbsent(cacheKey) {
-            val klibCompilation = if (isGivenKlibArtifact)
-                GivenLibraryCompilation(klibArtifact)
-            else
-                LibraryCompilation(
-                    settings = settings,
-                    freeCompilerArgs = freeCompilerArgs,
-                    sourceModules = sourceModules.flatMapToSet { sortDependsOnTopologically(it) },
-                    dependencies = dependencies.forKlib(),
-                    expectedArtifact = klibArtifact
-                )
+            val (klibCompilation, makePerFileCacheOverride) = if (isGivenKlibArtifact)
+                GivenLibraryCompilation(klibArtifact) to null
+            else {
+                val filesByExtension = sourceModules.first().files
+                    .map { it.location }
+                    .groupBy { it.name.split(".").last() }
+                when {
+                    filesByExtension.contains("kt") -> LibraryCompilation(
+                        settings = settings,
+                        freeCompilerArgs = freeCompilerArgs,
+                        sourceModules = sourceModules.flatMapToSet { sortDependsOnTopologically(it) },
+                        dependencies = dependencies.forKlib(),
+                        expectedArtifact = klibArtifact
+                    ) to null
+                    filesByExtension.contains("def") -> {
+                        val defFile = filesByExtension["def"]!!.single()
+                        muteCInteropTestIfNecessary(defFile, settings.get<KotlinNativeTargets>().testTarget)
+
+                        val cSourceFiles = buildList {
+                            for (ext in listOf("cpp", "c", "m")) {
+                                filesByExtension[ext]?.let { addAll(it) }
+                            }
+                        }
+                        CInteropCompilation(
+                            classLoader = settings.get(),
+                            targets = settings.get(),
+                            freeCompilerArgs = freeCompilerArgs,
+                            defFile = defFile,
+                            sources = cSourceFiles,
+                            expectedArtifact = klibArtifact
+                        ) to false // CInterop klib cannot be compiled into per-file cache
+                    }
+                    else -> error("Test module must contain either KT or DEF file")
+                }
+            }

            val staticCacheCompilation: StaticCacheCompilation? =
                staticCacheArtifactAndOptions?.let { (staticCacheArtifact, staticCacheOptions) ->
@@ -215,7 +241,8 @@ internal class TestCompilationFactory {
                        options = staticCacheOptions,
                        pipelineType = settings.get(),
                        dependencies = dependencies.forStaticCache(klibCompilation.asKlibDependency(type = /* does not matter in fact*/ Library)),
-                        expectedArtifact = staticCacheArtifact
+                        expectedArtifact = staticCacheArtifact,
+                        makePerFileCacheOverride = makePerFileCacheOverride,
                    )
                }