From 1c24f994db26750a69e420d91a2c8bed0195ed12 Mon Sep 17 00:00:00 2001 From: Sergey Bogolepov Date: Tue, 5 Dec 2023 12:03:06 +0200 Subject: [PATCH] [Native][tests] Initial support for CExport in the new test infra Very trivial implementation of CExport test infrastructure based on work from LLDB tests and ObjCExport tests. --- .../testData/CExport/smoke0/main.c | 7 ++ .../testData/CExport/smoke0/main.kt | 8 ++ .../generators/tests/GenerateNativeTests.kt | 29 +++++ .../blackbox/AbstractNativeCExportTest.kt | 117 ++++++++++++++++++ .../support/compilation/TestCompilation.kt | 39 ++++++ .../compilation/TestCompilationArtifact.kt | 18 +++ .../compilation/TestCompilationFactory.kt | 38 ++++++ .../konan/test/blackbox/support/util/Clang.kt | 10 +- 8 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 native/native.tests/testData/CExport/smoke0/main.c create mode 100644 native/native.tests/testData/CExport/smoke0/main.kt create mode 100644 native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/AbstractNativeCExportTest.kt diff --git a/native/native.tests/testData/CExport/smoke0/main.c b/native/native.tests/testData/CExport/smoke0/main.c new file mode 100644 index 00000000000..7679cea42d7 --- /dev/null +++ b/native/native.tests/testData/CExport/smoke0/main.c @@ -0,0 +1,7 @@ +#include "smoke0_api.h" + +int main() { + if (foo() != 42) { + return -1; + } +} \ No newline at end of file diff --git a/native/native.tests/testData/CExport/smoke0/main.kt b/native/native.tests/testData/CExport/smoke0/main.kt new file mode 100644 index 00000000000..edd27ad1263 --- /dev/null +++ b/native/native.tests/testData/CExport/smoke0/main.kt @@ -0,0 +1,8 @@ +@file:OptIn(kotlin.experimental.ExperimentalNativeApi::class) + +import kotlin.native.CName + +@CName("foo") +fun foo(): Int { + return 42 +} \ No newline at end of file diff --git a/native/native.tests/tests/org/jetbrains/kotlin/generators/tests/GenerateNativeTests.kt b/native/native.tests/tests/org/jetbrains/kotlin/generators/tests/GenerateNativeTests.kt index c6c9eb7e17a..5e578f4cc8c 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/generators/tests/GenerateNativeTests.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/generators/tests/GenerateNativeTests.kt @@ -445,6 +445,35 @@ fun main() { model("standalone") } } + // C Export + testGroup("native/native.tests/tests-gen", "native/native.tests/testData") { + testClass( + suiteTestClassName = "CExportTestDynamicGenerated" + ) { + model("CExport", pattern = "^([^_](.+))$", recursive = false) + } + testClass( + suiteTestClassName = "FirCExportTestDynamicGenerated", + annotations = listOf( + *frontendFir() + ), + ) { + model("CExport", pattern = "^([^_](.+))$", recursive = false) + } + testClass( + suiteTestClassName = "CExportTestStaticGenerated" + ) { + model("CExport", pattern = "^([^_](.+))$", recursive = false) + } + testClass( + suiteTestClassName = "FirCExportTestStaticGenerated", + annotations = listOf( + *frontendFir() + ), + ) { + model("CExport", pattern = "^([^_](.+))$", recursive = false) + } + } } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/AbstractNativeCExportTest.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/AbstractNativeCExportTest.kt new file mode 100644 index 00000000000..36a26bd116f --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/AbstractNativeCExportTest.kt @@ -0,0 +1,117 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.test.blackbox + +import com.intellij.testFramework.TestDataFile +import org.jetbrains.kotlin.konan.test.blackbox.support.* +import org.jetbrains.kotlin.konan.test.blackbox.support.PackageName +import org.jetbrains.kotlin.konan.test.blackbox.support.TestCase +import org.jetbrains.kotlin.konan.test.blackbox.support.TestCaseId +import org.jetbrains.kotlin.konan.test.blackbox.support.TestCompilerArgs +import org.jetbrains.kotlin.konan.test.blackbox.support.TestFile +import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind +import org.jetbrains.kotlin.konan.test.blackbox.support.TestModule +import org.jetbrains.kotlin.konan.test.blackbox.support.compilation.TestCompilationArtifact +import org.jetbrains.kotlin.konan.test.blackbox.support.compilation.TestCompilationFactory +import org.jetbrains.kotlin.konan.test.blackbox.support.compilation.TestCompilationResult.Companion.assertSuccess +import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestExecutable +import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunChecks +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.Timeouts +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.configurables +import org.jetbrains.kotlin.konan.test.blackbox.support.util.DEFAULT_MODULE_NAME +import org.jetbrains.kotlin.konan.test.blackbox.support.util.compileWithClang +import org.jetbrains.kotlin.konan.test.blackbox.support.util.getAbsoluteFile +import org.junit.jupiter.api.Tag +import java.io.File + +@Tag("cexport") +abstract class AbstractNativeCExportTest( + protected val libraryKind: BinaryLibraryKind, +) : AbstractNativeSimpleTest() { + + enum class BinaryLibraryKind { + STATIC, DYNAMIC + } + + protected open val kindSpecificClangFlags: List = emptyList() + + private val testCompilationFactory = TestCompilationFactory() + + protected fun runTest(@TestDataFile testDir: String) { + val testPathFull = getAbsoluteFile(testDir) + val ktSources = testPathFull.list()!! + .filter { it.endsWith(".kt") } + .map { testPathFull.resolve(it) } + ktSources.forEach { muteTestIfNecessary(it) } + + val cSources = testPathFull.list()!! + .filter { it.endsWith(".c") } + .map { testPathFull.resolve(it) } + + val testCase = generateCExportTestCase(testPathFull, ktSources) + val binaryLibrary = testCompilationFactory.testCaseToBinaryLibrary( + testCase, + testRunSettings, + kind = libraryKind.mapToArtifactKind(), + ).result.assertSuccess().resultingArtifact + + val clangExecutableName = "clangMain" + val executableFile = File(buildDir, clangExecutableName) + val includeDirectories = binaryLibrary.headerFile?.let { listOf(it.parentFile) } ?: emptyList() + val libraryName = binaryLibrary.libraryFile.nameWithoutExtension.substringAfter("lib") + compileWithClang( + sourceFiles = cSources, + includeDirectories = includeDirectories, + outputFile = executableFile, + libraryDirectories = listOf(binaryLibrary.libraryFile.parentFile), + libraries = listOf(libraryName), + additionalLinkerFlags = kindSpecificClangFlags, + ) + + val testExecutable = TestExecutable( + TestCompilationArtifact.Executable(executableFile), + loggedCompilationToolCall = LoggedData.NoopCompilerCall(buildDir), + testNames = listOf(TestName("TMP")), + ) + + runExecutableAndVerify(testCase, testExecutable) + } + + private fun generateCExportTestCase(testPathFull: File, sources: List): TestCase { + val moduleName: String = testPathFull.name + val module = TestModule.Exclusive(DEFAULT_MODULE_NAME, emptySet(), emptySet(), emptySet()) + sources.forEach { module.files += TestFile.createCommitted(it, module) } + + return TestCase( + id = TestCaseId.Named(moduleName), + kind = TestKind.STANDALONE_NO_TR, + modules = setOf(module), + freeCompilerArgs = TestCompilerArgs(listOf()), + nominalPackageName = PackageName(moduleName), + checks = TestRunChecks( + executionTimeoutCheck = TestRunCheck.ExecutionTimeout.ShouldNotExceed(testRunSettings.get().executionTimeout), + exitCodeCheck = TestRunCheck.ExitCode.Expected(0), + outputDataFile = goldenData?.let { TestRunCheck.OutputDataFile(file = it) }, + outputMatcher = null, + fileCheckMatcher = null, + ), + extras = TestCase.NoTestRunnerExtras(entryPoint = "main") + ).apply { + initialize(null, null) + } + } + + private fun BinaryLibraryKind.mapToArtifactKind(): TestCompilationArtifact.BinaryLibrary.Kind = when (this) { + BinaryLibraryKind.STATIC -> TestCompilationArtifact.BinaryLibrary.Kind.STATIC + BinaryLibraryKind.DYNAMIC -> TestCompilationArtifact.BinaryLibrary.Kind.DYNAMIC + } +} + +abstract class AbstractNativeCExportStaticTest() : AbstractNativeCExportTest(libraryKind = BinaryLibraryKind.STATIC) { + override val kindSpecificClangFlags: List + get() = testRunSettings.configurables.linkerKonanFlags.flatMap { listOf("-Xlinker", it) } +} +abstract class AbstractNativeCExportDynamicTest() : AbstractNativeCExportTest(libraryKind = BinaryLibraryKind.DYNAMIC) \ No newline at end of file diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/TestCompilation.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/TestCompilation.kt index 49ec90dff7c..65c93dcff92 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/TestCompilation.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/TestCompilation.kt @@ -271,6 +271,45 @@ internal class ObjCFrameworkCompilation( } } +internal class BinaryLibraryCompilation( + settings: Settings, + freeCompilerArgs: TestCompilerArgs, + sourceModules: Collection, + dependencies: Iterable>, + expectedArtifact: BinaryLibrary +) : SourceBasedCompilation( + targets = settings.get(), + home = settings.get(), + classLoader = settings.get(), + optimizationMode = settings.get(), + compilerOutputInterceptor = settings.get(), + threadStateChecker = settings.get(), + sanitizer = settings.get(), + gcType = settings.get(), + gcScheduler = settings.get(), + allocator = settings.get(), + pipelineType = settings.getStageDependentPipelineType(), + freeCompilerArgs = freeCompilerArgs, + compilerPlugins = settings.get(), + sourceModules = sourceModules, + dependencies = CategorizedDependencies(dependencies), + expectedArtifact = expectedArtifact +) { + override val binaryOptions get() = BinaryOptions.RuntimeAssertionsMode.defaultForTesting(optimizationMode, freeCompilerArgs.assertionsMode) + + override fun applySpecificArgs(argsBuilder: ArgsBuilder) = with(argsBuilder) { + val libraryKind = when (expectedArtifact.kind) { + BinaryLibrary.Kind.STATIC -> "static" + BinaryLibrary.Kind.DYNAMIC -> "dynamic" + } + add( + "-produce", libraryKind, + "-output", expectedArtifact.libraryFile.absolutePath + ) + super.applySpecificArgs(argsBuilder) + } +} + internal class GivenLibraryCompilation(givenArtifact: KLIB) : TestCompilation() { override val result = TestCompilationResult.Success(givenArtifact, LoggedData.NoopCompilerCall(givenArtifact.klibFile)) } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/TestCompilationArtifact.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/TestCompilationArtifact.kt index 6e97a8d9eaa..36871a8c916 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/TestCompilationArtifact.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/TestCompilationArtifact.kt @@ -39,4 +39,22 @@ internal sealed interface TestCompilationArtifact { val headersDir: File get () = frameworkDir.resolve("Headers") val mainHeader: File get() = headersDir.resolve("$frameworkName.h") } + + data class BinaryLibrary(val libraryFile: File, val kind: Kind) : TestCompilationArtifact { + + enum class Kind { + STATIC, DYNAMIC + } + + override val logFile: File get() = libraryFile.resolveSibling("${libraryFile.name}.log") + + /** + * Might not exist if the library was compiled without a header. + */ + val headerFile: File? + get() { + val expectedFile = libraryFile.resolveSibling("${libraryFile.nameWithoutExtension}_api.h") + return expectedFile.takeIf { it.exists() } + } + } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/TestCompilationFactory.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/TestCompilationFactory.kt index 8817460a356..9d2f5d5312f 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/TestCompilationFactory.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/TestCompilationFactory.kt @@ -28,10 +28,12 @@ internal class TestCompilationFactory { private val cachedKlibCompilations = ThreadSafeCache() private val cachedExecutableCompilations = ThreadSafeCache>() private val cachedObjCFrameworkCompilations = ThreadSafeCache() + private val cachedBinaryLibraryCompilations = ThreadSafeCache() private data class KlibCacheKey(val sourceModules: Set, val freeCompilerArgs: TestCompilerArgs) private data class ExecutableCacheKey(val sourceModules: Set) private data class ObjCFrameworkCacheKey(val sourceModules: Set) + private data class BinaryLibraryCacheKey(val sourceModules: Set, val kind: BinaryLibrary.Kind) // A pair of compilations for a KLIB itself and for its static cache that are created together. private data class KlibCompilations(val klib: TestCompilation, val staticCache: TestCompilation?) @@ -84,6 +86,29 @@ internal class TestCompilationFactory { } } + fun testCaseToBinaryLibrary(testCase: TestCase, settings: Settings, kind: BinaryLibrary.Kind): BinaryLibraryCompilation { + val rootModules = testCase.rootModules + val cacheKey = BinaryLibraryCacheKey(testCase.rootModules, kind) + cachedBinaryLibraryCompilations[cacheKey]?.let { return it } + + val ( + dependencies: Iterable>, + sourceModules: Set + ) = getDependenciesAndSourceModules(settings, testCase.rootModules, testCase.freeCompilerArgs) { + ProduceStaticCache.No + } + val expectedArtifact = BinaryLibrary(settings.artifactFileForBinaryLibrary(rootModules, kind), kind = kind) + return cachedBinaryLibraryCompilations.computeIfAbsent(cacheKey) { + BinaryLibraryCompilation( + settings = settings, + freeCompilerArgs = testCase.freeCompilerArgs, + sourceModules = sourceModules, + dependencies = dependencies, + expectedArtifact = expectedArtifact + ) + } + } + fun testCaseToObjCFrameworkCompilation(testCase: TestCase, settings: Settings): ObjCFrameworkCompilation { val cacheKey = ObjCFrameworkCacheKey(testCase.rootModules) cachedObjCFrameworkCompilations[cacheKey]?.let { return it } @@ -298,6 +323,19 @@ internal class TestCompilationFactory { private fun Settings.artifactFileForExecutable(module: TestModule.Exclusive) = singleModuleArtifactFile(module, get().testTarget.family.exeSuffix) + private fun Settings.pickBinaryLibrarySuffix(kind: BinaryLibrary.Kind) = when (kind) { + BinaryLibrary.Kind.STATIC -> get().testTarget.family.staticSuffix + BinaryLibrary.Kind.DYNAMIC -> get().testTarget.family.dynamicSuffix + } + + private fun Settings.artifactFileForBinaryLibrary(modules: Set, kind: BinaryLibrary.Kind) = when (modules.size) { + 1 -> artifactFileForBinaryLibrary(modules.first(), kind) + else -> multiModuleArtifactFile(modules, pickBinaryLibrarySuffix(kind)) + } + + private fun Settings.artifactFileForBinaryLibrary(module: TestModule.Exclusive, kind: BinaryLibrary.Kind) = + singleModuleArtifactFile(module, pickBinaryLibrarySuffix(kind)) + private fun Settings.artifactFileForKlib(modules: Set, freeCompilerArgs: TestCompilerArgs): File = when (modules.size) { 1 -> when (val module = modules.first()) { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/util/Clang.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/util/Clang.kt index a3b46b95fbd..fed57e61c1e 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/util/Clang.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/util/Clang.kt @@ -14,15 +14,23 @@ import java.io.File internal fun AbstractNativeSimpleTest.compileWithClang( sourceFiles: List, outputFile: File, - frameworkDirectories: List + includeDirectories: List = emptyList(), + frameworkDirectories: List = emptyList(), + libraryDirectories: List = emptyList(), + libraries: List = emptyList(), + additionalLinkerFlags: List = emptyList(), ) { val process = ProcessBuilder( "${testRunSettings.configurables.absoluteTargetToolchain}/bin/clang", *sourceFiles.map { it.absolutePath }.toTypedArray(), + *includeDirectories.flatMap { listOf("-I", it.absolutePath) }.toTypedArray(), "-isysroot", testRunSettings.configurables.absoluteTargetSysRoot, "-target", testRunSettings.configurables.targetTriple.toString(), "-g", "-fmodules", *frameworkDirectories.flatMap { listOf("-F", it.absolutePath) }.toTypedArray(), + *libraryDirectories.flatMap { listOf("-L", it.absolutePath) }.toTypedArray(), + *libraries.map { "-l$it" }.toTypedArray(), + *additionalLinkerFlags.toTypedArray(), "-o", outputFile.absolutePath ).redirectErrorStream(true).start() val clangOutput = process.inputStream.readBytes()