[Native][tests] Support passing custom (external) KLIBs as -l dependencies to Kotlin/Native compiler invocation

buildSrc/src/main/kotlin/nativeTest.kt

@@ -10,6 +10,7 @@ private enum class TestProperty(shortName: String) {
     // effect on other Gradle tasks (ex: :kotlin-native:dist) that might be executed along with test task.
     KOTLIN_NATIVE_HOME("nativeHome"),
     COMPILER_CLASSPATH("compilerClasspath"),
+    CUSTOM_KLIBS("customKlibs"),
     TEST_TARGET("target"),
     TEST_MODE("mode"),
     FORCE_STANDALONE("forceStandalone"),
@@ -33,7 +34,12 @@ private enum class TestProperty(shortName: String) {
     fun readGradleProperty(task: Test): String? = task.project.findProperty(propertyName)?.toString()
 }
 
-fun Project.nativeTest(taskName: String, tag: String?, vararg customDependencies: Configuration) = projectTest(
+fun Project.nativeTest(
+    taskName: String,
+    tag: String?,
+    customDependencies: List<Configuration> = emptyList(),
+    customKlibDependencies: List<Configuration> = emptyList()
+) = projectTest(
     taskName,
     jUnitMode = JUnitMode.JUnit5,
     maxHeapSizeMb = 3072 // Extra heap space for Kotlin/Native compiler.
@@ -88,6 +94,13 @@ fun Project.nativeTest(taskName: String, tag: String?, vararg customDependencies
             }.map { it.absoluteFile }.joinToString(";")
         }
 
+        TestProperty.CUSTOM_KLIBS.setUpFromGradleProperty(this) {
+            customKlibDependencies.flatMap { dependency ->
+                dependsOn(dependency)
+                dependency.files
+            }.map { it.absoluteFile }.joinToString(";")
+        }
+
         // Pass Gradle properties as JVM properties so test process can read them.
         TestProperty.TEST_TARGET.setUpFromGradleProperty(this)
         TestProperty.TEST_MODE.setUpFromGradleProperty(this)

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/AbstractNativeKlibABITest.kt

@@ -130,7 +130,7 @@ abstract class AbstractNativeKlibABITest : AbstractNativeSimpleTest() {
         checks = TestRunChecks.Default(testRunSettings.get<Timeouts>().executionTimeout),
         extras = DEFAULT_EXTRAS
     ).apply {
-        initialize(null)
+        initialize(null, null)
     }
 
     private fun createLibraryDependencies(dependencies: KlibABITestUtils.Dependencies): Iterable<TestCompilationDependency<KLIB>> =

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureDumpedTestListingTest.kt

@@ -116,7 +116,7 @@ class InfrastructureDumpedTestListingTest : AbstractNativeSimpleTest() {
             checks = TestRunChecks.Default(testRunSettings.get<Timeouts>().executionTimeout),
             extras = DEFAULT_EXTRAS
         ).apply {
-            initialize(null)
+            initialize(null, null)
         }
     }
 

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/ConfigurationProperties.kt

@@ -43,6 +43,7 @@ internal class EnforcedProperties(testClass: Class<*>) {
 internal enum class ClassLevelProperty(shortName: String) {
     TEST_TARGET("target"),
     TEST_MODE("mode"),
+    CUSTOM_KLIBS("customKlibs"),
     FORCE_STANDALONE("forceStandalone"),
     COMPILE_ONLY("compileOnly"),
     OPTIMIZATION_MODE("optimizationMode"),

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt

@@ -191,6 +191,7 @@ private object NativeTestSupport {
         output += sanitizer
         output += CacheMode::class to cacheMode
         output += computeTestMode(enforcedProperties)
+        output += computeCustomKlibs(enforcedProperties)
         output += computeForcedStandaloneTestKind(enforcedProperties)
         output += computeForcedNoopTestRunner(enforcedProperties)
         output += computeTimeouts(enforcedProperties)
@@ -260,6 +261,15 @@ private object NativeTestSupport {
     private fun computeTestMode(enforcedProperties: EnforcedProperties): TestMode =
         ClassLevelProperty.TEST_MODE.readValue(enforcedProperties, TestMode.values(), default = TestMode.TWO_STAGE_MULTI_MODULE)
 
+    private fun computeCustomKlibs(enforcedProperties: EnforcedProperties): CustomKlibs =
+        CustomKlibs(
+            ClassLevelProperty.CUSTOM_KLIBS.readValue(
+                enforcedProperties,
+                { it.split(':', ';').mapToSet(::File) },
+                default = emptySet()
+            )
+        )
+
     private fun computeForcedStandaloneTestKind(enforcedProperties: EnforcedProperties): ForcedStandaloneTestKind =
         ForcedStandaloneTestKind(
             ClassLevelProperty.FORCE_STANDALONE.readValue(
@@ -394,7 +404,7 @@ private object NativeTestSupport {
             .ensureExistsAndIsEmptyDirectory() // Clean-up the directory with all potentially stale generated sources.
 
         val sharedSourcesDir = testSourcesDir
-            .resolve("__shared_modules__")
+            .resolve(SHARED_MODULES_DIR_NAME)
             .ensureExistsAndIsEmptyDirectory()
 
         return GeneratedSources(testSourcesDir, sharedSourcesDir)
@@ -407,10 +417,14 @@ private object NativeTestSupport {
             .ensureExistsAndIsEmptyDirectory() // Clean-up the directory with all potentially stale artifacts.
 
         val sharedBinariesDir = testBinariesDir
-            .resolve("__shared_modules__")
+            .resolve(SHARED_MODULES_DIR_NAME)
             .ensureExistsAndIsEmptyDirectory()
 
-        return Binaries(testBinariesDir, sharedBinariesDir)
+        val givenBinariesDir = testBinariesDir
+            .resolve(GIVEN_MODULES_DIR_NAME)
+            .ensureExistsAndIsEmptyDirectory()
+
+        return Binaries(testBinariesDir, sharedBinariesDir, givenBinariesDir)
     }
 
     /*************** Test class settings (simplified) ***************/

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt

@@ -110,6 +110,11 @@ internal sealed class TestModule {
         override val files: FailOnDuplicatesSet<TestFile<Shared>> = FailOnDuplicatesSet()
     }
 
+    data class Given(val klibFile: File) : TestModule() {
+        override val name: String get() = klibFile.name
+        override val files: Set<TestFile<*>> get() = emptySet()
+    }
+
     final override fun equals(other: Any?) =
         other === this || (other is TestModule && other.javaClass == javaClass && other.name == name && other.files == files)
 
@@ -122,13 +127,13 @@ internal sealed class TestModule {
         val TestModule.allDependencies: Set<TestModule>
             get() = when (this) {
                 is Exclusive -> allDependencies
-                is Shared -> emptySet()
+                is Shared, is Given -> emptySet()
             }
 
         val TestModule.allFriends: Set<TestModule>
             get() = when (this) {
                 is Exclusive -> allFriends
-                is Shared -> emptySet()
+                is Shared, is Given -> emptySet()
             }
 
         private val SM = LockBasedStorageManager(TestModule::class.java.name)
@@ -222,7 +227,10 @@ internal class TestCase(
         }
     }
 
-    fun initialize(findSharedModule: ((moduleName: String) -> TestModule.Shared?)?) {
+    fun initialize(
+        givenModules: Set<TestModule.Given>?,
+        findSharedModule: ((moduleName: String) -> TestModule.Shared?)?
+    ) {
         // Check that there are no duplicated files among different modules.
         val duplicatedFiles = modules.flatMap { it.files }.groupingBy { it }.eachCount().filterValues { it > 1 }.keys
         assertTrue(duplicatedFiles.isEmpty()) { "$id: Duplicated test files encountered: $duplicatedFiles" }
@@ -242,7 +250,12 @@ internal class TestCase(
         modules.forEach { module ->
             module.commit() // Save to the file system and release the memory.
             module.testCase = this
-            module.directDependencies = module.directDependencySymbols.mapToSet(::findModule)
+
+            module.directDependencies = buildSet {
+                module.directDependencySymbols.mapTo(this, ::findModule)
+                givenModules?.let(this@buildSet::addAll)
+            }
+
             module.directFriends = module.directFriendSymbols.mapToSet(::findModule)
         }
     }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/compilation/TestCompilationFactory.kt

@@ -144,7 +144,7 @@ internal class TestCompilationFactory {
         val staticCacheArtifactAndOptions: Pair<KLIBStaticCache, StaticCacheCompilation.Options>? = when (produceStaticCache) {
             is ProduceStaticCache.No -> null // No artifact means no static cache should be compiled.
             is ProduceStaticCache.Yes -> KLIBStaticCache(
-                cacheDir = klibArtifact.cacheDirForStaticCache(),
+                cacheDir = settings.cacheDirForStaticCache(klibArtifact),
                 klib = klibArtifact
             ) to produceStaticCache.options
         }
@@ -221,6 +221,7 @@ internal class TestCompilationFactory {
                 1 -> when (val module = modules.first()) {
                     is TestModule.Exclusive -> singleModuleArtifactFile(module, "klib")
                     is TestModule.Shared -> get<Binaries>().sharedBinariesDir.resolve("${module.name}-${prettyHash(freeCompilerArgs.hashCode())}.klib")
+                    is TestModule.Given -> module.klibFile
                 }
                 else -> {
                     assertTrue(modules.none { module -> module is TestModule.Shared }) {
@@ -230,7 +231,19 @@ internal class TestCompilationFactory {
                 }
             }
 
-        private fun KLIB.cacheDirForStaticCache(): File = klibFile.parentFile.resolve(STATIC_CACHE_DIR_NAME).apply { mkdirs() }
+        private fun Settings.cacheDirForStaticCache(klibArtifact: KLIB): File {
+            val binaries = get<Binaries>()
+
+            val artifactBaseDir = if (klibArtifact.klibFile.startsWith(binaries.testBinariesDir)) {
+                // The KLIB artifact is located inside the build dir. This means it was built just a moment ago.
+                klibArtifact.klibFile.parentFile
+            } else {
+                // Special case for the given (external) KLIB artifacts.
+                binaries.givenBinariesDir
+            }
+
+            return artifactBaseDir.resolve(STATIC_CACHE_DIR_NAME).apply { mkdirs() }
+        }
 
         private fun Settings.singleModuleArtifactFile(module: TestModule.Exclusive, extension: String): File {
             val artifactFileName = buildString {

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt

@@ -72,6 +72,7 @@ internal class ExtTestCaseGroupProvider : TestCaseGroupProvider, TestDisposable(
                     customSourceTransformers = settings.get<ExternalSourceTransformersProvider>().getSourceTransformers(testDataFile),
                     testRoots = settings.get(),
                     generatedSources = settings.get(),
+                    customKlibs = settings.get(),
                     timeouts = settings.get()
                 )
 
@@ -95,6 +96,7 @@ private class ExtTestDataFile(
     customSourceTransformers: ExternalSourceTransformers?,
     testRoots: TestRoots,
     private val generatedSources: GeneratedSources,
+    private val customKlibs: CustomKlibs,
     private val timeouts: Timeouts
 ) {
     private val structure by lazy {
@@ -511,7 +513,10 @@ private class ExtTestDataFile(
             checks = TestRunChecks.Default(timeouts.executionTimeout),
             extras = WithTestRunnerExtras(runnerType = TestRunnerType.DEFAULT)
         )
-        testCase.initialize(sharedModules::get)
+        testCase.initialize(
+            givenModules = customKlibs.klibs.mapToSet(TestModule::Given),
+            findSharedModule = sharedModules::get
+        )
 
         return testCase
     }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCaseGroupProvider.kt

@@ -69,7 +69,7 @@ internal class PredefinedTestCaseGroupProvider(annotation: PredefinedTestCases)
                     ignoredTests = predefinedTestCase.ignoredTests.toSet()
                 )
             )
-            testCase.initialize(null)
+            testCase.initialize(null, null)
 
             TestCaseGroup.Default(disabledTestCaseIds = emptySet(), testCases = listOf(testCase))
         }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt

@@ -219,7 +219,10 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
             else
                 WithTestRunnerExtras(runnerType = parseTestRunner(registeredDirectives, location))
         )
-        testCase.initialize(findSharedModule = null)
+        testCase.initialize(
+            givenModules = settings.get<CustomKlibs>().klibs.mapToSet(TestModule::Given),
+            findSharedModule = null
+        )
 
         return testCase
     }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestClassSettings.kt

@@ -23,8 +23,9 @@ internal class GeneratedSources(val testSourcesDir: File, val sharedSourcesDir:
 /**
  * [testBinariesDir] - The directory with compiled test binaries (klibs and executable files).
  * [sharedBinariesDir] - The directory with compiled shared modules (klibs).
+ * [givenBinariesDir] - The directory with the given (external) modules (klibs).
  */
-internal class Binaries(val testBinariesDir: File, val sharedBinariesDir: File)
+internal class Binaries(val testBinariesDir: File, val sharedBinariesDir: File, val givenBinariesDir: File)
 
 /**
  * The [TestConfiguration] of the current test class and the [Annotation] with specific parameters.

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt

@@ -60,6 +60,12 @@ internal enum class TestMode(private val description: String) {
     override fun toString() = description
 }
 
+/**
+ * The set of custom (external) klibs that should be passed to the Kotlin/Native compiler.
+ */
+@JvmInline
+internal value class CustomKlibs(val klibs: Set<File>)
+
 /**
  * Whether to force [TestKind.STANDALONE] for all tests where [TestKind] is assumed to be [TestKind.REGULAR] otherwise:
  * - either explicitly specified in the test data file: // KIND: REGULAR

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/Names.kt

@@ -28,6 +28,9 @@ internal const val DEFAULT_MODULE_NAME = "default"
 internal const val SUPPORT_MODULE_NAME = "support"
 internal const val LAUNCHER_MODULE_NAME = "__launcher__" // Used only in KLIB tests.
 
+internal const val SHARED_MODULES_DIR_NAME = "__shared_modules__"
+internal const val GIVEN_MODULES_DIR_NAME = "__given_modules__"
+
 internal const val STATIC_CACHE_DIR_NAME = "__static_cache__"
 
 internal fun prettyHash(hash: Int): String = hash.toUInt().toString(16).padStart(8, '0')