[Native][tests] Run stdlib tests using new test infrastructure

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/NativeStdlibBlackBoxTest.kt

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+@file:Suppress("unused")
+
+package org.jetbrains.kotlin.konan.blackboxtest
+
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestCaseId
+import org.jetbrains.kotlin.konan.blackboxtest.support.group.PredefinedTestCase as TC
+import org.jetbrains.kotlin.konan.blackboxtest.support.group.PredefinedTestCases
+import org.junit.jupiter.api.TestFactory
+
+@PredefinedTestCases(
+    TC(
+        name = "nativeStdlib",
+        freeCompilerArgs = [
+            "-Xmulti-platform",
+            "-friend-modules=$KOTLIN_NATIVE_STDLIB_PATH",
+            "-opt-in=kotlin.RequiresOptIn,kotlin.ExperimentalStdlibApi"
+        ],
+        sourceLocations = [
+            "libraries/stdlib/test/**.kt",
+            "libraries/stdlib/common/test/**.kt",
+            "kotlin-native/backend.native/tests/stdlib_external/text/**.kt",
+            "kotlin-native/backend.native/tests/stdlib_external/utils.kt",
+            "kotlin-native/backend.native/tests/stdlib_external/jsCollectionFactoriesActuals.kt"
+        ]
+    ),
+    TC(
+        name = "kotlinTest",
+        freeCompilerArgs = ["-friend-modules=$KOTLIN_NATIVE_STDLIB_PATH"],
+        sourceLocations = ["libraries/kotlin.test/common/src/test/kotlin/**.kt"]
+    )
+)
+class NativeStdlibBlackBoxTest : AbstractNativeBlackBoxTest() {
+    @TestFactory
+    fun kotlinTest() = dynamicTestCase(TestCaseId.Named("kotlinTest"))
+
+    @TestFactory
+    fun nativeStdlib() = dynamicTestCase(TestCaseId.Named("nativeStdlib"))
+}
+
+private const val KOTLIN_NATIVE_STDLIB_PATH = "kotlin-native/dist/klib/common/stdlib"

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeBlackBoxTestSupport.kt

@@ -154,23 +154,17 @@ class NativeBlackBoxTestSupport : BeforeEachCallback {
             testSettings: TestSettings,
             testSettingsAnnotation: Annotation?
         ): TestCaseGroupProvider {
-            // First, try to find two-argument constructor.
+            // Try to find a constructor that accepts the annotation.
             if (testSettingsAnnotation != null) {
                 val testSettingsAnnotationClass = testSettingsAnnotation.annotationClass
-
                 testSettings.providerClass.constructors.asSequence()
                     .forEach { c ->
-                        val (p1, p2) = c.parameters.takeIf { it.size == 2 } ?: return@forEach
+                        val p = c.parameters.singleOrNull() ?: return@forEach
-                        @Suppress("Reformat") val provider = when {
+                        if (p.hasTypeOf(testSettingsAnnotationClass)) return c.call(testSettingsAnnotation).cast()
-                            p1.hasTypeOf<Settings>() && p2.hasTypeOf(testSettingsAnnotationClass) -> c.call(settings, testSettingsAnnotation)
-                            p1.hasTypeOf(testSettingsAnnotationClass) && p2.hasTypeOf<Settings>() -> c.call(testSettingsAnnotation, settings)
-                            else -> return@forEach
-                        }
-                        return provider.cast()
                     }
             }
 
-            // Next, try to find a single-argument constructor.
+            // ... or settings at least.
             testSettings.providerClass.constructors.asSequence()
                 .forEach { c ->
                     val p = c.parameters.singleOrNull() ?: return@forEach

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt

@@ -157,6 +157,11 @@ internal interface TestCaseId {
         override val testCaseGroupId = TestCaseGroupId.TestDataDir(file.parentFile) // The directory, containing testData file.
         override fun toString(): String = file.path
     }
+
+    data class Named(val uniqueName: String) : TestCaseId {
+        override val testCaseGroupId = TestCaseGroupId.Named(uniqueName) // The single test case inside the test group.
+        override fun toString() = "[$uniqueName]"
+    }
 }
 
 /**
@@ -247,6 +252,7 @@ internal class TestCase(
  */
 internal interface TestCaseGroupId {
     data class TestDataDir(val dir: File) : TestCaseGroupId
+    data class Named(val uniqueName: String) : TestCaseGroupId
 }
 
 /**

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt

@@ -5,7 +5,6 @@
 
 package org.jetbrains.kotlin.konan.blackboxtest.support
 
-import org.jetbrains.kotlin.konan.blackboxtest.support.TestCompilerArgs.Companion.EXPLICITLY_FORBIDDEN_COMPILER_ARGS
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.ENTRY_POINT
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.FREE_COMPILER_ARGS
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.INPUT_DATA_FILE
@@ -90,8 +89,10 @@ internal class TestCompilerArgs(val compilerArgs: List<String>) {
     companion object {
         val EMPTY = TestCompilerArgs(emptyList())
 
+        fun findForbiddenArgs(compilerArgs: Iterable<String>): Set<String> = compilerArgs intersect EXPLICITLY_FORBIDDEN_COMPILER_ARGS
+
         /** The set of compiler args that are not permitted to be explicitly specified using [FREE_COMPILER_ARGS]. */
-        internal val EXPLICITLY_FORBIDDEN_COMPILER_ARGS = setOf(
+        private val EXPLICITLY_FORBIDDEN_COMPILER_ARGS = setOf(
             "-trn", "-generate-no-exit-test-runner",
             "-tr", "-generate-test-runner",
             "-trw", "-generate-worker-test-runner",
@@ -170,7 +171,7 @@ internal fun parseFreeCompilerArgs(registeredDirectives: RegisteredDirectives, l
         return TestCompilerArgs.EMPTY
 
     val freeCompilerArgs = registeredDirectives[FREE_COMPILER_ARGS]
-    val forbiddenCompilerArgs = freeCompilerArgs intersect EXPLICITLY_FORBIDDEN_COMPILER_ARGS
+    val forbiddenCompilerArgs = TestCompilerArgs.findForbiddenArgs(freeCompilerArgs)
     assertTrue(forbiddenCompilerArgs.isEmpty()) {
         """
             $location: Forbidden compiler arguments found in $FREE_COMPILER_ARGS directive: $forbiddenCompilerArgs

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCaseGroupProvider.kt

@@ -0,0 +1,106 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.konan.blackboxtest.support.group
+
+import org.jetbrains.kotlin.konan.blackboxtest.support.*
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestCaseGroup
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestCaseGroupId
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestFile
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestModule
+import org.jetbrains.kotlin.konan.blackboxtest.support.util.ThreadSafeFactory
+import org.jetbrains.kotlin.konan.blackboxtest.support.util.expandGlobTo
+import org.jetbrains.kotlin.konan.blackboxtest.support.util.getAbsoluteFile
+import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
+import org.jetbrains.kotlin.utils.addToStdlib.cast
+import org.junit.jupiter.api.fail
+import java.io.File
+
+internal class PredefinedTestCaseGroupProvider(annotation: PredefinedTestCases) : TestCaseGroupProvider {
+    private val testCaseIdToPredefinedTestCase: Map<TestCaseId.Named, PredefinedTestCase> = buildMap {
+        annotation.testCases.forEach { predefinedTestCase ->
+            val testCaseId = TestCaseId.Named(predefinedTestCase.name)
+            if (put(testCaseId, predefinedTestCase) != null)
+                fail { "Duplicated test cases found: $testCaseId" }
+
+        }
+    }
+
+    // Assumption: Every test case group contains exactly one test case.
+    private val testCaseGroupIdToTestCaseId: Map<TestCaseGroupId.Named, TestCaseId.Named> = buildMap {
+        testCaseIdToPredefinedTestCase.keys.forEach { testCaseId ->
+            val testCaseGroupId = testCaseId.testCaseGroupId
+            if (put(testCaseGroupId, testCaseId) != null)
+                fail { "Duplicated test case groups found: $testCaseGroupId" }
+        }
+    }
+
+    private val lazyTestCaseGroups = ThreadSafeFactory<TestCaseGroupId.Named, TestCaseGroup?> { testCaseGroupId ->
+        val testCaseId = testCaseGroupIdToTestCaseId[testCaseGroupId] ?: return@ThreadSafeFactory null
+        val predefinedTestCase = testCaseIdToPredefinedTestCase[testCaseId] ?: return@ThreadSafeFactory null
+
+        val module = TestModule.Exclusive(
+            name = testCaseId.uniqueName,
+            directDependencySymbols = emptySet(),
+            directFriendSymbols = emptySet()
+        )
+
+        predefinedTestCase.sourceLocations
+            .expandGlobs { "No files found for test case $testCaseId" }
+            .forEach { file -> module.files += TestFile.createCommitted(file, module) }
+
+        val testCase = TestCase(
+            id = testCaseId,
+            kind = TestKind.STANDALONE,
+            modules = setOf(module),
+            freeCompilerArgs = predefinedTestCase.freeCompilerArgs
+                .parseCompilerArgs { "Failed to parse free compiler arguments for test case $testCaseId" },
+            nominalPackageName = testCaseId.uniqueName,
+            expectedOutputDataFile = null,
+            extras = WithTestRunnerExtras
+        )
+        testCase.initialize(null)
+
+        TestCaseGroup.Default(disabledTestCaseIds = emptySet(), testCases = listOf(testCase))
+    }
+
+    override fun setPreprocessors(testDataDir: File, preprocessors: List<(String) -> String>) {
+        // do nothing
+    }
+
+    override fun getTestCaseGroup(testCaseGroupId: TestCaseGroupId): TestCaseGroup? {
+        assertTrue(testCaseGroupId is TestCaseGroupId.Named)
+        return lazyTestCaseGroups[testCaseGroupId.cast()]
+    }
+
+    companion object {
+        private fun Array<String>.expandGlobs(noExpandedFilesErrorMessage: () -> String): Set<File> {
+            val files = buildSet {
+                this@expandGlobs.forEach { pathPattern -> expandGlobTo(getAbsoluteFile(pathPattern), this) }
+            }
+            assertTrue(files.isNotEmpty(), noExpandedFilesErrorMessage)
+            return files
+        }
+
+        private fun Array<String>.parseCompilerArgs(parsingErrorMessage: () -> String): TestCompilerArgs =
+            if (isEmpty())
+                TestCompilerArgs.EMPTY
+            else {
+                val freeCompilerArgs = asList()
+                val forbiddenCompilerArgs = TestCompilerArgs.findForbiddenArgs(freeCompilerArgs)
+                assertTrue(forbiddenCompilerArgs.isEmpty()) {
+                    """
+                        ${parsingErrorMessage()}
+                        
+                        Forbidden compiler arguments found: $forbiddenCompilerArgs
+                        All arguments: $this
+                    """.trimIndent()
+                }
+
+                TestCompilerArgs(freeCompilerArgs)
+            }
+    }
+}

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCases.kt

@@ -0,0 +1,15 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.konan.blackboxtest.support.group
+
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestSettings
+
+@Target(AnnotationTarget.CLASS)
+@TestSettings(providerClass = PredefinedTestCaseGroupProvider::class)
+annotation class PredefinedTestCases(vararg val testCases: PredefinedTestCase)
+
+@Target()
+annotation class PredefinedTestCase(val name: String, val freeCompilerArgs: Array<String>, val sourceLocations: Array<String>)

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/GTest.kt

@@ -48,8 +48,9 @@ internal fun parseGTestReport(stdOut: String): GTestReport {
     val cleanStdOut = StringBuilder()
 
     var expectStatusLine = false
-    stdOut.lineSequence().forEach { line ->
+    stdOut.lineSequence().forEachIndexed { index, line ->
         when {
+            index == 0 && line.startsWith(STDLIB_TESTS_IGNORED_LINE_PREFIX) -> Unit
             expectStatusLine -> {
                 val matcher = STATUS_LINE_REGEX.matchEntire(line)
                 if (matcher != null) {
@@ -97,6 +98,7 @@ internal fun parseGTestListing(rawGTestListing: String): Collection<TestFunction
         }
 
         state = when {
+            index == 0 && line.startsWith(STDLIB_TESTS_IGNORED_LINE_PREFIX) -> state
             line.isBlank() -> when (state) {
                 is NewTest, is End -> End
                 else -> parseError("Unexpected empty line")
@@ -128,6 +130,9 @@ internal fun parseGTestListing(rawGTestListing: String): Collection<TestFunction
 private val RUN_LINE_REGEX = Regex("""^\[\s+RUN\s+]\s+.*""")
 private val STATUS_LINE_REGEX = Regex("""^\[\s+([A-Z]+)\s+]\s+(\S+)\s+.*""")
 
+// The very first line of stdlib test output may contain seed of Random. Such line should be ignored.
+private const val STDLIB_TESTS_IGNORED_LINE_PREFIX = "Seed: "
+
 private sealed interface GTestListingParseState {
     object Begin : GTestListingParseState
     object End : GTestListingParseState