From 4a4760dc4b7beda0a09704046c8ea99c84f2690b Mon Sep 17 00:00:00 2001 From: Dmitriy Dolovov Date: Wed, 24 Nov 2021 18:01:51 +0300 Subject: [PATCH] [Native][tests] Run stdlib tests using new test infrastructure --- .../blackboxtest/NativeStdlibBlackBoxTest.kt | 45 ++++++++ .../support/NativeBlackBoxTestSupport.kt | 14 +-- .../konan/blackboxtest/support/TestCase.kt | 6 + .../blackboxtest/support/TestDirectives.kt | 7 +- .../group/PredefinedTestCaseGroupProvider.kt | 106 ++++++++++++++++++ .../support/group/PredefinedTestCases.kt | 15 +++ .../konan/blackboxtest/support/util/GTest.kt | 7 +- 7 files changed, 186 insertions(+), 14 deletions(-) create mode 100644 native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/NativeStdlibBlackBoxTest.kt create mode 100644 native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCaseGroupProvider.kt create mode 100644 native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCases.kt diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/NativeStdlibBlackBoxTest.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/NativeStdlibBlackBoxTest.kt new file mode 100644 index 00000000000..1d166367a9a --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/NativeStdlibBlackBoxTest.kt @@ -0,0 +1,45 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +@file:Suppress("unused") + +package org.jetbrains.kotlin.konan.blackboxtest + +import org.jetbrains.kotlin.konan.blackboxtest.support.TestCaseId +import org.jetbrains.kotlin.konan.blackboxtest.support.group.PredefinedTestCase as TC +import org.jetbrains.kotlin.konan.blackboxtest.support.group.PredefinedTestCases +import org.junit.jupiter.api.TestFactory + +@PredefinedTestCases( + TC( + name = "nativeStdlib", + freeCompilerArgs = [ + "-Xmulti-platform", + "-friend-modules=$KOTLIN_NATIVE_STDLIB_PATH", + "-opt-in=kotlin.RequiresOptIn,kotlin.ExperimentalStdlibApi" + ], + sourceLocations = [ + "libraries/stdlib/test/**.kt", + "libraries/stdlib/common/test/**.kt", + "kotlin-native/backend.native/tests/stdlib_external/text/**.kt", + "kotlin-native/backend.native/tests/stdlib_external/utils.kt", + "kotlin-native/backend.native/tests/stdlib_external/jsCollectionFactoriesActuals.kt" + ] + ), + TC( + name = "kotlinTest", + freeCompilerArgs = ["-friend-modules=$KOTLIN_NATIVE_STDLIB_PATH"], + sourceLocations = ["libraries/kotlin.test/common/src/test/kotlin/**.kt"] + ) +) +class NativeStdlibBlackBoxTest : AbstractNativeBlackBoxTest() { + @TestFactory + fun kotlinTest() = dynamicTestCase(TestCaseId.Named("kotlinTest")) + + @TestFactory + fun nativeStdlib() = dynamicTestCase(TestCaseId.Named("nativeStdlib")) +} + +private const val KOTLIN_NATIVE_STDLIB_PATH = "kotlin-native/dist/klib/common/stdlib" diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeBlackBoxTestSupport.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeBlackBoxTestSupport.kt index ce9bdab79ad..d94d8bffde4 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeBlackBoxTestSupport.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeBlackBoxTestSupport.kt @@ -154,23 +154,17 @@ class NativeBlackBoxTestSupport : BeforeEachCallback { testSettings: TestSettings, testSettingsAnnotation: Annotation? ): TestCaseGroupProvider { - // First, try to find two-argument constructor. + // Try to find a constructor that accepts the annotation. if (testSettingsAnnotation != null) { val testSettingsAnnotationClass = testSettingsAnnotation.annotationClass - testSettings.providerClass.constructors.asSequence() .forEach { c -> - val (p1, p2) = c.parameters.takeIf { it.size == 2 } ?: return@forEach - @Suppress("Reformat") val provider = when { - p1.hasTypeOf() && p2.hasTypeOf(testSettingsAnnotationClass) -> c.call(settings, testSettingsAnnotation) - p1.hasTypeOf(testSettingsAnnotationClass) && p2.hasTypeOf() -> c.call(testSettingsAnnotation, settings) - else -> return@forEach - } - return provider.cast() + val p = c.parameters.singleOrNull() ?: return@forEach + if (p.hasTypeOf(testSettingsAnnotationClass)) return c.call(testSettingsAnnotation).cast() } } - // Next, try to find a single-argument constructor. + // ... or settings at least. testSettings.providerClass.constructors.asSequence() .forEach { c -> val p = c.parameters.singleOrNull() ?: return@forEach diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt index e8cfd4efbcb..49ef11ae726 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt @@ -157,6 +157,11 @@ internal interface TestCaseId { override val testCaseGroupId = TestCaseGroupId.TestDataDir(file.parentFile) // The directory, containing testData file. override fun toString(): String = file.path } + + data class Named(val uniqueName: String) : TestCaseId { + override val testCaseGroupId = TestCaseGroupId.Named(uniqueName) // The single test case inside the test group. + override fun toString() = "[$uniqueName]" + } } /** @@ -247,6 +252,7 @@ internal class TestCase( */ internal interface TestCaseGroupId { data class TestDataDir(val dir: File) : TestCaseGroupId + data class Named(val uniqueName: String) : TestCaseGroupId } /** diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt index c25c2de5135..667af0ef594 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt @@ -5,7 +5,6 @@ package org.jetbrains.kotlin.konan.blackboxtest.support -import org.jetbrains.kotlin.konan.blackboxtest.support.TestCompilerArgs.Companion.EXPLICITLY_FORBIDDEN_COMPILER_ARGS import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.ENTRY_POINT import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.FREE_COMPILER_ARGS import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.INPUT_DATA_FILE @@ -90,8 +89,10 @@ internal class TestCompilerArgs(val compilerArgs: List) { companion object { val EMPTY = TestCompilerArgs(emptyList()) + fun findForbiddenArgs(compilerArgs: Iterable): Set = compilerArgs intersect EXPLICITLY_FORBIDDEN_COMPILER_ARGS + /** The set of compiler args that are not permitted to be explicitly specified using [FREE_COMPILER_ARGS]. */ - internal val EXPLICITLY_FORBIDDEN_COMPILER_ARGS = setOf( + private val EXPLICITLY_FORBIDDEN_COMPILER_ARGS = setOf( "-trn", "-generate-no-exit-test-runner", "-tr", "-generate-test-runner", "-trw", "-generate-worker-test-runner", @@ -170,7 +171,7 @@ internal fun parseFreeCompilerArgs(registeredDirectives: RegisteredDirectives, l return TestCompilerArgs.EMPTY val freeCompilerArgs = registeredDirectives[FREE_COMPILER_ARGS] - val forbiddenCompilerArgs = freeCompilerArgs intersect EXPLICITLY_FORBIDDEN_COMPILER_ARGS + val forbiddenCompilerArgs = TestCompilerArgs.findForbiddenArgs(freeCompilerArgs) assertTrue(forbiddenCompilerArgs.isEmpty()) { """ $location: Forbidden compiler arguments found in $FREE_COMPILER_ARGS directive: $forbiddenCompilerArgs diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCaseGroupProvider.kt new file mode 100644 index 00000000000..b1101622dc3 --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCaseGroupProvider.kt @@ -0,0 +1,106 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.blackboxtest.support.group + +import org.jetbrains.kotlin.konan.blackboxtest.support.* +import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras +import org.jetbrains.kotlin.konan.blackboxtest.support.TestCaseGroup +import org.jetbrains.kotlin.konan.blackboxtest.support.TestCaseGroupId +import org.jetbrains.kotlin.konan.blackboxtest.support.TestFile +import org.jetbrains.kotlin.konan.blackboxtest.support.TestModule +import org.jetbrains.kotlin.konan.blackboxtest.support.util.ThreadSafeFactory +import org.jetbrains.kotlin.konan.blackboxtest.support.util.expandGlobTo +import org.jetbrains.kotlin.konan.blackboxtest.support.util.getAbsoluteFile +import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue +import org.jetbrains.kotlin.utils.addToStdlib.cast +import org.junit.jupiter.api.fail +import java.io.File + +internal class PredefinedTestCaseGroupProvider(annotation: PredefinedTestCases) : TestCaseGroupProvider { + private val testCaseIdToPredefinedTestCase: Map = buildMap { + annotation.testCases.forEach { predefinedTestCase -> + val testCaseId = TestCaseId.Named(predefinedTestCase.name) + if (put(testCaseId, predefinedTestCase) != null) + fail { "Duplicated test cases found: $testCaseId" } + + } + } + + // Assumption: Every test case group contains exactly one test case. + private val testCaseGroupIdToTestCaseId: Map = buildMap { + testCaseIdToPredefinedTestCase.keys.forEach { testCaseId -> + val testCaseGroupId = testCaseId.testCaseGroupId + if (put(testCaseGroupId, testCaseId) != null) + fail { "Duplicated test case groups found: $testCaseGroupId" } + } + } + + private val lazyTestCaseGroups = ThreadSafeFactory { testCaseGroupId -> + val testCaseId = testCaseGroupIdToTestCaseId[testCaseGroupId] ?: return@ThreadSafeFactory null + val predefinedTestCase = testCaseIdToPredefinedTestCase[testCaseId] ?: return@ThreadSafeFactory null + + val module = TestModule.Exclusive( + name = testCaseId.uniqueName, + directDependencySymbols = emptySet(), + directFriendSymbols = emptySet() + ) + + predefinedTestCase.sourceLocations + .expandGlobs { "No files found for test case $testCaseId" } + .forEach { file -> module.files += TestFile.createCommitted(file, module) } + + val testCase = TestCase( + id = testCaseId, + kind = TestKind.STANDALONE, + modules = setOf(module), + freeCompilerArgs = predefinedTestCase.freeCompilerArgs + .parseCompilerArgs { "Failed to parse free compiler arguments for test case $testCaseId" }, + nominalPackageName = testCaseId.uniqueName, + expectedOutputDataFile = null, + extras = WithTestRunnerExtras + ) + testCase.initialize(null) + + TestCaseGroup.Default(disabledTestCaseIds = emptySet(), testCases = listOf(testCase)) + } + + override fun setPreprocessors(testDataDir: File, preprocessors: List<(String) -> String>) { + // do nothing + } + + override fun getTestCaseGroup(testCaseGroupId: TestCaseGroupId): TestCaseGroup? { + assertTrue(testCaseGroupId is TestCaseGroupId.Named) + return lazyTestCaseGroups[testCaseGroupId.cast()] + } + + companion object { + private fun Array.expandGlobs(noExpandedFilesErrorMessage: () -> String): Set { + val files = buildSet { + this@expandGlobs.forEach { pathPattern -> expandGlobTo(getAbsoluteFile(pathPattern), this) } + } + assertTrue(files.isNotEmpty(), noExpandedFilesErrorMessage) + return files + } + + private fun Array.parseCompilerArgs(parsingErrorMessage: () -> String): TestCompilerArgs = + if (isEmpty()) + TestCompilerArgs.EMPTY + else { + val freeCompilerArgs = asList() + val forbiddenCompilerArgs = TestCompilerArgs.findForbiddenArgs(freeCompilerArgs) + assertTrue(forbiddenCompilerArgs.isEmpty()) { + """ + ${parsingErrorMessage()} + + Forbidden compiler arguments found: $forbiddenCompilerArgs + All arguments: $this + """.trimIndent() + } + + TestCompilerArgs(freeCompilerArgs) + } + } +} diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCases.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCases.kt new file mode 100644 index 00000000000..a753addf4de --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCases.kt @@ -0,0 +1,15 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.blackboxtest.support.group + +import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestSettings + +@Target(AnnotationTarget.CLASS) +@TestSettings(providerClass = PredefinedTestCaseGroupProvider::class) +annotation class PredefinedTestCases(vararg val testCases: PredefinedTestCase) + +@Target() +annotation class PredefinedTestCase(val name: String, val freeCompilerArgs: Array, val sourceLocations: Array) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/GTest.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/GTest.kt index 2f2c54665f2..f394d0efc9c 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/GTest.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/GTest.kt @@ -48,8 +48,9 @@ internal fun parseGTestReport(stdOut: String): GTestReport { val cleanStdOut = StringBuilder() var expectStatusLine = false - stdOut.lineSequence().forEach { line -> + stdOut.lineSequence().forEachIndexed { index, line -> when { + index == 0 && line.startsWith(STDLIB_TESTS_IGNORED_LINE_PREFIX) -> Unit expectStatusLine -> { val matcher = STATUS_LINE_REGEX.matchEntire(line) if (matcher != null) { @@ -97,6 +98,7 @@ internal fun parseGTestListing(rawGTestListing: String): Collection state line.isBlank() -> when (state) { is NewTest, is End -> End else -> parseError("Unexpected empty line") @@ -128,6 +130,9 @@ internal fun parseGTestListing(rawGTestListing: String): Collection