From 5d234c0cdc21ab7cccfcb1ca658234db65ce07b5 Mon Sep 17 00:00:00 2001 From: Dmitriy Dolovov Date: Mon, 25 Jul 2022 16:26:42 +0200 Subject: [PATCH] [Native][tests] Ability to programmatically mute codegen/box & regular tests --- .../blackboxtest/support/NativeTestSupport.kt | 27 +++++++++++++++++ .../group/ConditionallyDisabledTests.kt | 19 ++++++++++++ .../support/group/ExtTestCaseGroupProvider.kt | 11 +++---- .../group/StandardTestCaseGroupProvider.kt | 21 +++++++++----- .../group/UseExtTestCaseGroupProvider.kt | 6 +++- .../group/UseStandardTestCaseGroupProvider.kt | 6 +++- .../support/settings/TestClassSettings.kt | 8 +++++ .../blackboxtest/support/util/Annotations.kt | 29 +++++++++++++++++++ 8 files changed, 113 insertions(+), 14 deletions(-) create mode 100644 native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ConditionallyDisabledTests.kt create mode 100644 native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/Annotations.kt diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt index 0a7f828b627..1bc605ce1b4 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt @@ -9,6 +9,8 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.NativeTestSupport.createS import org.jetbrains.kotlin.konan.blackboxtest.support.NativeTestSupport.createTestRunSettings import org.jetbrains.kotlin.konan.blackboxtest.support.NativeTestSupport.getOrCreateSimpleTestRunProvider import org.jetbrains.kotlin.konan.blackboxtest.support.NativeTestSupport.getOrCreateTestRunProvider +import org.jetbrains.kotlin.konan.blackboxtest.support.group.DisabledTests +import org.jetbrains.kotlin.konan.blackboxtest.support.group.DisabledTestsIfProperty import org.jetbrains.kotlin.konan.blackboxtest.support.group.TestCaseGroupProvider import org.jetbrains.kotlin.konan.blackboxtest.support.runner.SimpleTestRunProvider import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunProvider @@ -299,6 +301,7 @@ private object NativeTestSupport { this += when (clazz) { TestRoots::class -> computeTestRoots(enclosingTestClass) GeneratedSources::class -> computeGeneratedSourceDirs(testProcessSettings.get(), nativeTargets, enclosingTestClass) + DisabledTestDataFiles::class -> computeDisabledTestDataFiles(enclosingTestClass) else -> fail { "Unknown test class setting type: $clazz" } } } @@ -320,6 +323,30 @@ private object NativeTestSupport { ?: fail { "No @${TestConfiguration::class.simpleName} annotation found on test classes" } } + private fun computeDisabledTestDataFiles(enclosingTestClass: Class<*>): DisabledTestDataFiles { + val filesAndDirectories = buildSet { + fun contributeSourceLocations(sourceLocations: Array) { + sourceLocations.forEach { expandGlobTo(getAbsoluteFile(it), this) } + } + + fun recurse(clazz: Class<*>) { + clazz.allInheritedAnnotations.forEach { annotation -> + when (annotation) { + is DisabledTests -> contributeSourceLocations(annotation.sourceLocations) + is DisabledTestsIfProperty -> if (System.getProperty(annotation.property.propertyName) == annotation.propertyValue) { + contributeSourceLocations(annotation.sourceLocations) + } + } + } + clazz.declaredClasses.forEach(::recurse) + } + + recurse(enclosingTestClass) + } + + return DisabledTestDataFiles(filesAndDirectories) + } + private fun computeTestRoots(enclosingTestClass: Class<*>): TestRoots { fun TestMetadata.testRoot() = getAbsoluteFile(localPath = value) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ConditionallyDisabledTests.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ConditionallyDisabledTests.kt new file mode 100644 index 00000000000..523e93f3982 --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ConditionallyDisabledTests.kt @@ -0,0 +1,19 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ +package org.jetbrains.kotlin.konan.blackboxtest.support.group + +import org.jetbrains.kotlin.konan.blackboxtest.support.ClassLevelProperty + +@Target(AnnotationTarget.CLASS) +internal annotation class DisabledTests( + val sourceLocations: Array +) + +@Target(AnnotationTarget.CLASS) +internal annotation class DisabledTestsIfProperty( + val sourceLocations: Array, + val property: ClassLevelProperty, + val propertyValue: String +) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt index 5e7d740d8b1..442a2a91e20 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt @@ -49,9 +49,13 @@ internal class ExtTestCaseGroupProvider : TestCaseGroupProvider, TestDisposable( check(testCaseGroupId is TestCaseGroupId.TestDataDir) return cachedTestCaseGroups.computeIfAbsent(testCaseGroupId) { - if (testCaseGroupId.dir in excludes) return@computeIfAbsent TestCaseGroup.ALL_DISABLED + val testDataDir = testCaseGroupId.dir - val (excludedTestDataFiles, testDataFiles) = testCaseGroupId.dir.listFiles() + val excludes: Set = settings.get().filesAndDirectories + if (testDataDir in excludes) + return@computeIfAbsent TestCaseGroup.ALL_DISABLED + + val (excludedTestDataFiles, testDataFiles) = testDataDir.listFiles() ?.filter { file -> file.isFile && file.extension == "kt" } ?.partition { file -> file in excludes } ?: return@computeIfAbsent null @@ -85,9 +89,6 @@ internal class ExtTestCaseGroupProvider : TestCaseGroupProvider, TestDisposable( } companion object { - /** Test data files or test data directories that are excluded from testing. */ - private val excludes: Set = listOf().mapToSet(::getAbsoluteFile) - /** Tests that should be compiled and executed as standalone tests. */ private val standalones: Set = listOf( // Comparison of type information obtained with reflection against non-patched string literal: diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt index ba08f98b0c2..dd42d24a987 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt @@ -31,17 +31,24 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { check(testCaseGroupId is TestCaseGroupId.TestDataDir) return cachedTestCaseGroups.computeIfAbsent(testCaseGroupId) { - val testDataFiles = testCaseGroupId.dir.listFiles() + val testDataDir = testCaseGroupId.dir + val testDataFiles = testDataDir.listFiles() ?: return@computeIfAbsent null // `null` means that there is no such testDataDir. - val testCases = testDataFiles.mapNotNull { testDataFile -> - if (!testDataFile.isFile || testDataFile.extension != "kt") - return@mapNotNull null + val excludes: Set = settings.get().filesAndDirectories + if (testDataDir in excludes) + return@computeIfAbsent TestCaseGroup.ALL_DISABLED - createTestCase(testDataFile, settings) - } + val (excludedTestDataFiles, includedTestDataFiles) = testDataFiles + .filter { file -> file.isFile && file.extension == "kt" } + .partition { file -> file in excludes } - TestCaseGroup.Default(disabledTestCaseIds = emptySet(), testCases = testCases) + val disabledTestCaseIds = hashSetOf() + excludedTestDataFiles.mapTo(disabledTestCaseIds, TestCaseId::TestDataFile) + + val testCases = includedTestDataFiles.map { testDataFile -> createTestCase(testDataFile, settings) } + + TestCaseGroup.Default(disabledTestCaseIds, testCases) } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/UseExtTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/UseExtTestCaseGroupProvider.kt index a23daa08f5c..20271de513d 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/UseExtTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/UseExtTestCaseGroupProvider.kt @@ -5,10 +5,14 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.group +import org.jetbrains.kotlin.konan.blackboxtest.support.settings.DisabledTestDataFiles import org.jetbrains.kotlin.konan.blackboxtest.support.settings.GeneratedSources import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestRoots import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestConfiguration @Target(AnnotationTarget.CLASS) -@TestConfiguration(providerClass = ExtTestCaseGroupProvider::class, requiredSettings = [TestRoots::class, GeneratedSources::class]) +@TestConfiguration( + providerClass = ExtTestCaseGroupProvider::class, + requiredSettings = [TestRoots::class, GeneratedSources::class, DisabledTestDataFiles::class] +) annotation class UseExtTestCaseGroupProvider diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/UseStandardTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/UseStandardTestCaseGroupProvider.kt index ccd1fa5632a..8615fc34bb5 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/UseStandardTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/UseStandardTestCaseGroupProvider.kt @@ -5,10 +5,14 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.group +import org.jetbrains.kotlin.konan.blackboxtest.support.settings.DisabledTestDataFiles import org.jetbrains.kotlin.konan.blackboxtest.support.settings.GeneratedSources import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestRoots import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestConfiguration @Target(AnnotationTarget.CLASS) -@TestConfiguration(providerClass = StandardTestCaseGroupProvider::class, requiredSettings = [TestRoots::class, GeneratedSources::class]) +@TestConfiguration( + providerClass = StandardTestCaseGroupProvider::class, + requiredSettings = [TestRoots::class, GeneratedSources::class, DisabledTestDataFiles::class] +) annotation class UseStandardTestCaseGroupProvider diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestClassSettings.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestClassSettings.kt index 68819279d6c..b7340631126 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestClassSettings.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestClassSettings.kt @@ -5,6 +5,8 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.settings +import org.jetbrains.kotlin.konan.blackboxtest.AbstractNativeBlackBoxTest +import org.jetbrains.kotlin.konan.blackboxtest.support.TestCaseId import java.io.File /** @@ -28,3 +30,9 @@ internal class Binaries(val testBinariesDir: File, val sharedBinariesDir: File) * The [TestConfiguration] of the current test class and the [Annotation] with specific parameters. */ internal data class ComputedTestConfiguration(val configuration: TestConfiguration, val annotation: Annotation) + +/** + * The tests (test data files and directories with test data files) that are disabled. + * This is applicable only to inheritors of [AbstractNativeBlackBoxTest] that use [TestCaseId.TestDataFile]. + */ +internal class DisabledTestDataFiles(val filesAndDirectories: Set) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/Annotations.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/Annotations.kt new file mode 100644 index 00000000000..b0f220ac679 --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/Annotations.kt @@ -0,0 +1,29 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.blackboxtest.support.util + +/** + * Since Kotlin annotations are not inherited (see KT-22265), we need a utility to collect all of them. + */ +internal val Class<*>.allInheritedAnnotations: List + get() { + val annotations = mutableListOf() + + val processedClasses = hashSetOf>() + fun process(clazz: Class<*>) { + if (processedClasses.add(clazz)) { + clazz.declaredAnnotations.mapNotNullTo(annotations) { annotation -> + annotation.takeIf { it !is Metadata } + } + clazz.interfaces.forEach(::process) + clazz.superclass.takeIf { it != Object::class.java }?.let(::process) + } + } + + process(this) + + return annotations + }