[Native][tests] Ability to programmatically mute codegen/box & regular tests

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt

@@ -9,6 +9,8 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.NativeTestSupport.createS
 import org.jetbrains.kotlin.konan.blackboxtest.support.NativeTestSupport.createTestRunSettings
 import org.jetbrains.kotlin.konan.blackboxtest.support.NativeTestSupport.getOrCreateSimpleTestRunProvider
 import org.jetbrains.kotlin.konan.blackboxtest.support.NativeTestSupport.getOrCreateTestRunProvider
+import org.jetbrains.kotlin.konan.blackboxtest.support.group.DisabledTests
+import org.jetbrains.kotlin.konan.blackboxtest.support.group.DisabledTestsIfProperty
 import org.jetbrains.kotlin.konan.blackboxtest.support.group.TestCaseGroupProvider
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.SimpleTestRunProvider
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunProvider
@@ -299,6 +301,7 @@ private object NativeTestSupport {
                     this += when (clazz) {
                         TestRoots::class -> computeTestRoots(enclosingTestClass)
                         GeneratedSources::class -> computeGeneratedSourceDirs(testProcessSettings.get(), nativeTargets, enclosingTestClass)
+                        DisabledTestDataFiles::class -> computeDisabledTestDataFiles(enclosingTestClass)
                         else -> fail { "Unknown test class setting type: $clazz" }
                     }
                 }
@@ -320,6 +323,30 @@ private object NativeTestSupport {
             ?: fail { "No @${TestConfiguration::class.simpleName} annotation found on test classes" }
     }
 
+    private fun computeDisabledTestDataFiles(enclosingTestClass: Class<*>): DisabledTestDataFiles {
+        val filesAndDirectories = buildSet {
+            fun contributeSourceLocations(sourceLocations: Array<String>) {
+                sourceLocations.forEach { expandGlobTo(getAbsoluteFile(it), this) }
+            }
+
+            fun recurse(clazz: Class<*>) {
+                clazz.allInheritedAnnotations.forEach { annotation ->
+                    when (annotation) {
+                        is DisabledTests -> contributeSourceLocations(annotation.sourceLocations)
+                        is DisabledTestsIfProperty -> if (System.getProperty(annotation.property.propertyName) == annotation.propertyValue) {
+                            contributeSourceLocations(annotation.sourceLocations)
+                        }
+                    }
+                }
+                clazz.declaredClasses.forEach(::recurse)
+            }
+
+            recurse(enclosingTestClass)
+        }
+
+        return DisabledTestDataFiles(filesAndDirectories)
+    }
+
     private fun computeTestRoots(enclosingTestClass: Class<*>): TestRoots {
         fun TestMetadata.testRoot() = getAbsoluteFile(localPath = value)
 

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ConditionallyDisabledTests.kt

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+package org.jetbrains.kotlin.konan.blackboxtest.support.group
+
+import org.jetbrains.kotlin.konan.blackboxtest.support.ClassLevelProperty
+
+@Target(AnnotationTarget.CLASS)
+internal annotation class DisabledTests(
+    val sourceLocations: Array<String>
+)
+
+@Target(AnnotationTarget.CLASS)
+internal annotation class DisabledTestsIfProperty(
+    val sourceLocations: Array<String>,
+    val property: ClassLevelProperty,
+    val propertyValue: String
+)

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt

@@ -49,9 +49,13 @@ internal class ExtTestCaseGroupProvider : TestCaseGroupProvider, TestDisposable(
         check(testCaseGroupId is TestCaseGroupId.TestDataDir)
 
         return cachedTestCaseGroups.computeIfAbsent(testCaseGroupId) {
-            if (testCaseGroupId.dir in excludes) return@computeIfAbsent TestCaseGroup.ALL_DISABLED
+            val testDataDir = testCaseGroupId.dir
 
-            val (excludedTestDataFiles, testDataFiles) = testCaseGroupId.dir.listFiles()
+            val excludes: Set<File> = settings.get<DisabledTestDataFiles>().filesAndDirectories
+            if (testDataDir in excludes)
+                return@computeIfAbsent TestCaseGroup.ALL_DISABLED
+
+            val (excludedTestDataFiles, testDataFiles) = testDataDir.listFiles()
                 ?.filter { file -> file.isFile && file.extension == "kt" }
                 ?.partition { file -> file in excludes }
                 ?: return@computeIfAbsent null
@@ -85,9 +89,6 @@ internal class ExtTestCaseGroupProvider : TestCaseGroupProvider, TestDisposable(
     }
 
     companion object {
-        /** Test data files or test data directories that are excluded from testing. */
-        private val excludes: Set<File> = listOf<String>().mapToSet(::getAbsoluteFile)
-
         /** Tests that should be compiled and executed as standalone tests. */
         private val standalones: Set<File> = listOf(
             // Comparison of type information obtained with reflection against non-patched string literal:

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt

@@ -31,17 +31,24 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
         check(testCaseGroupId is TestCaseGroupId.TestDataDir)
 
         return cachedTestCaseGroups.computeIfAbsent(testCaseGroupId) {
-            val testDataFiles = testCaseGroupId.dir.listFiles()
+            val testDataDir = testCaseGroupId.dir
+            val testDataFiles = testDataDir.listFiles()
                 ?: return@computeIfAbsent null // `null` means that there is no such testDataDir.
 
-            val testCases = testDataFiles.mapNotNull { testDataFile ->
+            val excludes: Set<File> = settings.get<DisabledTestDataFiles>().filesAndDirectories
-                if (!testDataFile.isFile || testDataFile.extension != "kt")
+            if (testDataDir in excludes)
-                    return@mapNotNull null
+                return@computeIfAbsent TestCaseGroup.ALL_DISABLED
 
-                createTestCase(testDataFile, settings)
+            val (excludedTestDataFiles, includedTestDataFiles) = testDataFiles
-            }
+                .filter { file -> file.isFile && file.extension == "kt" }
+                .partition { file -> file in excludes }
 
-            TestCaseGroup.Default(disabledTestCaseIds = emptySet(), testCases = testCases)
+            val disabledTestCaseIds = hashSetOf<TestCaseId>()
+            excludedTestDataFiles.mapTo(disabledTestCaseIds, TestCaseId::TestDataFile)
+
+            val testCases = includedTestDataFiles.map { testDataFile -> createTestCase(testDataFile, settings) }
+
+            TestCaseGroup.Default(disabledTestCaseIds, testCases)
         }
     }
 

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/UseExtTestCaseGroupProvider.kt

@@ -5,10 +5,14 @@
 
 package org.jetbrains.kotlin.konan.blackboxtest.support.group
 
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.DisabledTestDataFiles
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.GeneratedSources
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestRoots
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestConfiguration
 
 @Target(AnnotationTarget.CLASS)
-@TestConfiguration(providerClass = ExtTestCaseGroupProvider::class, requiredSettings = [TestRoots::class, GeneratedSources::class])
+@TestConfiguration(
+    providerClass = ExtTestCaseGroupProvider::class,
+    requiredSettings = [TestRoots::class, GeneratedSources::class, DisabledTestDataFiles::class]
+)
 annotation class UseExtTestCaseGroupProvider

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/UseStandardTestCaseGroupProvider.kt

@@ -5,10 +5,14 @@
 
 package org.jetbrains.kotlin.konan.blackboxtest.support.group
 
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.DisabledTestDataFiles
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.GeneratedSources
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestRoots
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestConfiguration
 
 @Target(AnnotationTarget.CLASS)
-@TestConfiguration(providerClass = StandardTestCaseGroupProvider::class, requiredSettings = [TestRoots::class, GeneratedSources::class])
+@TestConfiguration(
+    providerClass = StandardTestCaseGroupProvider::class,
+    requiredSettings = [TestRoots::class, GeneratedSources::class, DisabledTestDataFiles::class]
+)
 annotation class UseStandardTestCaseGroupProvider

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestClassSettings.kt

@@ -5,6 +5,8 @@
 
 package org.jetbrains.kotlin.konan.blackboxtest.support.settings
 
+import org.jetbrains.kotlin.konan.blackboxtest.AbstractNativeBlackBoxTest
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestCaseId
 import java.io.File
 
 /**
@@ -28,3 +30,9 @@ internal class Binaries(val testBinariesDir: File, val sharedBinariesDir: File)
  * The [TestConfiguration] of the current test class and the [Annotation] with specific parameters.
  */
 internal data class ComputedTestConfiguration(val configuration: TestConfiguration, val annotation: Annotation)
+
+/**
+ * The tests (test data files and directories with test data files) that are disabled.
+ * This is applicable only to inheritors of [AbstractNativeBlackBoxTest] that use [TestCaseId.TestDataFile].
+ */
+internal class DisabledTestDataFiles(val filesAndDirectories: Set<File>)

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/Annotations.kt

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.konan.blackboxtest.support.util
+
+/**
+ * Since Kotlin annotations are not inherited (see KT-22265), we need a utility to collect all of them.
+ */
+internal val Class<*>.allInheritedAnnotations: List<Annotation>
+    get() {
+        val annotations = mutableListOf<Annotation>()
+
+        val processedClasses = hashSetOf<Class<*>>()
+        fun process(clazz: Class<*>) {
+            if (processedClasses.add(clazz)) {
+                clazz.declaredAnnotations.mapNotNullTo(annotations) { annotation ->
+                    annotation.takeIf { it !is Metadata }
+                }
+                clazz.interfaces.forEach(::process)
+                clazz.superclass.takeIf { it != Object::class.java }?.let(::process)
+            }
+        }
+
+        process(this)
+
+        return annotations
+    }