JSpecify: implement tests generator and test runner with checking compliance of kotlin diagnostics and jspecify marks

2020-10-09 14:57:18 +03:00
parent f39e2d4a2d
commit 9dab052266
8 changed files with 349 additions and 245 deletions
@@ -242,9 +242,7 @@ abstract class AbstractDiagnosticsTest : BaseDiagnosticsTest() {
            )
        }

-        KotlinTestUtils.assertEqualsToFile(getExpectedDiagnosticsFile(testDataFile), actualText.cleanupInferenceDiagnostics()) { s ->
-            s.replace("COROUTINES_PACKAGE", coroutinesPackage)
-        }
+        checkDiagnostics(actualText.cleanupInferenceDiagnostics(), testDataFile)

        assertTrue("Diagnostics mismatch. See the output above", ok)

@@ -261,6 +259,12 @@ abstract class AbstractDiagnosticsTest : BaseDiagnosticsTest() {
        }
    }

+    protected open fun checkDiagnostics(actualText: String, testDataFile: File) {
+        KotlinTestUtils.assertEqualsToFile(getExpectedDiagnosticsFile(testDataFile), actualText) { s ->
+            s.replace("COROUTINES_PACKAGE", coroutinesPackage)
+        }
+    }
+
    private fun checkFirTestdata(testDataFile: File, files: List<TestFile>) {
        val firTestDataFile = File(testDataFile.absolutePath.replace(".kt", ".fir.kt"))
        val firFailFile = File(testDataFile.absolutePath.replace(".kt", ".fir.fail"))
@@ -0,0 +1,166 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.checkers
+
+import com.intellij.openapi.util.io.FileUtil
+import org.jetbrains.kotlin.checkers.utils.CheckerTestUtil
+import org.jetbrains.kotlin.resolve.jvm.diagnostics.ErrorsJvm.NULLABILITY_MISMATCH_BASED_ON_JAVA_ANNOTATIONS
+import org.jetbrains.kotlin.test.MockLibraryUtil
+import java.io.File
+import java.util.*
+import java.util.regex.Pattern
+
+const val JSPECIFY_NULLNESS_MISMATCH_MARK = "jspecify_nullness_mismatch"
+
+const val JSPECIFY_NULLABLE_ANNOTATION = "@Nullable"
+const val JSPECIFY_NULLNESS_UNSPECIFIED_ANNOTATION = "@NullnessUnspecified"
+
+abstract class AbstractJspecifyAnnotationsTest : AbstractDiagnosticsTest() {
+    override fun doMultiFileTest(
+        wholeFile: File,
+        files: List<TestFile>
+    ) {
+        super.doMultiFileTest(
+            wholeFile,
+            files,
+            MockLibraryUtil.compileJavaFilesLibraryToJar(FOREIGN_ANNOTATIONS_SOURCES_PATH, "foreign-annotations")
+        )
+    }
+
+    override fun doTest(filePath: String) {
+        val ktSourceCode = File(filePath).readText()
+        val javaSourcesFilename = javaSourcesPathRegex.matcher(ktSourceCode).also { it.find() }.group(1)
+            ?: throw Exception("Java sources' path not found")
+        val javaSourcesFile = File("$JSPECIFY_JAVA_SOURCES_PATH/$javaSourcesFilename")
+        val mergedSourceCode = buildString {
+            appendLine("// ORIGINAL_KT_FILE: $filePath")
+
+            if (javaSourcesFilename.endsWith(".java")) {
+                appendLine(makeJavaClassesPublicAndSeparatedByFiles(javaSourcesFile.readText()))
+            } else {
+                assert(javaSourcesFile.isDirectory) { "Specified Java sources should be a file with `java` extension or directory" }
+
+                for (javaFile in javaSourcesFile.walkTopDown().filter { it.isFile && it.extension == "java" }) {
+                    appendLine("// FILE: ${javaFile.name}")
+                    appendLine(makeJavaClassesPublicAndSeparatedByFiles(javaFile.readText()))
+                }
+            }
+            appendLine("// FILE: main.kt\n$ktSourceCode")
+        }
+
+        super.doTest(createTempFile().apply { writeText(mergedSourceCode) }.path)
+    }
+
+    private fun makeJavaClassesPublicAndSeparatedByFiles(javaCode: String): String {
+        val importSectionMatch = importSectionRegex.find(javaCode)
+        val importSection = if (importSectionMatch != null) "\n${importSectionMatch.groups[1]!!.value}\n" else ""
+
+        return javaCode.replace(importSectionRegex, "")
+            .replace(publicClassOrInterfaceRegex, "$1")
+            .replace(classOrInterfaceRegex, "public $1")
+            .replace(classShapeRegex, "\n// FILE: $3.java$importSection$1\npublic $2 $3$4")
+    }
+
+    private fun getJspecifyMarkRegex(jspecifyMark: String) = Regex("""^\s*// $jspecifyMark$""")
+
+    private fun checkIfAllJspecifyMarksByDiagnosticsArePresent(
+        diagnosedRanges: List<DiagnosedRange>,
+        lineIndexesByRanges: TreeMap<Int, Int>,
+        textLines: List<String>
+    ) {
+        for (diagnosticRange in diagnosedRanges) {
+            val lineIndex = lineIndexesByRanges.floorEntry(diagnosticRange.start).value
+
+            for (diagnostic in diagnosticRange.getDiagnostics()) {
+                val requiredJspecifyMark = diagnosticsToJspecifyMarksMap[diagnostic.name] ?: continue
+
+                fun getErrorMessage(lineIndex: Int) =
+                    "Jspecify mark '$requiredJspecifyMark' not found for diagnostic '${diagnostic}' at ${lineIndex + 1} line.\n" +
+                            "It should be located at the previous line as a comment."
+
+                assert(lineIndex != 0) { getErrorMessage(0) }
+
+                val previousLine = textLines[lineIndex - 1]
+
+                assert(getJspecifyMarkRegex(requiredJspecifyMark).matches(previousLine)) { getErrorMessage(lineIndex) }
+            }
+        }
+    }
+
+    private fun checkIfAllDiagnosticsByJspecifyMarksArePresent(
+        diagnosedRanges: List<DiagnosedRange>,
+        lineIndexesByRanges: TreeMap<Int, Int>,
+        textLines: List<String>
+    ) {
+        for ((diagnostic, jspecifyMark) in diagnosticsToJspecifyMarksMap) {
+            val diagnosticRanges = diagnosedRanges.filter { diagnostics ->
+                diagnostic.name in diagnostics.getDiagnostics().map { it.name }
+            }.map { it.start }
+            val lineIndexesWithJspecifyMarks =
+                textLines.mapIndexedNotNull { index, it -> getJspecifyMarkRegex(jspecifyMark).find(it)?.let { index } }
+
+            for (lineIndex in lineIndexesWithJspecifyMarks) {
+                val lineStartPosition = lineIndexesByRanges.entries.find { (_, index) -> index == lineIndex + 1 }?.key
+                val errorMessage = "Diagnostic '$diagnostic' not found for jspecify mark '$jspecifyMark' at ${lineIndex + 1} line"
+
+                assertNotNull(errorMessage, lineStartPosition)
+
+                val lineEndPosition = lineStartPosition!! + textLines[lineIndex].length
+                val isCorrespondingDiagnosticPresent = diagnosticRanges.any { it in lineStartPosition..lineEndPosition }
+
+                assertTrue(errorMessage, isCorrespondingDiagnosticPresent)
+            }
+        }
+    }
+
+    override fun checkDiagnostics(actualText: String, testDataFile: File) {
+        val mergedTestFilePath = originalKtFileRegex.matcher(actualText).also { it.find() }.group(1)
+            ?: throw Exception("Path for original kt file in the merged file not found")
+
+        val textWithDiagnostics = actualText.substringAfter(MAIN_KT_FILE_DIRECTIVE).removeSuffix("\n")
+        val diagnosedRanges = mutableListOf<DiagnosedRange>()
+        val textWithoutDiagnostics = CheckerTestUtil.parseDiagnosedRanges(textWithDiagnostics, diagnosedRanges)
+
+        val textLines = textWithoutDiagnostics.lines()
+        val lineIndexesByRanges = TreeMap<Int, Int>().apply {
+            textLines.scanIndexed(0) { index, position, line ->
+                put(position, index)
+                position + line.length + 1 // + new line symbol
+            }
+        }
+
+        checkIfAllJspecifyMarksByDiagnosticsArePresent(diagnosedRanges, lineIndexesByRanges, textLines)
+
+        checkIfAllDiagnosticsByJspecifyMarksArePresent(diagnosedRanges, lineIndexesByRanges, textLines)
+
+        super.checkDiagnostics(textWithDiagnostics, File(mergedTestFilePath))
+    }
+
+    override fun getExpectedDescriptorFile(testDataFile: File, files: List<TestFile>): File {
+        val originalKtFilePath = originalKtFileRegex.matcher(testDataFile.readText()).also { it.find() }.group(1)
+            ?: throw Exception("Path for original kt file in the merged file isn't found")
+
+        return File(FileUtil.getNameWithoutExtension(originalKtFilePath) + ".txt")
+    }
+
+    companion object {
+        const val FOREIGN_ANNOTATIONS_SOURCES_PATH = "third-party/jdk8-annotations"
+        const val JSPECIFY_JAVA_SOURCES_PATH = "compiler/testData/foreignAnnotationsJava8/tests/jspecify/java"
+        const val MAIN_KT_FILE_DIRECTIVE = "// FILE: main.kt\n"
+
+        private val originalKtFileRegex = Pattern.compile("""// ORIGINAL_KT_FILE: (.*?\.kts?)\n""")
+        private val javaSourcesPathRegex = Pattern.compile("""// JAVA_SOURCES: (.*?(?:\.java)?)\n""")
+
+        val diagnosticsToJspecifyMarksMap = mapOf(
+            NULLABILITY_MISMATCH_BASED_ON_JAVA_ANNOTATIONS to "jspecify_nullness_mismatch"
+        )
+
+        private val importSectionRegex = Regex("""((?:import .*?;\n)+)""")
+        private val classOrInterfaceRegex = Regex("""(class|interface)""")
+        private val publicClassOrInterfaceRegex = Regex("""public (class|interface)""")
+        private val classShapeRegex = Regex("""(\n@DefaultNonNull)?\npublic (class|interface) (\w+)(<[^>]+>)?""")
+    }
+}
@@ -86,12 +86,12 @@ abstract class BaseDiagnosticsTest : KotlinMultiFileTestWithJava<TestModule, Tes
    override fun createTestFile(module: TestModule?, fileName: String, text: String, directives: Directives): TestFile =
        TestFile(module, fileName, text, directives)

-
-    override fun doMultiFileTest(
+    fun doMultiFileTest(
        wholeFile: File,
-        files: List<TestFile>
+        files: List<TestFile>,
+        additionalClasspath: File? = null
    ) {
-        environment = createEnvironment(wholeFile, files)
+        environment = createEnvironment(wholeFile, files, additionalClasspath)
        //after environment initialization cause of `tearDown` logic, maybe it's obsolete
        if (shouldSkipTest(wholeFile, files)) {
            println("${wholeFile.name} test is skipped")
@@ -101,6 +101,10 @@ abstract class BaseDiagnosticsTest : KotlinMultiFileTestWithJava<TestModule, Tes
        analyzeAndCheck(wholeFile, files)
    }

+    override fun doMultiFileTest(wholeFile: File, files: List<TestFile>) {
+        doMultiFileTest(wholeFile, files, null)
+    }
+
    protected open fun shouldSkipTest(wholeFile: File, files: List<TestFile>): Boolean = false

    protected abstract fun analyzeAndCheck(testDataFile: File, files: List<TestFile>)
@@ -41,13 +41,14 @@ abstract class KotlinMultiFileTestWithJava<M : KotlinBaseTest.TestModule, F : Ko

    protected fun createEnvironment(
        file: File,
-        files: List<F>
+        files: List<F>,
+        additionalClasspath: File? = null
    ): KotlinCoreEnvironment {
        val configuration = createConfiguration(
            extractConfigurationKind(files),
            getTestJdkKind(files),
            backend,
-            getClasspath(file),
+            if (additionalClasspath == null) getClasspath(file) else getClasspath(file) + additionalClasspath,
            if (isJavaSourceRootNeeded()) listOf(javaFilesDir) else emptyList(),
            files
        )
@@ -25,7 +25,7 @@ public class ForeignJava8AnnotationsTestGenerated extends AbstractForeignJava8An
    }

    public void testAllFilesPresentInTests() throws Exception {
-        KotlinTestUtils.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/foreignAnnotationsJava8/tests"), Pattern.compile("^(.+)\\.kt$"), null, true);
+        KotlinTestUtils.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/foreignAnnotationsJava8/tests"), Pattern.compile("^(.+)\\.kt$"), null, true, "jspecify");
    }

    @TestMetadata("checkerFramework.kt")
@@ -43,122 +43,6 @@ public class ForeignJava8AnnotationsTestGenerated extends AbstractForeignJava8An
        runTest("compiler/testData/foreignAnnotationsJava8/tests/typeUseOnObject.kt");
    }

-    @TestMetadata("compiler/testData/foreignAnnotationsJava8/tests/jspecify")
-    @TestDataPath("$PROJECT_ROOT")
-    @RunWith(JUnit3RunnerWithInners.class)
-    public static class Jspecify extends AbstractForeignJava8AnnotationsTest {
-        private void runTest(String testDataFilePath) throws Exception {
-            KotlinTestUtils.runTest(this::doTest, this, testDataFilePath);
-        }
-
-        public void testAllFilesPresentInJspecify() throws Exception {
-            KotlinTestUtils.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/foreignAnnotationsJava8/tests/jspecify"), Pattern.compile("^(.+)\\.kt$"), null, true);
-        }
-
-        @TestMetadata("annotatedWildcards.kt")
-        public void testAnnotatedWildcards() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/annotatedWildcards.kt");
-        }
-
-        @TestMetadata("defaults.kt")
-        public void testDefaults() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/defaults.kt");
-        }
-
-        @TestMetadata("ignoreAnnotations.kt")
-        public void testIgnoreAnnotations() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/ignoreAnnotations.kt");
-        }
-
-        @TestMetadata("nonPlatformTypeParameter.kt")
-        public void testNonPlatformTypeParameter() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/nonPlatformTypeParameter.kt");
-        }
-
-        @TestMetadata("selfType.kt")
-        public void testSelfType() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/selfType.kt");
-        }
-
-        @TestMetadata("simple.kt")
-        public void testSimple() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/simple.kt");
-        }
-
-        @TestMetadata("typeArgumentsFromParameterBounds.kt")
-        public void testTypeArgumentsFromParameterBounds() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/typeArgumentsFromParameterBounds.kt");
-        }
-
-        @TestMetadata("typeParameterBounds.kt")
-        public void testTypeParameterBounds() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/typeParameterBounds.kt");
-        }
-
-        @TestMetadata("unknownNullnessTypeParameter.kt")
-        public void testUnknownNullnessTypeParameter() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/unknownNullnessTypeParameter.kt");
-        }
-
-        @TestMetadata("wildcardsWithDefault.kt")
-        public void testWildcardsWithDefault() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/wildcardsWithDefault.kt");
-        }
-
-        @TestMetadata("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings")
-        @TestDataPath("$PROJECT_ROOT")
-        @RunWith(JUnit3RunnerWithInners.class)
-        public static class Warnings extends AbstractForeignJava8AnnotationsTest {
-            private void runTest(String testDataFilePath) throws Exception {
-                KotlinTestUtils.runTest(this::doTest, this, testDataFilePath);
-            }
-
-            public void testAllFilesPresentInWarnings() throws Exception {
-                KotlinTestUtils.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings"), Pattern.compile("^(.+)\\.kt$"), null, true);
-            }
-
-            @TestMetadata("annotatedWildcards.kt")
-            public void testAnnotatedWildcards() throws Exception {
-                runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings/annotatedWildcards.kt");
-            }
-
-            @TestMetadata("defaults.kt")
-            public void testDefaults() throws Exception {
-                runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings/defaults.kt");
-            }
-
-            @TestMetadata("nonPlatformTypeParameter.kt")
-            public void testNonPlatformTypeParameter() throws Exception {
-                runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings/nonPlatformTypeParameter.kt");
-            }
-
-            @TestMetadata("simple.kt")
-            public void testSimple() throws Exception {
-                runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings/simple.kt");
-            }
-
-            @TestMetadata("typeArgumentsFromParameterBounds.kt")
-            public void testTypeArgumentsFromParameterBounds() throws Exception {
-                runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings/typeArgumentsFromParameterBounds.kt");
-            }
-
-            @TestMetadata("typeParameterBounds.kt")
-            public void testTypeParameterBounds() throws Exception {
-                runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings/typeParameterBounds.kt");
-            }
-
-            @TestMetadata("unknownNullnessTypeParameter.kt")
-            public void testUnknownNullnessTypeParameter() throws Exception {
-                runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings/unknownNullnessTypeParameter.kt");
-            }
-
-            @TestMetadata("wildcardsWithDefault.kt")
-            public void testWildcardsWithDefault() throws Exception {
-                runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings/wildcardsWithDefault.kt");
-            }
-        }
-    }
-
    @TestMetadata("compiler/testData/foreignAnnotationsJava8/tests/jsr305")
    @TestDataPath("$PROJECT_ROOT")
    @RunWith(JUnit3RunnerWithInners.class)
@@ -0,0 +1,156 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.checkers;
+
+import com.intellij.testFramework.TestDataPath;
+import org.jetbrains.kotlin.test.JUnit3RunnerWithInners;
+import org.jetbrains.kotlin.test.KotlinTestUtils;
+import org.jetbrains.kotlin.test.TestMetadata;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.util.regex.Pattern;
+
+/** This class is generated by {@link org.jetbrains.kotlin.generators.tests.TestsPackage}. DO NOT MODIFY MANUALLY */
+@SuppressWarnings("all")
+@TestMetadata("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin")
+@TestDataPath("$PROJECT_ROOT")
+@RunWith(JUnit3RunnerWithInners.class)
+public class JspecifyAnnotationsTestGenerated extends AbstractJspecifyAnnotationsTest {
+    private void runTest(String testDataFilePath) throws Exception {
+        KotlinTestUtils.runTest(this::doTest, this, testDataFilePath);
+    }
+
+    public void testAllFilesPresentInKotlin() throws Exception {
+        KotlinTestUtils.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin"), Pattern.compile("^(.+)\\.kt$"), null, true);
+    }
+
+    @TestMetadata("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/strictMode")
+    @TestDataPath("$PROJECT_ROOT")
+    @RunWith(JUnit3RunnerWithInners.class)
+    public static class StrictMode extends AbstractJspecifyAnnotationsTest {
+        private void runTest(String testDataFilePath) throws Exception {
+            KotlinTestUtils.runTest(this::doTest, this, testDataFilePath);
+        }
+
+        public void testAllFilesPresentInStrictMode() throws Exception {
+            KotlinTestUtils.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/strictMode"), Pattern.compile("^(.+)\\.kt$"), null, true);
+        }
+
+        @TestMetadata("AnnotatedBoundsOfWildcard.kt")
+        public void testAnnotatedBoundsOfWildcard() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/strictMode/AnnotatedBoundsOfWildcard.kt");
+        }
+
+        @TestMetadata("Defaults.kt")
+        public void testDefaults() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/strictMode/Defaults.kt");
+        }
+
+        @TestMetadata("IgnoreAnnotations.kt")
+        public void testIgnoreAnnotations() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/strictMode/IgnoreAnnotations.kt");
+        }
+
+        @TestMetadata("NonPlatformTypeParameter.kt")
+        public void testNonPlatformTypeParameter() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/strictMode/NonPlatformTypeParameter.kt");
+        }
+
+        @TestMetadata("NullnessUnspecifiedTypeParameter.kt")
+        public void testNullnessUnspecifiedTypeParameter() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/strictMode/NullnessUnspecifiedTypeParameter.kt");
+        }
+
+        @TestMetadata("SelfType.kt")
+        public void testSelfType() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/strictMode/SelfType.kt");
+        }
+
+        @TestMetadata("Simple.kt")
+        public void testSimple() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/strictMode/Simple.kt");
+        }
+
+        @TestMetadata("TypeArgumentsFromParameterBounds.kt")
+        public void testTypeArgumentsFromParameterBounds() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/strictMode/TypeArgumentsFromParameterBounds.kt");
+        }
+
+        @TestMetadata("TypeParameterBounds.kt")
+        public void testTypeParameterBounds() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/strictMode/TypeParameterBounds.kt");
+        }
+
+        @TestMetadata("WildcardsWithDefault.kt")
+        public void testWildcardsWithDefault() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/strictMode/WildcardsWithDefault.kt");
+        }
+    }
+
+    @TestMetadata("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/warnMode")
+    @TestDataPath("$PROJECT_ROOT")
+    @RunWith(JUnit3RunnerWithInners.class)
+    public static class WarnMode extends AbstractJspecifyAnnotationsTest {
+        private void runTest(String testDataFilePath) throws Exception {
+            KotlinTestUtils.runTest(this::doTest, this, testDataFilePath);
+        }
+
+        public void testAllFilesPresentInWarnMode() throws Exception {
+            KotlinTestUtils.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/warnMode"), Pattern.compile("^(.+)\\.kt$"), null, true);
+        }
+
+        @TestMetadata("AnnotatedBoundsOfWildcard.kt")
+        public void testAnnotatedBoundsOfWildcard() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/warnMode/AnnotatedBoundsOfWildcard.kt");
+        }
+
+        @TestMetadata("Defaults.kt")
+        public void testDefaults() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/warnMode/Defaults.kt");
+        }
+
+        @TestMetadata("IgnoreAnnotations.kt")
+        public void testIgnoreAnnotations() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/warnMode/IgnoreAnnotations.kt");
+        }
+
+        @TestMetadata("NonPlatformTypeParameter.kt")
+        public void testNonPlatformTypeParameter() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/warnMode/NonPlatformTypeParameter.kt");
+        }
+
+        @TestMetadata("NullnessUnspecifiedTypeParameter.kt")
+        public void testNullnessUnspecifiedTypeParameter() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/warnMode/NullnessUnspecifiedTypeParameter.kt");
+        }
+
+        @TestMetadata("SelfType.kt")
+        public void testSelfType() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/warnMode/SelfType.kt");
+        }
+
+        @TestMetadata("Simple.kt")
+        public void testSimple() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/warnMode/Simple.kt");
+        }
+
+        @TestMetadata("TypeArgumentsFromParameterBounds.kt")
+        public void testTypeArgumentsFromParameterBounds() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/warnMode/TypeArgumentsFromParameterBounds.kt");
+        }
+
+        @TestMetadata("TypeParameterBounds.kt")
+        public void testTypeParameterBounds() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/warnMode/TypeParameterBounds.kt");
+        }
+
+        @TestMetadata("WildcardsWithDefault.kt")
+        public void testWildcardsWithDefault() throws Exception {
+            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/kotlin/warnMode/WildcardsWithDefault.kt");
+        }
+    }
+}
@@ -25,7 +25,7 @@ public class JavacForeignJava8AnnotationsTestGenerated extends AbstractJavacFore
    }

    public void testAllFilesPresentInTests() throws Exception {
-        KotlinTestUtils.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/foreignAnnotationsJava8/tests"), Pattern.compile("^(.+)\\.kt$"), null, true);
+        KotlinTestUtils.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/foreignAnnotationsJava8/tests"), Pattern.compile("^(.+)\\.kt$"), null, true, "jspecify");
    }

    @TestMetadata("checkerFramework.kt")
@@ -43,122 +43,6 @@ public class JavacForeignJava8AnnotationsTestGenerated extends AbstractJavacFore
        runTest("compiler/testData/foreignAnnotationsJava8/tests/typeUseOnObject.kt");
    }

-    @TestMetadata("compiler/testData/foreignAnnotationsJava8/tests/jspecify")
-    @TestDataPath("$PROJECT_ROOT")
-    @RunWith(JUnit3RunnerWithInners.class)
-    public static class Jspecify extends AbstractJavacForeignJava8AnnotationsTest {
-        private void runTest(String testDataFilePath) throws Exception {
-            KotlinTestUtils.runTest(this::doTest, this, testDataFilePath);
-        }
-
-        public void testAllFilesPresentInJspecify() throws Exception {
-            KotlinTestUtils.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/foreignAnnotationsJava8/tests/jspecify"), Pattern.compile("^(.+)\\.kt$"), null, true);
-        }
-
-        @TestMetadata("annotatedWildcards.kt")
-        public void testAnnotatedWildcards() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/annotatedWildcards.kt");
-        }
-
-        @TestMetadata("defaults.kt")
-        public void testDefaults() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/defaults.kt");
-        }
-
-        @TestMetadata("ignoreAnnotations.kt")
-        public void testIgnoreAnnotations() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/ignoreAnnotations.kt");
-        }
-
-        @TestMetadata("nonPlatformTypeParameter.kt")
-        public void testNonPlatformTypeParameter() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/nonPlatformTypeParameter.kt");
-        }
-
-        @TestMetadata("selfType.kt")
-        public void testSelfType() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/selfType.kt");
-        }
-
-        @TestMetadata("simple.kt")
-        public void testSimple() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/simple.kt");
-        }
-
-        @TestMetadata("typeArgumentsFromParameterBounds.kt")
-        public void testTypeArgumentsFromParameterBounds() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/typeArgumentsFromParameterBounds.kt");
-        }
-
-        @TestMetadata("typeParameterBounds.kt")
-        public void testTypeParameterBounds() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/typeParameterBounds.kt");
-        }
-
-        @TestMetadata("unknownNullnessTypeParameter.kt")
-        public void testUnknownNullnessTypeParameter() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/unknownNullnessTypeParameter.kt");
-        }
-
-        @TestMetadata("wildcardsWithDefault.kt")
-        public void testWildcardsWithDefault() throws Exception {
-            runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/wildcardsWithDefault.kt");
-        }
-
-        @TestMetadata("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings")
-        @TestDataPath("$PROJECT_ROOT")
-        @RunWith(JUnit3RunnerWithInners.class)
-        public static class Warnings extends AbstractJavacForeignJava8AnnotationsTest {
-            private void runTest(String testDataFilePath) throws Exception {
-                KotlinTestUtils.runTest(this::doTest, this, testDataFilePath);
-            }
-
-            public void testAllFilesPresentInWarnings() throws Exception {
-                KotlinTestUtils.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings"), Pattern.compile("^(.+)\\.kt$"), null, true);
-            }
-
-            @TestMetadata("annotatedWildcards.kt")
-            public void testAnnotatedWildcards() throws Exception {
-                runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings/annotatedWildcards.kt");
-            }
-
-            @TestMetadata("defaults.kt")
-            public void testDefaults() throws Exception {
-                runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings/defaults.kt");
-            }
-
-            @TestMetadata("nonPlatformTypeParameter.kt")
-            public void testNonPlatformTypeParameter() throws Exception {
-                runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings/nonPlatformTypeParameter.kt");
-            }
-
-            @TestMetadata("simple.kt")
-            public void testSimple() throws Exception {
-                runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings/simple.kt");
-            }
-
-            @TestMetadata("typeArgumentsFromParameterBounds.kt")
-            public void testTypeArgumentsFromParameterBounds() throws Exception {
-                runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings/typeArgumentsFromParameterBounds.kt");
-            }
-
-            @TestMetadata("typeParameterBounds.kt")
-            public void testTypeParameterBounds() throws Exception {
-                runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings/typeParameterBounds.kt");
-            }
-
-            @TestMetadata("unknownNullnessTypeParameter.kt")
-            public void testUnknownNullnessTypeParameter() throws Exception {
-                runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings/unknownNullnessTypeParameter.kt");
-            }
-
-            @TestMetadata("wildcardsWithDefault.kt")
-            public void testWildcardsWithDefault() throws Exception {
-                runTest("compiler/testData/foreignAnnotationsJava8/tests/jspecify/warnings/wildcardsWithDefault.kt");
-            }
-        }
-    }
-
    @TestMetadata("compiler/testData/foreignAnnotationsJava8/tests/jsr305")
    @TestDataPath("$PROJECT_ROOT")
    @RunWith(JUnit3RunnerWithInners.class)
@@ -19,6 +19,7 @@ package org.jetbrains.kotlin.generators.tests
 import org.jetbrains.kotlin.checkers.AbstractForeignJava8AnnotationsNoAnnotationInClasspathTest
 import org.jetbrains.kotlin.checkers.AbstractForeignJava8AnnotationsNoAnnotationInClasspathWithPsiClassReadingTest
 import org.jetbrains.kotlin.checkers.AbstractForeignJava8AnnotationsTest
+import org.jetbrains.kotlin.checkers.AbstractJspecifyAnnotationsTest
 import org.jetbrains.kotlin.checkers.javac.AbstractJavacForeignJava8AnnotationsTest
 import org.jetbrains.kotlin.generators.tests.generator.testGroupSuite
 import org.jetbrains.kotlin.jvm.compiler.AbstractLoadJava8Test
@@ -32,11 +33,11 @@ fun main(args: Array<String>) {
    testGroupSuite(args) {
        testGroup("compiler/tests-java8/tests", "compiler/testData") {
            testClass<AbstractForeignJava8AnnotationsTest> {
-                model("foreignAnnotationsJava8/tests")
+                model("foreignAnnotationsJava8/tests", excludeDirs = listOf("jspecify"))
            }

            testClass<AbstractJavacForeignJava8AnnotationsTest> {
-                model("foreignAnnotationsJava8/tests")
+                model("foreignAnnotationsJava8/tests", excludeDirs = listOf("jspecify"))
            }

            testClass<AbstractForeignJava8AnnotationsNoAnnotationInClasspathTest> {
@@ -47,6 +48,10 @@ fun main(args: Array<String>) {
                model("foreignAnnotationsJava8/tests", excludeDirs = listOf("jspecify"))
            }

+            testClass<AbstractJspecifyAnnotationsTest> {
+                model("foreignAnnotationsJava8/tests/jspecify/kotlin")
+            }
+
            testClass<AbstractLoadJava8Test> {
                model("loadJava8/compiledJava", extension = "java", testMethod = "doTestCompiledJava")
                model("loadJava8/sourceJava", extension = "java", testMethod = "doTestSourceJava")