Improve test exceptions fixation:

- exceptions is analyzed if it's specified explicitly only, - compute test case number for diagnostic tests in which an exception is thrown.
2018-12-28 15:43:37 +03:00
parent 317da3d481
commit 06b5a42d0d
45 changed files with 117 additions and 59 deletions
@@ -13,14 +13,20 @@ import java.io.PrintWriter
 import java.util.regex.Matcher
 import java.util.regex.Pattern

-enum class TestsExceptionFilePostfix(val text: String) {
+enum class TestsExceptionType(val postfix: String) {
    COMPILER_ERROR("compiler"),
    COMPILETIME_ERROR("compiletime"),
    RUNTIME_ERROR("runtime"),
-    INFRASTRUCTURE_ERROR("infrastructure")
+    INFRASTRUCTURE_ERROR("infrastructure");
+
+    companion object {
+        private val map = values().associateBy(TestsExceptionType::postfix)
+
+        fun fromValue(type: String) = map[type]
+    }
 }

-sealed class TestsError(val original: Throwable, val postfix: TestsExceptionFilePostfix) : Error() {
+sealed class TestsError(val original: Throwable, val type: TestsExceptionType) : Error() {
    override fun toString(): String = original.toString()
    override fun getStackTrace(): Array<out StackTraceElement> = original.stackTrace
    override fun initCause(cause: Throwable?): Throwable = original.initCause(cause)
@@ -38,10 +44,10 @@ sealed class TestsError(val original: Throwable, val postfix: TestsExceptionFile
    override fun printStackTrace(s: PrintWriter?) = original.printStackTrace(s)
 }

-class TestsCompilerError(original: Throwable) : TestsError(original, TestsExceptionFilePostfix.COMPILER_ERROR)
-class TestsInfrastructureError(original: Throwable) : TestsError(original, TestsExceptionFilePostfix.INFRASTRUCTURE_ERROR)
-class TestsCompiletimeError(original: Throwable) : TestsError(original, TestsExceptionFilePostfix.COMPILETIME_ERROR)
-class TestsRuntimeError(original: Throwable) : TestsError(original, TestsExceptionFilePostfix.RUNTIME_ERROR)
+class TestsCompilerError(original: Throwable) : TestsError(original, TestsExceptionType.COMPILER_ERROR)
+class TestsInfrastructureError(original: Throwable) : TestsError(original, TestsExceptionType.INFRASTRUCTURE_ERROR)
+class TestsCompiletimeError(original: Throwable) : TestsError(original, TestsExceptionType.COMPILETIME_ERROR)
+class TestsRuntimeError(original: Throwable) : TestsError(original, TestsExceptionType.RUNTIME_ERROR)

 private enum class ExceptionType {
    ANALYZING_EXPRESSION,
@@ -71,8 +77,8 @@ class TestExceptionsComparator(wholeFile: File) {
        return null
    }

-    private fun getExceptionInfo(e: TestsError, cases: Set<Int>?): String {
-        val casesAsString = cases?.run { "CASES: " + joinToString() + ls } ?: ""
+    private fun getExceptionInfo(e: TestsError, exceptionByCases: Set<Int>?): String {
+        val casesAsString = exceptionByCases?.run { "CASES: " + joinToString() + ls } ?: ""

        return when (e) {
            is TestsRuntimeError ->
@@ -84,28 +90,35 @@ class TestExceptionsComparator(wholeFile: File) {
    }

    private fun validateExistingExceptionFiles(e: TestsError?) {
-        val postfixesOfFilesToCheck = TestsExceptionFilePostfix.values().toMutableSet().filter { it != e?.postfix }
+        val postfixesOfFilesToCheck = TestsExceptionType.values().toMutableSet().filter { it != e?.type }

        postfixesOfFilesToCheck.forEach {
-            if (File("$filePathPrefix.${it.text}.txt").exists())
-                Assert.fail("No $it, but file $filePathPrefix.${it.text}.txt exists.")
+            if (File("$filePathPrefix.${it.postfix}.txt").exists())
+                Assert.fail("No $it, but file $filePathPrefix.${it.postfix}.txt exists.")
        }
    }

-    fun runAndCompareWithExpected(checkUnexpectedBehaviour: ((Matcher?) -> Pair<Boolean, Set<Int>?>)? = null, runnable: () -> Unit) {
+    fun run(
+        expectedException: TestsExceptionType?,
+        exceptionByCases: Map<Int, TestsExceptionType?> = mapOf(),
+        computeExceptionPoint: ((Matcher?) -> Set<Int>?)? = null,
+        runnable: () -> Unit
+    ) {
        try {
            runnable()
        } catch (e: TestsError) {
-            val exceptionInfo = analyze(e.original)
-            val unexpectedBehaviourCheckResult = checkUnexpectedBehaviour?.invoke(exceptionInfo)
+            val analyzeResult = analyze(e.original)
+            val casesWithExpectedException =
+                computeExceptionPoint?.invoke(analyzeResult)?.filter { exceptionByCases[it] == e.type }?.toSet()

-            if (e is TestsCompilerError && unexpectedBehaviourCheckResult?.first == false)
-                throw e.original
+            if (casesWithExpectedException == null && e.type != expectedException) {
+                throw e
+            }

-            val exceptionsFile = File("$filePathPrefix.${e.postfix.text}.txt")
+            val exceptionsFile = File("$filePathPrefix.${e.type.postfix}.txt")

            try {
-                KotlinTestUtils.assertEqualsToFile(exceptionsFile, getExceptionInfo(e, unexpectedBehaviourCheckResult?.second))
+                KotlinTestUtils.assertEqualsToFile(exceptionsFile, getExceptionInfo(e, casesWithExpectedException))
            } catch (t: AssertionError) {
                e.original.printStackTrace()
                throw t
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_1Kt.box(1.kt:18)
+_1Kt.box(1.kt:19)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Bar(val x: Int)
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-Foo$A.<init>(10.kt:12)
+Foo$A.<init>(10.kt:13)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(a: Any) {
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_11Kt.box(11.kt:20)
+_11Kt.box(11.kt:21)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 object MyObject : Foo(prop)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 val prop = MyObject
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(val prop: Int) {
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_14Kt.box(14.kt:16)
+_14Kt.box(14.kt:17)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(val prop: Int) {
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_15Kt.box(15.kt:16)
+_15Kt.box(15.kt:17)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(val prop: Int) {
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(val prop: Int) {
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(val prop: Int) {
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_18Kt.box(18.kt:16)
+_18Kt.box(18.kt:17)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(val prop: Int) {
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_19Kt.box(19.kt:16)
+_19Kt.box(19.kt:17)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(val prop: Int) {
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_2Kt.box(2.kt:18)
+_2Kt.box(2.kt:19)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Bar(val x: Int)
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_3Kt.box(3.kt:18)
+_3Kt.box(3.kt:19)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Bar(val x: Int)
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_4Kt.box(4.kt:18)
+_4Kt.box(4.kt:19)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Bar(val x: Int)
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_5Kt.box(5.kt:20)
+_5Kt.box(5.kt:21)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Bar(val x: Int)
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_6Kt.box(6.kt:20)
+_6Kt.box(6.kt:21)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Bar(val x: Any)
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_7Kt.box(7.kt:16)
+_7Kt.box(7.kt:17)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(val x: Int)
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_8Kt.box(8.kt:16)
+_8Kt.box(8.kt:17)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(a: Any) {
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(a: Any) {
@@ -33,6 +33,7 @@ fun case_3(): Boolean {
 * TESTCASE NUMBER: 4
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-26386
+ * EXCEPTION: compiler
 */
 fun case_4(): Boolean? {
    contract { returns(null) implies case_4() }
@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.checkers

 import com.intellij.openapi.util.io.FileUtil
 import org.jetbrains.kotlin.TestExceptionsComparator
+import org.jetbrains.kotlin.TestsExceptionType
 import org.jetbrains.kotlin.config.LanguageVersionSettings
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.psi.KtFile
@@ -78,9 +79,8 @@ abstract class AbstractDiagnosticsTestSpec : AbstractDiagnosticsTest() {

        println(specTest)

-        val checkUnexpectedBehaviour: (Matcher?) -> Pair<Boolean, Set<Int>?> = l@{ matches ->
-            if (specTest.unexpectedBehavior) return@l Pair(true, null)
-            if (matches == null) return@l Pair(false, null)
+        val computeExceptionPoint: (Matcher?) -> Set<Int>? = l@{ matches ->
+            if (matches == null) return@l null

            val lineNumber = matches.group("lineNumber").toInt()
            val symbolNumber = matches.group("symbolNumber").toInt()
@@ -90,10 +90,11 @@ abstract class AbstractDiagnosticsTestSpec : AbstractDiagnosticsTest() {
            val testCases = specTest.cases.byRanges[filename]
            val testCasesWithSamePosition = testCases!!.floorEntry(exceptionPosition).value

-            return@l Pair(testCasesWithSamePosition.all { it.value.unexpectedBehavior }, testCasesWithSamePosition.keys.toSet())
+            return@l testCasesWithSamePosition.keys.toSet()
        }

-        TestExceptionsComparator(testDataFile).runAndCompareWithExpected(checkUnexpectedBehaviour) {
+        val exceptionsInCases = specTest.cases.byNumbers.entries.associate { it.key to it.value.exception }
+        TestExceptionsComparator(testDataFile).run(specTest.exception, exceptionsInCases, computeExceptionPoint) {
            super.analyzeAndCheck(testDataFile, files)
        }
    }
@@ -61,8 +61,12 @@ abstract class AbstractBlackBoxCodegenTestSpec : AbstractBlackBoxCodegenTest() {

        includeHelpers(wholeFile, files, specTest)

-        TestExceptionsComparator(wholeFile).runAndCompareWithExpected {
+        if (specTest.exception == null) {
            super.doMultiFileTest(wholeFile, files, javaFilesDir)
+        } else {
+            TestExceptionsComparator(wholeFile).run(specTest.exception) {
+                super.doMultiFileTest(wholeFile, files, javaFilesDir)
+            }
        }
    }
 }
@@ -22,16 +22,20 @@ abstract class AbstractParsingTestSpec : AbstractParsingTest() {

        println(specTest)

-        TestExceptionsComparator(file).runAndCompareWithExpected({ Pair(specTest.unexpectedBehavior, null) }) {
+        if (specTest.exception == null) {
            super.doParsingTest(filePath, CommonParser::testInfoFilter)
-
-            try {
-                val psiTestValidator = ParsingTestTypeValidator(myFile, File(filePath), specTest)
-                psiTestValidator.validatePathConsistency(testLinkedType)
-                psiTestValidator.validateTestType()
-            } catch (e: SpecTestValidationException) {
-                Assert.fail(e.description)
+        } else {
+            TestExceptionsComparator(file).run(specTest.exception) {
+                super.doParsingTest(filePath, CommonParser::testInfoFilter)
            }
        }
+
+        try {
+            val psiTestValidator = ParsingTestTypeValidator(myFile, File(filePath), specTest)
+            psiTestValidator.validatePathConsistency(testLinkedType)
+            psiTestValidator.validateTestType()
+        } catch (e: SpecTestValidationException) {
+            Assert.fail(e.description)
+        }
    }
 }
@@ -5,6 +5,7 @@

 package org.jetbrains.kotlin.spec

+import org.jetbrains.kotlin.TestsExceptionType
 import org.jetbrains.kotlin.spec.models.LinkedSpecTestFileInfoElementType
 import org.jetbrains.kotlin.spec.models.NotLinkedSpecTestFileInfoElementType
 import org.jetbrains.kotlin.spec.parsers.BasePatterns
@@ -74,7 +75,8 @@ data class SpecTestCase(
    var ranges: MutableList<IntRange>,
    var unexpectedBehavior: Boolean,
    var unspecifiedBehavior: Boolean,
-    val issues: MutableList<String>?
+    val issues: MutableList<String>?,
+    val exception: TestsExceptionType?
 )

 data class SpecTestCasesSet(
@@ -5,6 +5,7 @@

 package org.jetbrains.kotlin.spec.models

+import org.jetbrains.kotlin.TestsExceptionType
 import org.jetbrains.kotlin.spec.*
 import org.jetbrains.kotlin.spec.parsers.CommonPatterns
 import org.jetbrains.kotlin.spec.parsers.CommonPatterns.issuesPattern
@@ -22,7 +23,8 @@ enum class CommonInfoElementType(
    UNEXPECTED_BEHAVIOUR,
    ISSUES(valuePattern = issuesPattern),
    DISCUSSION,
-    NOTE
+    NOTE,
+    EXCEPTION
 }

 enum class CommonSpecTestFileInfoElementType(
@@ -52,7 +54,8 @@ abstract class AbstractSpecTest(
    val cases: SpecTestCasesSet,
    val unexpectedBehavior: Boolean,
    val issues: Set<String>,
-    val helpers: Set<String>?
+    val helpers: Set<String>?,
+    val exception: TestsExceptionType?
 ) {
    companion object {
        private fun issuesToString(issues: Set<String>) = issues.joinToString(", ") { CommonPatterns.ISSUE_TRACKER + it }
@@ -5,6 +5,7 @@

 package org.jetbrains.kotlin.spec.models

+import org.jetbrains.kotlin.TestsExceptionType
 import org.jetbrains.kotlin.spec.SpecTestCasesSet
 import org.jetbrains.kotlin.spec.SpecTestInfoElementType
 import org.jetbrains.kotlin.spec.TestArea
@@ -46,8 +47,9 @@ class LinkedSpecTest(
    unexpectedBehavior: Boolean,
    private val unspecifiedBehavior: Boolean,
    issues: Set<String>,
-    helpers: Set<String>?
-) : AbstractSpecTest(testArea, testType, place.sections, testNumber, description, cases, unexpectedBehavior, issues, helpers) {
+    helpers: Set<String>?,
+    exception: TestsExceptionType?
+) : AbstractSpecTest(testArea, testType, place.sections, testNumber, description, cases, unexpectedBehavior, issues, helpers, exception) {
    override fun checkPathConsistency(pathMatcher: Matcher) =
        testArea == TestArea.valueOf(pathMatcher.group("testArea").withUnderscores())
                && testType == TestType.fromValue(pathMatcher.group("testType"))!!
@@ -5,6 +5,7 @@

 package org.jetbrains.kotlin.spec.models

+import org.jetbrains.kotlin.TestsExceptionType
 import org.jetbrains.kotlin.spec.SpecTestCasesSet
 import org.jetbrains.kotlin.spec.SpecTestInfoElementType
 import org.jetbrains.kotlin.spec.TestArea
@@ -33,8 +34,9 @@ class NotLinkedSpecTest(
    cases: SpecTestCasesSet,
    unexpectedBehavior: Boolean,
    issues: Set<String>,
-    helpers: Set<String>?
-) : AbstractSpecTest(testArea, testType, sections, testNumber, description, cases, unexpectedBehavior, issues, helpers) {
+    helpers: Set<String>?,
+    exception: TestsExceptionType?
+) : AbstractSpecTest(testArea, testType, sections, testNumber, description, cases, unexpectedBehavior, issues, helpers, exception) {
    override fun checkPathConsistency(pathMatcher: Matcher) =
        testArea == TestArea.valueOf(pathMatcher.group("testArea").withUnderscores())
                && testType == TestType.fromValue(pathMatcher.group("testType"))!!
@@ -72,7 +72,8 @@ object CommonParser {
            parsedTestFile.unexpectedBehavior,
            LinkedSpecTestFileInfoElementType.UNSPECIFIED_BEHAVIOR in testInfoElements,
            parsedTestFile.issues,
-            parsedTestFile.helpers
+            parsedTestFile.helpers,
+            parsedTestFile.exception
        )
    }

@@ -90,7 +91,8 @@ object CommonParser {
            parsedTestFile.testCasesSet,
            parsedTestFile.unexpectedBehavior,
            parsedTestFile.issues,
-            parsedTestFile.helpers
+            parsedTestFile.helpers,
+            parsedTestFile.exception
        )
    }

@@ -5,6 +5,7 @@

 package org.jetbrains.kotlin.spec.parsers

+import org.jetbrains.kotlin.TestsExceptionType
 import org.jetbrains.kotlin.spec.*
 import org.jetbrains.kotlin.spec.models.CommonInfoElementType
 import org.jetbrains.kotlin.spec.models.LinkedSpecTestFileInfoElementType
@@ -74,7 +75,8 @@ fun parseTestCases(testFiles: TestFiles): SpecTestCasesSet {
                ranges = mutableListOf(range),
                unexpectedBehavior = caseInfoElements.contains(CommonInfoElementType.UNEXPECTED_BEHAVIOUR),
                unspecifiedBehavior = caseInfoElements.contains(LinkedSpecTestFileInfoElementType.UNSPECIFIED_BEHAVIOR),
-                issues = CommonParser.parseIssues(caseInfoElements[CommonInfoElementType.ISSUES]).toMutableList()
+                issues = CommonParser.parseIssues(caseInfoElements[CommonInfoElementType.ISSUES]).toMutableList(),
+                exception = caseInfoElements[CommonInfoElementType.EXCEPTION]?.content?.let { TestsExceptionType.fromValue(it) }
            ).save(testCasesSet.byNumbers, testCasesOfFile, testCasesByRangesOfFile, caseInfoElements)

            startFind = matcher.end() - nextDirective.length
@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.spec.parsers

 import com.intellij.openapi.util.io.FileUtil
+import org.jetbrains.kotlin.TestsExceptionType
 import org.jetbrains.kotlin.spec.*
 import org.jetbrains.kotlin.spec.models.CommonInfoElementType
 import org.jetbrains.kotlin.spec.models.CommonSpecTestFileInfoElementType
@@ -24,7 +25,8 @@ data class ParsedTestFile(
    val testCasesSet: SpecTestCasesSet,
    val unexpectedBehavior: Boolean,
    val issues: Set<String>,
-    val helpers: Set<String>?
+    val helpers: Set<String>?,
+    val exception: TestsExceptionType?
 )

 fun parseTestInfo(testFilePath: String, testFiles: TestFiles, linkedTestType: SpecTestLinkedType): ParsedTestFile {
@@ -54,6 +56,7 @@ fun parseTestInfo(testFilePath: String, testFiles: TestFiles, linkedTestType: Sp
        testCasesSet = parseTestCases(testFiles),
        unexpectedBehavior = testInfoElements.contains(CommonInfoElementType.UNEXPECTED_BEHAVIOUR),
        issues = CommonParser.parseIssues(testInfoElements[CommonInfoElementType.ISSUES]),
-        helpers = helpers
+        helpers = helpers,
+        exception = testInfoElements[CommonInfoElementType.EXCEPTION]?.content?.let { TestsExceptionType.fromValue(it) }
    )
 }