Improve test exceptions fixation:

- exceptions is analyzed if it's specified explicitly only, - compute test case number for diagnostic tests in which an exception is thrown.
2018-12-28 15:43:37 +03:00
parent 317da3d481
commit 06b5a42d0d
45 changed files with 117 additions and 59 deletions
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_1Kt.box(1.kt:18)
+_1Kt.box(1.kt:19)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Bar(val x: Int)
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-Foo$A.<init>(10.kt:12)
+Foo$A.<init>(10.kt:13)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(a: Any) {
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_11Kt.box(11.kt:20)
+_11Kt.box(11.kt:21)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 object MyObject : Foo(prop)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 val prop = MyObject
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(val prop: Int) {
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_14Kt.box(14.kt:16)
+_14Kt.box(14.kt:17)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(val prop: Int) {
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_15Kt.box(15.kt:16)
+_15Kt.box(15.kt:17)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(val prop: Int) {
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(val prop: Int) {
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(val prop: Int) {
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_18Kt.box(18.kt:16)
+_18Kt.box(18.kt:17)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(val prop: Int) {
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_19Kt.box(19.kt:16)
+_19Kt.box(19.kt:17)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(val prop: Int) {
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_2Kt.box(2.kt:18)
+_2Kt.box(2.kt:19)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Bar(val x: Int)
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_3Kt.box(3.kt:18)
+_3Kt.box(3.kt:19)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Bar(val x: Int)
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_4Kt.box(4.kt:18)
+_4Kt.box(4.kt:19)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Bar(val x: Int)
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_5Kt.box(5.kt:20)
+_5Kt.box(5.kt:21)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Bar(val x: Int)
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_6Kt.box(6.kt:20)
+_6Kt.box(6.kt:21)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Bar(val x: Any)
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_7Kt.box(7.kt:16)
+_7Kt.box(7.kt:17)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(val x: Int)
@@ -1,2 +1,2 @@
 java.lang.ExceptionInInitializerError
-_8Kt.box(8.kt:16)
+_8Kt.box(8.kt:17)
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(a: Any) {
@@ -6,6 +6,7 @@
 * DESCRIPTION: Access to class members in the super constructor call of an object.
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-25289
+ * EXCEPTION: runtime
 */

 open class Foo(a: Any) {
@@ -33,6 +33,7 @@ fun case_3(): Boolean {
 * TESTCASE NUMBER: 4
 * UNEXPECTED BEHAVIOUR
 * ISSUES: KT-26386
+ * EXCEPTION: compiler
 */
 fun case_4(): Boolean? {
    contract { returns(null) implies case_4() }
@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.checkers

 import com.intellij.openapi.util.io.FileUtil
 import org.jetbrains.kotlin.TestExceptionsComparator
+import org.jetbrains.kotlin.TestsExceptionType
 import org.jetbrains.kotlin.config.LanguageVersionSettings
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.psi.KtFile
@@ -78,9 +79,8 @@ abstract class AbstractDiagnosticsTestSpec : AbstractDiagnosticsTest() {

        println(specTest)

-        val checkUnexpectedBehaviour: (Matcher?) -> Pair<Boolean, Set<Int>?> = l@{ matches ->
-            if (specTest.unexpectedBehavior) return@l Pair(true, null)
-            if (matches == null) return@l Pair(false, null)
+        val computeExceptionPoint: (Matcher?) -> Set<Int>? = l@{ matches ->
+            if (matches == null) return@l null

            val lineNumber = matches.group("lineNumber").toInt()
            val symbolNumber = matches.group("symbolNumber").toInt()
@@ -90,10 +90,11 @@ abstract class AbstractDiagnosticsTestSpec : AbstractDiagnosticsTest() {
            val testCases = specTest.cases.byRanges[filename]
            val testCasesWithSamePosition = testCases!!.floorEntry(exceptionPosition).value

-            return@l Pair(testCasesWithSamePosition.all { it.value.unexpectedBehavior }, testCasesWithSamePosition.keys.toSet())
+            return@l testCasesWithSamePosition.keys.toSet()
        }

-        TestExceptionsComparator(testDataFile).runAndCompareWithExpected(checkUnexpectedBehaviour) {
+        val exceptionsInCases = specTest.cases.byNumbers.entries.associate { it.key to it.value.exception }
+        TestExceptionsComparator(testDataFile).run(specTest.exception, exceptionsInCases, computeExceptionPoint) {
            super.analyzeAndCheck(testDataFile, files)
        }
    }
@@ -61,8 +61,12 @@ abstract class AbstractBlackBoxCodegenTestSpec : AbstractBlackBoxCodegenTest() {

        includeHelpers(wholeFile, files, specTest)

-        TestExceptionsComparator(wholeFile).runAndCompareWithExpected {
+        if (specTest.exception == null) {
            super.doMultiFileTest(wholeFile, files, javaFilesDir)
+        } else {
+            TestExceptionsComparator(wholeFile).run(specTest.exception) {
+                super.doMultiFileTest(wholeFile, files, javaFilesDir)
+            }
        }
    }
 }
@@ -22,16 +22,20 @@ abstract class AbstractParsingTestSpec : AbstractParsingTest() {

        println(specTest)

-        TestExceptionsComparator(file).runAndCompareWithExpected({ Pair(specTest.unexpectedBehavior, null) }) {
+        if (specTest.exception == null) {
            super.doParsingTest(filePath, CommonParser::testInfoFilter)
-
-            try {
-                val psiTestValidator = ParsingTestTypeValidator(myFile, File(filePath), specTest)
-                psiTestValidator.validatePathConsistency(testLinkedType)
-                psiTestValidator.validateTestType()
-            } catch (e: SpecTestValidationException) {
-                Assert.fail(e.description)
+        } else {
+            TestExceptionsComparator(file).run(specTest.exception) {
+                super.doParsingTest(filePath, CommonParser::testInfoFilter)
            }
        }
+
+        try {
+            val psiTestValidator = ParsingTestTypeValidator(myFile, File(filePath), specTest)
+            psiTestValidator.validatePathConsistency(testLinkedType)
+            psiTestValidator.validateTestType()
+        } catch (e: SpecTestValidationException) {
+            Assert.fail(e.description)
+        }
    }
 }
@@ -5,6 +5,7 @@

 package org.jetbrains.kotlin.spec

+import org.jetbrains.kotlin.TestsExceptionType
 import org.jetbrains.kotlin.spec.models.LinkedSpecTestFileInfoElementType
 import org.jetbrains.kotlin.spec.models.NotLinkedSpecTestFileInfoElementType
 import org.jetbrains.kotlin.spec.parsers.BasePatterns
@@ -74,7 +75,8 @@ data class SpecTestCase(
    var ranges: MutableList<IntRange>,
    var unexpectedBehavior: Boolean,
    var unspecifiedBehavior: Boolean,
-    val issues: MutableList<String>?
+    val issues: MutableList<String>?,
+    val exception: TestsExceptionType?
 )

 data class SpecTestCasesSet(
@@ -5,6 +5,7 @@

 package org.jetbrains.kotlin.spec.models

+import org.jetbrains.kotlin.TestsExceptionType
 import org.jetbrains.kotlin.spec.*
 import org.jetbrains.kotlin.spec.parsers.CommonPatterns
 import org.jetbrains.kotlin.spec.parsers.CommonPatterns.issuesPattern
@@ -22,7 +23,8 @@ enum class CommonInfoElementType(
    UNEXPECTED_BEHAVIOUR,
    ISSUES(valuePattern = issuesPattern),
    DISCUSSION,
-    NOTE
+    NOTE,
+    EXCEPTION
 }

 enum class CommonSpecTestFileInfoElementType(
@@ -52,7 +54,8 @@ abstract class AbstractSpecTest(
    val cases: SpecTestCasesSet,
    val unexpectedBehavior: Boolean,
    val issues: Set<String>,
-    val helpers: Set<String>?
+    val helpers: Set<String>?,
+    val exception: TestsExceptionType?
 ) {
    companion object {
        private fun issuesToString(issues: Set<String>) = issues.joinToString(", ") { CommonPatterns.ISSUE_TRACKER + it }
@@ -5,6 +5,7 @@

 package org.jetbrains.kotlin.spec.models

+import org.jetbrains.kotlin.TestsExceptionType
 import org.jetbrains.kotlin.spec.SpecTestCasesSet
 import org.jetbrains.kotlin.spec.SpecTestInfoElementType
 import org.jetbrains.kotlin.spec.TestArea
@@ -46,8 +47,9 @@ class LinkedSpecTest(
    unexpectedBehavior: Boolean,
    private val unspecifiedBehavior: Boolean,
    issues: Set<String>,
-    helpers: Set<String>?
-) : AbstractSpecTest(testArea, testType, place.sections, testNumber, description, cases, unexpectedBehavior, issues, helpers) {
+    helpers: Set<String>?,
+    exception: TestsExceptionType?
+) : AbstractSpecTest(testArea, testType, place.sections, testNumber, description, cases, unexpectedBehavior, issues, helpers, exception) {
    override fun checkPathConsistency(pathMatcher: Matcher) =
        testArea == TestArea.valueOf(pathMatcher.group("testArea").withUnderscores())
                && testType == TestType.fromValue(pathMatcher.group("testType"))!!
@@ -5,6 +5,7 @@

 package org.jetbrains.kotlin.spec.models

+import org.jetbrains.kotlin.TestsExceptionType
 import org.jetbrains.kotlin.spec.SpecTestCasesSet
 import org.jetbrains.kotlin.spec.SpecTestInfoElementType
 import org.jetbrains.kotlin.spec.TestArea
@@ -33,8 +34,9 @@ class NotLinkedSpecTest(
    cases: SpecTestCasesSet,
    unexpectedBehavior: Boolean,
    issues: Set<String>,
-    helpers: Set<String>?
-) : AbstractSpecTest(testArea, testType, sections, testNumber, description, cases, unexpectedBehavior, issues, helpers) {
+    helpers: Set<String>?,
+    exception: TestsExceptionType?
+) : AbstractSpecTest(testArea, testType, sections, testNumber, description, cases, unexpectedBehavior, issues, helpers, exception) {
    override fun checkPathConsistency(pathMatcher: Matcher) =
        testArea == TestArea.valueOf(pathMatcher.group("testArea").withUnderscores())
                && testType == TestType.fromValue(pathMatcher.group("testType"))!!
@@ -72,7 +72,8 @@ object CommonParser {
            parsedTestFile.unexpectedBehavior,
            LinkedSpecTestFileInfoElementType.UNSPECIFIED_BEHAVIOR in testInfoElements,
            parsedTestFile.issues,
-            parsedTestFile.helpers
+            parsedTestFile.helpers,
+            parsedTestFile.exception
        )
    }

@@ -90,7 +91,8 @@ object CommonParser {
            parsedTestFile.testCasesSet,
            parsedTestFile.unexpectedBehavior,
            parsedTestFile.issues,
-            parsedTestFile.helpers
+            parsedTestFile.helpers,
+            parsedTestFile.exception
        )
    }

@@ -5,6 +5,7 @@

 package org.jetbrains.kotlin.spec.parsers

+import org.jetbrains.kotlin.TestsExceptionType
 import org.jetbrains.kotlin.spec.*
 import org.jetbrains.kotlin.spec.models.CommonInfoElementType
 import org.jetbrains.kotlin.spec.models.LinkedSpecTestFileInfoElementType
@@ -74,7 +75,8 @@ fun parseTestCases(testFiles: TestFiles): SpecTestCasesSet {
                ranges = mutableListOf(range),
                unexpectedBehavior = caseInfoElements.contains(CommonInfoElementType.UNEXPECTED_BEHAVIOUR),
                unspecifiedBehavior = caseInfoElements.contains(LinkedSpecTestFileInfoElementType.UNSPECIFIED_BEHAVIOR),
-                issues = CommonParser.parseIssues(caseInfoElements[CommonInfoElementType.ISSUES]).toMutableList()
+                issues = CommonParser.parseIssues(caseInfoElements[CommonInfoElementType.ISSUES]).toMutableList(),
+                exception = caseInfoElements[CommonInfoElementType.EXCEPTION]?.content?.let { TestsExceptionType.fromValue(it) }
            ).save(testCasesSet.byNumbers, testCasesOfFile, testCasesByRangesOfFile, caseInfoElements)

            startFind = matcher.end() - nextDirective.length
@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.spec.parsers

 import com.intellij.openapi.util.io.FileUtil
+import org.jetbrains.kotlin.TestsExceptionType
 import org.jetbrains.kotlin.spec.*
 import org.jetbrains.kotlin.spec.models.CommonInfoElementType
 import org.jetbrains.kotlin.spec.models.CommonSpecTestFileInfoElementType
@@ -24,7 +25,8 @@ data class ParsedTestFile(
    val testCasesSet: SpecTestCasesSet,
    val unexpectedBehavior: Boolean,
    val issues: Set<String>,
-    val helpers: Set<String>?
+    val helpers: Set<String>?,
+    val exception: TestsExceptionType?
 )

 fun parseTestInfo(testFilePath: String, testFiles: TestFiles, linkedTestType: SpecTestLinkedType): ParsedTestFile {
@@ -54,6 +56,7 @@ fun parseTestInfo(testFilePath: String, testFiles: TestFiles, linkedTestType: Sp
        testCasesSet = parseTestCases(testFiles),
        unexpectedBehavior = testInfoElements.contains(CommonInfoElementType.UNEXPECTED_BEHAVIOUR),
        issues = CommonParser.parseIssues(testInfoElements[CommonInfoElementType.ISSUES]),
-        helpers = helpers
+        helpers = helpers,
+        exception = testInfoElements[CommonInfoElementType.EXCEPTION]?.content?.let { TestsExceptionType.fromValue(it) }
    )
 }