[Spec tests] fix test parser exceptions handler

2019-12-12 11:20:04 +03:00
parent fdef51e8b7
commit 8187405dbd
17 changed files with 63 additions and 51 deletions
@@ -6,7 +6,7 @@
 * SPEC VERSION: 0.1-213
 * PLACE: expressions, built-in-types-and-their-semantics, kotlin.nothing-1 -> paragraph 1 -> sentence 1
 * NUMBER: 2
- * DESCRIPTION:
+ * DESCRIPTION: check kotlin.Nothing type
 */

 fun box(): String {
@@ -6,7 +6,7 @@
 * SPEC VERSION: 0.1-213
 * PLACE: expressions, built-in-types-and-their-semantics, kotlin.nothing-1 -> paragraph 1 -> sentence 1
 * NUMBER: 3
- * DESCRIPTION:
+ * DESCRIPTION: check kotlin.Nothing type
 */

 fun box(): String {
@@ -6,7 +6,7 @@
 * SPEC VERSION: 0.1-213
 * PLACE: expressions, built-in-types-and-their-semantics, kotlin.nothing-1 -> paragraph 1 -> sentence 1
 * NUMBER: 4
- * DESCRIPTION:
+ * DESCRIPTION: check kotlin.Nothing type
 */


@@ -6,7 +6,7 @@
 * SPEC VERSION: 0.1-213
 * PLACE: expressions, built-in-types-and-their-semantics, kotlin.nothing-1 -> paragraph 1 -> sentence 1
 * NUMBER: 5
- * DESCRIPTION:
+ * DESCRIPTION: check kotlin.Nothing type
 */


@@ -6,7 +6,7 @@
 * SPEC VERSION: 0.1-213
 * PLACE: expressions, built-in-types-and-their-semantics, kotlin.nothing-1 -> paragraph 1 -> sentence 1
 * NUMBER: 6
- * DESCRIPTION:
+ * DESCRIPTION: check kotlin.Nothing type
 */


@@ -7,7 +7,7 @@
 * SPEC VERSION: 0.1-213
 * PLACE: expressions, built-in-types-and-their-semantics, kotlin.nothing-1 -> paragraph 1 -> sentence 1
 * NUMBER: 7
- * DESCRIPTION:
+ * DESCRIPTION: check kotlin.Nothing type
 */

 import java.util.*
@@ -6,7 +6,7 @@
 * SPEC VERSION: 0.1-213
 * PLACE: expressions, built-in-types-and-their-semantics, kotlin.nothing-1 -> paragraph 1 -> sentence 1
 * NUMBER: 8
- * DESCRIPTION:
+ * DESCRIPTION: check kotlin.Nothing type
 */

 fun box(): String {
@@ -7,7 +7,7 @@
 * SPEC VERSION: 0.1-213
 * PLACE: expressions, built-in-types-and-their-semantics, kotlin.nothing-1 -> paragraph 1 -> sentence 1
 * NUMBER: 9
- * DESCRIPTION:
+ * DESCRIPTION: check kotlin.Nothing type
 */

 fun box(): String {
@@ -5,25 +5,25 @@
        {
          "specVersion": "0.1-213",
          "casesNumber": 0,
-          "description": "",
+          "description": "check kotlin.Nothing type",
          "unexpectedBehaviour": false
        },
        {
          "specVersion": "0.1-213",
          "casesNumber": 0,
-          "description": "",
+          "description": "check kotlin.Nothing type",
          "unexpectedBehaviour": false
        },
        {
          "specVersion": "0.1-213",
          "casesNumber": 0,
-          "description": "",
+          "description": "check kotlin.Nothing type",
          "unexpectedBehaviour": false
        },
        {
          "specVersion": "0.1-213",
          "casesNumber": 0,
-          "description": "",
+          "description": "check kotlin.Nothing type",
          "unexpectedBehaviour": false
        },
        {
@@ -35,25 +35,25 @@
        {
          "specVersion": "0.1-213",
          "casesNumber": 0,
-          "description": "",
+          "description": "check kotlin.Nothing type",
          "unexpectedBehaviour": false
        },
        {
          "specVersion": "0.1-213",
          "casesNumber": 0,
-          "description": "",
+          "description": "check kotlin.Nothing type",
          "unexpectedBehaviour": false
        },
        {
          "specVersion": "0.1-213",
          "casesNumber": 0,
-          "description": "",
+          "description": "check kotlin.Nothing type",
          "unexpectedBehaviour": false
        },
        {
          "specVersion": "0.1-213",
          "casesNumber": 0,
-          "description": "",
+          "description": "check kotlin.Nothing type",
          "unexpectedBehaviour": false
        },
        {
@@ -6,7 +6,7 @@
 * SPEC VERSION: 0.1-213
 * PLACE: expressions, cast-expression -> paragraph 1 -> sentence 1
 * NUMBER: 1
- * DESCRIPTION: check of the cast operators as or as?
+ * DESCRIPTION: check of the cast operators as or as?
 */

 fun box(): String {
@@ -5,7 +5,7 @@
        {
          "specVersion": "0.1-213",
          "casesNumber": 0,
-          "description": " check of the cast operators as or as?",
+          "description": "check of the cast operators as or as?",
          "unexpectedBehaviour": false
        }
      ]
@@ -9,7 +9,6 @@
 * PLACE: control--and-data-flow-analysis, control-flow-graph, expressions-1, conditional-expressions -> paragraph 1 -> sentence 1
 * NUMBER: 1
 * DESCRIPTION: check if-expressions must have both branches.
- * HELPERS:
 */

 // TESTCASE NUMBER: 1
@@ -5,7 +5,7 @@
        {
          "specVersion": "0.1-213",
          "casesNumber": 2,
-          "description": "* HELPERS: checkType",
+          "description": "check any if-statement in kotlin may be trivially turned into such an expression by replacing the missing branch with a kotlin.Unit object expression.",
          "unexpectedBehaviour": false
        }
      ],
@@ -13,7 +13,7 @@
        {
          "specVersion": "0.1-213",
          "casesNumber": 2,
-          "description": "* HELPERS:",
+          "description": "check if-expressions must have both branches.",
          "unexpectedBehaviour": false
        }
      ]
@@ -23,7 +23,7 @@
        {
          "specVersion": "0.1-213",
          "casesNumber": 2,
-          "description": "* HELPERS: checkType, functions",
+          "description": "check any if-statement in kotlin may be trivially turned into such an expression by replacing the missing branch with a kotlin.Unit object expression.",
          "unexpectedBehaviour": false
        }
      ],
@@ -31,7 +31,7 @@
        {
          "specVersion": "0.1-213",
          "casesNumber": 3,
-          "description": "* HELPERS: checkType, functions",
+          "description": "check if-expressions must have both branches.",
          "unexpectedBehaviour": false
        },
        {
@@ -39,7 +39,7 @@ object CommonPatterns {
        """${SINGLE_LINE_COMMENT_REGEX.format("""[\w\s]+:""")}|${MULTILINE_COMMENT_REGEX.format(""" $ASTERISK_REGEX [\w\s]+:[\s\S]*?""")}"""
    val testAreaRegex = """(?<testArea>${TestArea.joinedValues})"""
    val testTypeRegex = """(?<testType>${TestType.joinedValues})"""
-    val testInfoElementPattern: Pattern = Pattern.compile("""(?: \* )?(?<name>[A-Z ]+?)(?::\s*(?<value>.*?))?\n""")
+    val testInfoElementPattern: Pattern = Pattern.compile("""(?: \* )?(?<name>[A-Z ]+?)(?::[ ]?(?<value>.*?))?\n""")
    val testPathBaseRegexTemplate = """^.*?$ps(?<testArea>diagnostics|psi|(?:codegen${ps}box))$ps%s"""
    val testPathRegexTemplate = """$testPathBaseRegexTemplate$ps(?<testType>pos|neg)$ps%s$"""
    val issuesPattern: Pattern = Pattern.compile("""(KT-[1-9]\d*)(,\s*KT-[1-9]\d*)*""")
@@ -13,6 +13,8 @@ import org.jetbrains.kotlin.spec.utils.models.SpecTestCaseInfoElementType
 import org.jetbrains.kotlin.spec.utils.models.SpecTestInfoElements
 import org.jetbrains.kotlin.spec.utils.parsers.TestCasePatterns.testCaseInfoPattern
 import org.jetbrains.kotlin.spec.utils.parsers.CommonParser.splitByComma
+import org.jetbrains.kotlin.spec.utils.validators.SpecTestValidationException
+import org.jetbrains.kotlin.spec.utils.validators.SpecTestValidationFailedReason
 import java.util.*

 private operator fun SpecTestCase.plusAssign(addTestCase: SpecTestCase) {
@@ -30,7 +32,13 @@ private fun SpecTestCase.save(
    caseInfoElements: SpecTestInfoElements<SpecTestInfoElementType>
 ) {
    val testCaseNumbers =
-        caseInfoElements[SpecTestCaseInfoElementType.TESTCASE_NUMBER]!!.content.splitByComma().map { it.trim().toInt() }
+        caseInfoElements[SpecTestCaseInfoElementType.TESTCASE_NUMBER]!!.content.splitByComma().map {
+            it.trim().toIntOrNull()
+                ?: throw SpecTestValidationException(
+                    SpecTestValidationFailedReason.TEST_CASE_NUMBER_FORMAT,
+                    "impossible to parse number '${it.trim()}'"
+                )
+        }
    val startPosition = this.ranges[0].first

    testCaseNumbers.forEach { testCaseNumber ->
@@ -31,33 +31,37 @@ data class ParsedTestFile(
 )

 fun parseTestInfo(testFilePath: String, testFiles: TestFiles, linkedTestType: SpecTestLinkedType): ParsedTestFile {
-    val patterns = linkedTestType.patterns.value
-    val testInfoByFilenameMatcher = patterns.testPathPattern.matcher(testFilePath)
+    try {
+        val patterns = linkedTestType.patterns.value
+        val testInfoByFilenameMatcher = patterns.testPathPattern.matcher(testFilePath)

-    if (!testInfoByFilenameMatcher.find())
-        throw SpecTestValidationException(SpecTestValidationFailedReason.FILENAME_NOT_VALID)
+        if (!testInfoByFilenameMatcher.find())
+            throw SpecTestValidationException(SpecTestValidationFailedReason.FILENAME_NOT_VALID)

-    val testInfoByContentMatcher = patterns.testInfoPattern.matcher(FileUtil.loadFile(File(testFilePath), true))
+        val testInfoByContentMatcher = patterns.testInfoPattern.matcher(FileUtil.loadFile(File(testFilePath), true))

-    if (!testInfoByContentMatcher.find())
-        throw SpecTestValidationException(SpecTestValidationFailedReason.TESTINFO_NOT_VALID)
+        if (!testInfoByContentMatcher.find())
+            throw SpecTestValidationException(SpecTestValidationFailedReason.TESTINFO_NOT_VALID)

-    val testInfoElements = CommonParser.parseTestInfoElements(
-        arrayOf(*CommonInfoElementType.values(), *CommonSpecTestFileInfoElementType.values(), *linkedTestType.infoElements.value),
-        testInfoByContentMatcher.group("infoElements")
-    )
-    val helpers = testInfoElements[CommonSpecTestFileInfoElementType.HELPERS]?.content?.splitByComma()?.toSet()
+        val testInfoElements = CommonParser.parseTestInfoElements(
+            arrayOf(*CommonInfoElementType.values(), *CommonSpecTestFileInfoElementType.values(), *linkedTestType.infoElements.value),
+            testInfoByContentMatcher.group("infoElements")
+        )
+        val helpers = testInfoElements[CommonSpecTestFileInfoElementType.HELPERS]?.content?.splitByComma()?.toSet()

-    return ParsedTestFile(
-        testArea = TestArea.valueOf(testInfoByContentMatcher.group("testArea").withUnderscores()),
-        testType = TestType.valueOf(testInfoByContentMatcher.group("testType")),
-        testNumber = testInfoElements[CommonSpecTestFileInfoElementType.NUMBER]!!.content.toInt(),
-        testDescription = testInfoElements[CommonSpecTestFileInfoElementType.DESCRIPTION]!!.content,
-        testInfoElements = testInfoElements,
-        testCasesSet = parseTestCases(testFiles),
-        unexpectedBehavior = testInfoElements.contains(CommonInfoElementType.UNEXPECTED_BEHAVIOUR),
-        issues = CommonParser.parseIssues(testInfoElements[CommonInfoElementType.ISSUES]),
-        helpers = helpers,
-        exception = testInfoElements[CommonInfoElementType.EXCEPTION]?.content?.let { TestsExceptionType.fromValue(it) }
-    )
+        return ParsedTestFile(
+            testArea = TestArea.valueOf(testInfoByContentMatcher.group("testArea").withUnderscores()),
+            testType = TestType.valueOf(testInfoByContentMatcher.group("testType")),
+            testNumber = testInfoElements[CommonSpecTestFileInfoElementType.NUMBER]!!.content.toInt(),
+            testDescription = testInfoElements[CommonSpecTestFileInfoElementType.DESCRIPTION]!!.content,
+            testInfoElements = testInfoElements,
+            testCasesSet = parseTestCases(testFiles),
+            unexpectedBehavior = testInfoElements.contains(CommonInfoElementType.UNEXPECTED_BEHAVIOUR),
+            issues = CommonParser.parseIssues(testInfoElements[CommonInfoElementType.ISSUES]),
+            helpers = helpers,
+            exception = testInfoElements[CommonInfoElementType.EXCEPTION]?.content?.let { TestsExceptionType.fromValue(it) }
+        )
+    } catch (e: Exception) {
+        error("Wrong format of file:\nfile://$testFilePath \n${e.message}")
+    }
 }
@@ -23,10 +23,11 @@ enum class SpecTestValidationFailedReason(val description: String) {
    UNKNOWN_FRONTEND_EXCEPTION("Unknown frontend exception. Manual analysis is required."),
    UNMATCHED_FRONTEND_EXCEPTION("Unmatched frontend exception. Manual analysis is required."),
    UNKNOWN("Unknown validation error."),
-    INCONSISTENT_REASONS("Inconsistent fail reasons: all test cases should have one fail reason within one test.")
+    INCONSISTENT_REASONS("Inconsistent fail reasons: all test cases should have one fail reason within one test."),
+    TEST_CASE_NUMBER_FORMAT("Wrong format of testcase number: only integers are allowed.")
 }

-class SpecTestValidationException(reason: SpecTestValidationFailedReason, details: String = "") : Exception() {
+class SpecTestValidationException(reason: SpecTestValidationFailedReason, details: String = "") : Exception("${reason.description} \nDetails: $details") {
    val description = "${reason.description} $details"
 }