[Spec tests] Change metadata structure of implementation and spec tests

This commit is contained in:
anastasiia.spaseeva
2020-04-01 19:14:46 +03:00
parent 80cd26c9df
commit d32aca87d1
925 changed files with 2330 additions and 1657 deletions
@@ -12,7 +12,7 @@ import com.google.gson.JsonObject
import org.jetbrains.kotlin.spec.utils.models.LinkedSpecTest
import org.jetbrains.kotlin.spec.utils.models.SpecPlace
import org.jetbrains.kotlin.spec.utils.parsers.CommonParser
import org.jetbrains.kotlin.spec.utils.parsers.CommonParser.parseImplementationTest
import org.jetbrains.kotlin.spec.utils.parsers.LinkedSpecTestPatterns
import java.io.File
object TestsJsonMapGenerator {
@@ -52,18 +52,9 @@ object TestsJsonMapGenerator {
File("${GeneralConfiguration.SPEC_TESTDATA_PATH}/${testArea.testDataPath}/$LINKED_TESTS_PATH").walkTopDown()
.forEach testFiles@{ file ->
if (!file.isFile || file.extension != "kt" || file.name.endsWith(".fir.kt")) return@testFiles
val (specTest, _) = CommonParser.parseSpecTest(file.canonicalPath, mapOf("main.kt" to file.readText()))
if (specTest is LinkedSpecTest) {
val testInfo = getTestInfo(specTest)
val testInfoWithFilePath = getTestInfo(specTest, file)
testsMap.getOrCreateSpecTestObject(specTest.place, specTest.testArea, specTest.testType).add(testInfo)
specTest.relevantPlaces?.forEach {
testsMap.getOrCreateSpecTestObject(it, specTest.testArea, specTest.testType).add(testInfoWithFilePath)
}
collectInfoFromTests(testsMap, specTest, getTestInfo(specTest), getTestInfo(specTest, file))
}
}
}
@@ -71,22 +62,26 @@ object TestsJsonMapGenerator {
private fun collectInfoFromImplementationTests(testsMap: JsonObject) {
TestArea.values().forEach { testArea ->
val files = File("${GeneralConfiguration.TESTDATA_PATH}/${testArea.testDataPath}").walkTopDown()
for (file in files) {
if (!file.isFile || file.extension != "kt") continue
val parsedImplementationTest = parseImplementationTest(file, testArea) ?: continue
val relevantPlaces = parsedImplementationTest.relevantPlaces ?: listOf()
(relevantPlaces + parsedImplementationTest.place).forEach specPlaces@ { specPlace ->
val parsedAdditionalImplementationTest = parseImplementationTest(file, testArea) ?: return@specPlaces
testsMap.getOrCreateSpecTestObject(specPlace, testArea, parsedImplementationTest.testType).add(
getTestInfo(parsedAdditionalImplementationTest, file)
)
File("${GeneralConfiguration.TESTDATA_PATH}/${testArea.testDataPath}").walkTopDown()
.forEach testFiles@{ file ->
if (!file.isFile || file.extension != "kt") return@testFiles
if (!LinkedSpecTestPatterns.testInfoPattern.matcher(file.readText()).find())
return@testFiles
val (specTest, _) = CommonParser.parseImplTest(file.canonicalPath, mapOf("main.kt" to file.readText()))
collectInfoFromTests(testsMap, specTest, getTestInfo(specTest, file))
}
}
}
}
private fun collectInfoFromTests(
testsMap: JsonObject,
specTest: LinkedSpecTest,
testInfoForMainLink: JsonObject,
testInfoForRelevantLink: JsonObject = testInfoForMainLink
) {
testsMap.getOrCreateSpecTestObject(specTest.place, specTest.testArea, specTest.testType).add(testInfoForMainLink)
specTest.relevantPlaces?.forEach {
testsMap.getOrCreateSpecTestObject(it, specTest.testArea, specTest.testType).add(testInfoForRelevantLink)
}
}
@@ -9,7 +9,7 @@ import org.jetbrains.kotlin.TestsExceptionType
import org.jetbrains.kotlin.spec.utils.*
import org.jetbrains.kotlin.spec.utils.parsers.CommonPatterns
import org.jetbrains.kotlin.spec.utils.parsers.CommonPatterns.issuesPattern
import org.jetbrains.kotlin.spec.utils.parsers.LinkedSpecTestPatterns.relevantPlacesPattern
import org.jetbrains.kotlin.spec.utils.parsers.LinkedSpecTestPatterns.relevantLinksPattern
import org.jetbrains.kotlin.spec.utils.parsers.TestCasePatterns.testCaseNumberPattern
import java.util.regex.Matcher
import java.util.regex.Pattern
@@ -41,7 +41,7 @@ enum class SpecTestCaseInfoElementType(
override val required: Boolean = false
) : SpecTestInfoElementType {
TESTCASE_NUMBER(valuePattern = testCaseNumberPattern, required = true),
RELEVANT_PLACES(valuePattern = relevantPlacesPattern),
RELEVANT_PLACES(valuePattern = relevantLinksPattern),
UNSPECIFIED_BEHAVIOR
}
@@ -10,8 +10,8 @@ import org.jetbrains.kotlin.spec.utils.SpecTestCasesSet
import org.jetbrains.kotlin.spec.utils.SpecTestInfoElementType
import org.jetbrains.kotlin.spec.utils.TestArea
import org.jetbrains.kotlin.spec.utils.TestType
import org.jetbrains.kotlin.spec.utils.parsers.LinkedSpecTestPatterns.placePattern
import org.jetbrains.kotlin.spec.utils.parsers.LinkedSpecTestPatterns.relevantPlacesPattern
import org.jetbrains.kotlin.spec.utils.parsers.LinkedSpecTestPatterns.mainLinkPattern
import org.jetbrains.kotlin.spec.utils.parsers.LinkedSpecTestPatterns.relevantLinksPattern
import org.jetbrains.kotlin.spec.utils.parsers.CommonParser.withSpaces
import org.jetbrains.kotlin.spec.utils.parsers.CommonParser.withUnderscores
import org.jetbrains.kotlin.spec.utils.parsers.CommonParser.splitByPathSeparator
@@ -24,9 +24,10 @@ enum class LinkedSpecTestFileInfoElementType(
override val required: Boolean = false
) : SpecTestInfoElementType {
SPEC_VERSION(required = true),
PLACE(valuePattern = placePattern, required = true),
RELEVANT_PLACES(valuePattern = relevantPlacesPattern),
UNSPECIFIED_BEHAVIOR
MAIN_LINK(valuePattern = mainLinkPattern),
PRIMARY_LINKS(valuePattern = relevantLinksPattern),
SECONDARY_LINKS(valuePattern = relevantLinksPattern),
UNSPECIFIED_BEHAVIOR;
}
data class SpecPlace(
@@ -5,7 +5,10 @@
package org.jetbrains.kotlin.spec.utils.parsers
import org.jetbrains.kotlin.spec.utils.*
import org.jetbrains.kotlin.spec.utils.SpecTestInfoElementContent
import org.jetbrains.kotlin.spec.utils.SpecTestInfoElementType
import org.jetbrains.kotlin.spec.utils.SpecTestLinkedType
import org.jetbrains.kotlin.spec.utils.TestFiles
import org.jetbrains.kotlin.spec.utils.models.*
import org.jetbrains.kotlin.spec.utils.parsers.CommonPatterns.testInfoElementPattern
import org.jetbrains.kotlin.spec.utils.parsers.CommonPatterns.testPathBaseRegexTemplate
@@ -32,8 +35,8 @@ object CommonParser {
private fun parseBasePath(pathPartRegex: String, testFilePath: String) =
Pattern.compile(testPathBaseRegexTemplate.format(pathPartRegex)).matcher(testFilePath)
fun parsePath(pathPartRegex: String, testFilePath: String) =
Pattern.compile(testPathBaseRegexTemplate.format(pathPartRegex)).matcher(testFilePath)
fun parseImplTest(testFilePath: String, files: TestFiles): Pair<LinkedSpecTest, SpecTestLinkedType> =
Pair(parseLinkedSpecTest(testFilePath, files, isImplementationTest = true), SpecTestLinkedType.LINKED)
fun parseSpecTest(testFilePath: String, files: TestFiles) = when {
isPathMatched(LinkedSpecTestPatterns.pathPartRegex, testFilePath) ->
@@ -44,32 +47,6 @@ object CommonParser {
throw SpecTestValidationException(SpecTestValidationFailedReason.FILENAME_NOT_VALID)
}
fun parseImplementationTest(file: File, testArea: TestArea): LinkedSpecTest? {
val matcher = ImplementationTestPatterns.testInfoPattern.matcher(file.readText())
if (!matcher.find())
return null
val testType = TestType.fromValue(matcher.group("testType"))
?: throw SpecTestValidationException(SpecTestValidationFailedReason.TESTINFO_NOT_VALID)
val specVersion = matcher.group("specVersion")
val testSpecSentenceList = matcher.group("testSpecSentenceList")
val specSentenceListMatcher = ImplementationTestPatterns.relevantSpecSentencesPattern.matcher(testSpecSentenceList)
val specPlaces = mutableListOf<SpecPlace>()
while (specSentenceListMatcher.find()) {
specPlaces.add(
SpecPlace(
sections = specSentenceListMatcher.group("specSections").split(Regex(""",\s*""")),
paragraphNumber = specSentenceListMatcher.group("specParagraph").toInt(),
sentenceNumber = specSentenceListMatcher.group("specSentence").toInt()
)
)
}
return LinkedSpecTest.getInstanceForImplementationTest(specVersion, testArea, testType, specPlaces, file.nameWithoutExtension)
}
private fun createSpecPlace(placeMatcher: Matcher, basePlaceMatcher: Matcher = placeMatcher) =
SpecPlace(
placeMatcher.group("sections")?.splitByComma() ?: basePlaceMatcher.group("sections").splitByComma(),
@@ -77,26 +54,44 @@ object CommonParser {
placeMatcher.group("sentenceNumber").toInt()
)
fun parseLinkedSpecTest(testFilePath: String, testFiles: TestFiles): LinkedSpecTest {
val parsedTestFile = tryParseTestInfo(testFilePath, testFiles, SpecTestLinkedType.LINKED)
val testInfoElements = parsedTestFile.testInfoElements
val placeMatcher = testInfoElements[LinkedSpecTestFileInfoElementType.PLACE]!!.additionalMatcher!!
val relevantPlacesMatcher = testInfoElements[LinkedSpecTestFileInfoElementType.RELEVANT_PLACES]?.additionalMatcher
val relevantPlaces = relevantPlacesMatcher?.let {
mutableListOf<SpecPlace>().apply {
add(createSpecPlace(it, placeMatcher))
while (it.find()) {
add(createSpecPlace(it, placeMatcher))
}
private fun parseRelevantPlaces(
placesMatcher: Matcher?,
relevantPlaces: MutableList<SpecPlace>
) {
if (placesMatcher == null)
return
placesMatcher?.let {
relevantPlaces.add(createSpecPlace(it, placesMatcher))
while (it.find()) {
relevantPlaces.add(createSpecPlace(it, placesMatcher))
}
}
}
fun parseLinkedSpecTest(testFilePath: String, testFiles: TestFiles, isImplementationTest: Boolean = false): LinkedSpecTest {
val relevantAndAlternativePlaces = mutableListOf<SpecPlace>()
val parsedTestFile = tryParseTestInfo(testFilePath, testFiles, SpecTestLinkedType.LINKED, isImplementationTest)
val testInfoElements = parsedTestFile.testInfoElements
parseRelevantPlaces(
testInfoElements[LinkedSpecTestFileInfoElementType.PRIMARY_LINKS]?.additionalMatcher,
relevantAndAlternativePlaces
)
parseRelevantPlaces(
testInfoElements[LinkedSpecTestFileInfoElementType.SECONDARY_LINKS]?.additionalMatcher,
relevantAndAlternativePlaces
)
val placeMatcher = testInfoElements[LinkedSpecTestFileInfoElementType.MAIN_LINK]?.additionalMatcher
return LinkedSpecTest(
testInfoElements[LinkedSpecTestFileInfoElementType.SPEC_VERSION]!!.content,
parsedTestFile.testArea,
parsedTestFile.testType,
createSpecPlace(placeMatcher),
relevantPlaces,
if (placeMatcher != null) createSpecPlace(placeMatcher) else relevantAndAlternativePlaces.first(),
relevantAndAlternativePlaces,
parsedTestFile.testNumber,
parsedTestFile.testDescription,
parsedTestFile.testCasesSet,
@@ -136,34 +131,59 @@ object CommonParser {
while (testInfoElementMatcher.find()) {
val testInfoOriginalElementName = testInfoElementMatcher.group("name")
val testInfoElementName = rules.find {
it as Enum<*>
it.name == testInfoOriginalElementName.withUnderscores()
} ?: throw SpecTestValidationException(
SpecTestValidationFailedReason.TESTINFO_NOT_VALID,
"Unknown '$testInfoOriginalElementName' test info element name."
)
val testInfoElementValue: String?
testInfoElementValue = if (testInfoOriginalElementName == "RELEVANT PLACES") {
val relevantPlacesMatcher = LinkedSpecTestPatterns.relevantPlaces.matcher(rawElements)
if (relevantPlacesMatcher.find()) {
relevantPlacesMatcher.group("places")
} else throw Exception("Relevant link is incorrect")
} else {
testInfoElementMatcher.group("value")
}
val testInfoElementValue = parseTestInfoElementValue(testInfoOriginalElementName, testInfoElementMatcher, rawElements)
val testInfoElementName = parseSpecTestInfoElementType(rules, testInfoOriginalElementName)
val testInfoElementValueMatcher = testInfoElementName.valuePattern?.matcher(testInfoElementValue)
if (testInfoElementValueMatcher != null && !testInfoElementValueMatcher.find())
throw SpecTestValidationException(
SpecTestValidationFailedReason.TESTINFO_NOT_VALID,
"'$testInfoElementValue' in '$testInfoElementName' is not parsed."
)
checkTestInfoElementIsCorrect(testInfoElementValueMatcher, testInfoElementName, testInfoElementValue)
testInfoElementsMap[testInfoElementName] =
SpecTestInfoElementContent(testInfoElementValue ?: "", testInfoElementValueMatcher)
}
checkRulesObservance(rules, testInfoElementsMap)
return testInfoElementsMap
}
private fun parseTestInfoElementValue(
testInfoOriginalElementName: String?,
testInfoElementMatcher: Matcher,
rawElements: String,
) = when (testInfoOriginalElementName) {
LinkedSpecTestPatterns.PRIMARY_LINKS ->
groupRelevantAndAlternativePlaces(LinkedSpecTestPatterns.primaryLinks, rawElements, testInfoOriginalElementName)
LinkedSpecTestPatterns.SECONDARY_LINKS ->
groupRelevantAndAlternativePlaces(LinkedSpecTestPatterns.secondaryLinks, rawElements, testInfoOriginalElementName)
else ->
testInfoElementMatcher.group("value")
}
private fun parseSpecTestInfoElementType(
rules: Array<SpecTestInfoElementType>,
testInfoOriginalElementName: String
) = rules.find {
it as Enum<*>
it.name == testInfoOriginalElementName.withUnderscores()
} ?: throw SpecTestValidationException(
SpecTestValidationFailedReason.TESTINFO_NOT_VALID,
"Unknown '$testInfoOriginalElementName' test info element name."
)
private fun checkTestInfoElementIsCorrect(
testInfoElementValueMatcher: Matcher?,
testInfoElementName: SpecTestInfoElementType,
testInfoElementValue: String?
) {
if (testInfoElementValueMatcher != null && !testInfoElementValueMatcher.find())
throw SpecTestValidationException(
SpecTestValidationFailedReason.TESTINFO_NOT_VALID,
"'$testInfoElementValue' in '$testInfoElementName' is not parsed."
)
}
private fun checkRulesObservance(
rules: Array<SpecTestInfoElementType>,
testInfoElementsMap: MutableMap<SpecTestInfoElementType, SpecTestInfoElementContent>
) {
rules.forEach {
if (it.required && !testInfoElementsMap.contains(it)) {
throw SpecTestValidationException(
@@ -172,8 +192,14 @@ object CommonParser {
)
}
}
}
return testInfoElementsMap
private fun groupRelevantAndAlternativePlaces(placesPattern: Pattern, rawElements: String, linkType: String): String {
val placesMatcher = placesPattern.matcher(rawElements)
if (placesMatcher.find()) {
return placesMatcher.group("places")
} else throw Exception("$linkType link is incorrect")
}
fun testInfoFilter(fileContent: String) =
@@ -21,6 +21,7 @@ import org.jetbrains.kotlin.spec.utils.parsers.CommonPatterns.sectionsInPathRege
import org.jetbrains.kotlin.spec.utils.parsers.CommonPatterns.testAreaRegex
import org.jetbrains.kotlin.spec.utils.parsers.CommonPatterns.testPathRegexTemplate
import org.jetbrains.kotlin.spec.utils.parsers.CommonPatterns.testTypeRegex
import org.jetbrains.kotlin.spec.utils.parsers.CommonPatterns.ws
import java.io.File
import java.util.regex.Pattern
@@ -34,6 +35,7 @@ object CommonPatterns {
val ls: String = System.lineSeparator()
val ps: String = Pattern.quote(File.separator)
val ws = """\s*"""
val directiveRegex =
"""${SINGLE_LINE_COMMENT_REGEX.format("""[\w\s]+:""")}|${MULTILINE_COMMENT_REGEX.format(""" $ASTERISK_REGEX [\w\s]+:[\s\S]*?""")}"""
@@ -67,6 +69,9 @@ object NotLinkedSpecTestPatterns : BasePatterns {
object LinkedSpecTestPatterns : BasePatterns {
private const val FILENAME_REGEX = """(?<sentenceNumber>$INTEGER_REGEX)\.(?<testNumber>$INTEGER_REGEX)(?:\.fir)?\.kt"""
const val PRIMARY_LINKS = "PRIMARY LINKS"
const val SECONDARY_LINKS = "SECONDARY LINKS"
override val pathPartRegex =
"""${SpecTestLinkedType.LINKED.testDataPath}$ps$sectionsInPathRegex${ps}p-(?<paragraphNumber>$INTEGER_REGEX)"""
override val testPathPattern: Pattern =
@@ -74,16 +79,17 @@ object LinkedSpecTestPatterns : BasePatterns {
override val testInfoPattern: Pattern =
Pattern.compile(MULTILINE_COMMENT_REGEX.format(""" $ASTERISK_REGEX KOTLIN $testAreaRegex SPEC TEST \($testTypeRegex\)\n(?<infoElements>[\s\S]*?\n)"""))
val placePattern: Pattern =
val mainLinkPattern: Pattern =
Pattern.compile("""(?<sections>$SECTIONS_IN_FILE_REGEX) -> paragraph (?<paragraphNumber>$INTEGER_REGEX) -> sentence (?<sentenceNumber>$INTEGER_REGEX)""")
val relevantPlacesPattern: Pattern =
val relevantLinksPattern: Pattern =
Pattern.compile("""(( $ASTERISK_REGEX )?\s*((?<sections>$SECTIONS_IN_FILE_REGEX) -> )?(paragraph (?<paragraphNumber>$INTEGER_REGEX) -> )?sentence (?<sentenceNumber>$INTEGER_REGEX))+""")
private val relevantPlaceRegex =
private val linkRegex =
Regex("""(( $ASTERISK_REGEX )?\s*($SECTIONS_IN_FILE_REGEX -> )?(paragraph $INTEGER_REGEX -> )?sentence $INTEGER_REGEX)""")
val relevantPlaces: Pattern = Pattern.compile("""RELEVANT PLACES: (?<places>(${relevantPlaceRegex}(\s)*\n)+)""")
val primaryLinks: Pattern = Pattern.compile("""$PRIMARY_LINKS$ws:$ws(?<places>(${linkRegex}(\s)*\n)+)""")
val secondaryLinks: Pattern = Pattern.compile("""$SECONDARY_LINKS$ws:$ws(?<places>(${linkRegex}(\s)*\n)+)""")
}
object TestCasePatterns {
@@ -102,11 +108,4 @@ object TestCasePatterns {
val testCaseInfoPattern: Pattern = Pattern.compile("(?:$testCaseInfoSingleLineRegex)|(?:$testCaseInfoMultilineRegex)")
val testCaseNumberPattern: Pattern = Pattern.compile("""([1-9]\d*)(,\s*[1-9]\d*)*""")
}
object ImplementationTestPatterns {
val testInfoPattern: Pattern =
Pattern.compile(MULTILINE_COMMENT_REGEX.format("""\*\s+RELEVANT SPEC SENTENCES \(spec version: (?<specVersion>\d+\.[0-9]\d*\-[0-9]\d*), test type: (?<testType>pos|neg)\):(?<testSpecSentenceList>(\n\s+\*\s+-\s+.*?)+)"""))
val relevantSpecSentencesPattern: Pattern =
Pattern.compile("""\n\s+\*\s+-\s+(?<specSections>.*?) -> paragraph (?<specParagraph>[1-9]\d*) -> sentence (?<specSentence>[1-9]\d*)""")
}
}
@@ -11,10 +11,10 @@ import org.jetbrains.kotlin.spec.utils.*
import org.jetbrains.kotlin.spec.utils.models.CommonInfoElementType
import org.jetbrains.kotlin.spec.utils.models.CommonSpecTestFileInfoElementType
import org.jetbrains.kotlin.spec.utils.models.SpecTestInfoElements
import org.jetbrains.kotlin.spec.utils.validators.SpecTestValidationException
import org.jetbrains.kotlin.spec.utils.validators.SpecTestValidationFailedReason
import org.jetbrains.kotlin.spec.utils.parsers.CommonParser.splitByComma
import org.jetbrains.kotlin.spec.utils.parsers.CommonParser.withUnderscores
import org.jetbrains.kotlin.spec.utils.validators.SpecTestValidationException
import org.jetbrains.kotlin.spec.utils.validators.SpecTestValidationFailedReason
import java.io.File
data class ParsedTestFile(
@@ -62,10 +62,54 @@ private fun parseTestInfo(testFilePath: String, testFiles: TestFiles, linkedTest
)
}
fun tryParseTestInfo(testFilePath: String, testFiles: TestFiles, linkedTestType: SpecTestLinkedType): ParsedTestFile {
private fun parseImplementationTestInfo(testFilePath: String, linkedTestType: SpecTestLinkedType): ParsedTestFile {
val patterns = linkedTestType.patterns.value
val testInfoByContentMatcher = patterns.testInfoPattern.matcher(FileUtil.loadFile(File(testFilePath), true))
if (!testInfoByContentMatcher.find()) {
throw SpecTestValidationException(SpecTestValidationFailedReason.TESTINFO_NOT_VALID)
}
val testInfoElements = CommonParser.parseTestInfoElements(
arrayOf(
*CommonInfoElementType.values(),
*linkedTestType.infoElements.value
),
testInfoByContentMatcher.group("infoElements")
)
val fileNameWithoutExtension = testFilePath.split("/").last().replace(".kt", "")
return ParsedTestFile(
testArea = TestArea.valueOf(testInfoByContentMatcher.group("testArea").withUnderscores()),
testType = TestType.valueOf(testInfoByContentMatcher.group("testType")),
testNumber = testInfoElements[CommonSpecTestFileInfoElementType.NUMBER]?.content?.toInt() ?: 0,
testDescription = fileNameWithoutExtension.toUpperCase()[0] + fileNameWithoutExtension.substring(1)
.replace(Regex("""([A-Z])"""), " $1").toLowerCase(),
testInfoElements = testInfoElements,
testCasesSet = SpecTestCasesSet(mutableMapOf(), mutableMapOf(), mutableMapOf()), //todo
unexpectedBehavior = testInfoElements.contains(CommonInfoElementType.UNEXPECTED_BEHAVIOUR),
issues = CommonParser.parseIssues(testInfoElements[CommonInfoElementType.ISSUES]),
helpers = testInfoElements[CommonSpecTestFileInfoElementType.HELPERS]?.content?.splitByComma()?.toSet(),
exception = testInfoElements[CommonInfoElementType.EXCEPTION]?.content?.let { TestsExceptionType.fromValue(it) }
)
}
fun tryParseTestInfo(
testFilePath: String,
testFiles: TestFiles,
linkedTestType: SpecTestLinkedType,
isImplementationTest: Boolean = false
): ParsedTestFile {
try {
return parseTestInfo(testFilePath, testFiles, linkedTestType)
return if (isImplementationTest)
parseImplementationTestInfo(testFilePath, linkedTestType)
else
parseTestInfo(testFilePath, testFiles, linkedTestType)
} catch (e: Exception) {
error("Wrong format of file:\nfile://$testFilePath \n${e.message}")
}
}