azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement tests exceptions fixation mechanism

Browse Source
This commit is contained in:
victor.petukhov
2018-11-29 18:13:46 +03:00
parent 8da9bdf928
commit b9d1825765
14 changed files with 200 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/tests-common/tests/org/jetbrains/kotlin/TestExceptionsComparator.kt
+107
View File
@@ -0,0 +1,107 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import org.junit.Assert
import org.jetbrains.kotlin.test.KotlinTestUtils
import java.io.File
import java.lang.Error
import java.util.regex.Matcher
import java.util.regex.Pattern
enum class TestsExceptionFilePostfix(val text: String) {
COMPILER_ERROR("compiler"),
COMPILETIME_ERROR("compiletime"),
RUNTIME_ERROR("runtime"),
INFRASTRUCTURE_ERROR("infrastructure")
}
sealed class TestsError(val postfix: TestsExceptionFilePostfix) : Error() {
abstract val original: Throwable
override fun toString() = original.toString()
}
class TestsCompilerError(override val original: Throwable) : TestsError(TestsExceptionFilePostfix.COMPILER_ERROR)
class TestsInfrastructureError(override val original: Throwable) : TestsError(TestsExceptionFilePostfix.INFRASTRUCTURE_ERROR)
class TestsCompiletimeError(override val original: Throwable) : TestsError(TestsExceptionFilePostfix.COMPILETIME_ERROR)
class TestsRuntimeError(override val original: Throwable) : TestsError(TestsExceptionFilePostfix.RUNTIME_ERROR)
private enum class ExceptionType {
ANALYZING_EXPRESSION,
UNKNOWN
}
class TestExceptionsComparator(wholeFile: File) {
companion object {
private const val EXCEPTIONS_FILE_PREFIX = "exceptions"
private val exceptionMessagePatterns = mapOf(
ExceptionType.ANALYZING_EXPRESSION to
Pattern.compile("""Exception while analyzing expression at \((?<lineNumber>\d+),(?<symbolNumber>\d+)\) in /(?<filename>.*?)$""")
)
private val ls = System.lineSeparator()
}
private val filePathPrefix = "${wholeFile.parent}/${wholeFile.nameWithoutExtension}.$EXCEPTIONS_FILE_PREFIX"
private fun analyze(e: Throwable): Matcher? {
for ((_, pattern) in exceptionMessagePatterns) {
if (e.message == null) continue
val matches = pattern.matcher(e.message)
if (matches.find()) return matches
}
return null
}
private fun getExceptionInfo(e: TestsError, cases: Set<Int>?): String {
val casesAsString = cases?.run { "CASES: " + joinToString() + ls } ?: ""
return when (e) {
is TestsRuntimeError ->
(e.original.cause ?: e.original).run {
casesAsString + toString() + stackTrace[0]?.let { ls + it }
}
is TestsCompilerError, is TestsInfrastructureError -> casesAsString + (e.original.cause ?: e.original).toString()
is TestsCompiletimeError -> throw e.original
}
}
private fun validateExistingExceptionFiles(e: TestsError?) {
val postfixesOfFilesToCheck = TestsExceptionFilePostfix.values().toMutableSet().filter { it != e?.postfix }
postfixesOfFilesToCheck.forEach {
if (File("$filePathPrefix.${it.text}.txt").exists())
Assert.fail("No $it, but file $filePathPrefix.${it.text}.txt exists.")
}
}
fun runAndCompareWithExpected(checkUnexpectedBehaviour: ((Matcher?) -> Pair<Boolean, Set<Int>?>)? = null, runnable: () -> Unit) {
try {
runnable()
} catch (e: TestsError) {
val exceptionInfo = analyze(e.original)
val unexpectedBehaviourCheckResult = checkUnexpectedBehaviour?.invoke(exceptionInfo)
if (e is TestsCompilerError && unexpectedBehaviourCheckResult?.first == false)
throw e.original
val exceptionsFile = File("$filePathPrefix.${e.postfix.text}.txt")
try {
KotlinTestUtils.assertEqualsToFile(exceptionsFile, getExceptionInfo(e, unexpectedBehaviourCheckResult?.second))
} catch (t: AssertionError) {
e.original.printStackTrace()
throw t
}
e.original.printStackTrace()
validateExistingExceptionFiles(e)
return
}
validateExistingExceptionFiles(null)
}
}
compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsTest.kt
+11
View File
@@ -10,6 +10,7 @@ import com.intellij.openapi.util.text.StringUtil
import com.intellij.psi.PsiElement
import com.intellij.psi.search.GlobalSearchScope
import junit.framework.TestCase
import org.jetbrains.kotlin.TestsCompilerError
import org.jetbrains.kotlin.analyzer.AnalysisResult
import org.jetbrains.kotlin.analyzer.common.CommonAnalyzerFacade
import org.jetbrains.kotlin.builtins.jvm.JvmBuiltIns
@@ -64,6 +65,16 @@ import java.util.regex.Pattern
abstract class AbstractDiagnosticsTest : BaseDiagnosticsTest() {
override fun analyzeAndCheck(testDataFile: File, files: List<TestFile>) {
try {
analyzeAndCheckUnhandled(testDataFile, files)
} catch (t: AssertionError) {
throw t
} catch (t: Throwable) {
throw TestsCompilerError(t)
}
}
private fun analyzeAndCheckUnhandled(testDataFile: File, files: List<TestFile>) {
val groupedByModule = files.groupBy(TestFile::module)
var lazyOperationsLog: LazyOperationsLog? = null
compiler/tests-common/tests/org/jetbrains/kotlin/checkers/BaseDiagnosticsTest.kt
+1 -1
View File
@@ -121,7 +121,7 @@ abstract class BaseDiagnosticsTest : KotlinMultiFileTestWithJava<TestModule, Tes
inner class TestFile(
val module: TestModule?,
fileName: String,
val fileName: String,
textWithMarkers: String,
val directives: Map<String, String>
) {
compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractBlackBoxCodegenTest.java
+5 -2
View File
@@ -9,6 +9,7 @@ import com.intellij.openapi.util.io.FileUtil;
import kotlin.io.FilesKt;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.kotlin.TestsRuntimeError;
import org.jetbrains.kotlin.backend.common.CodegenUtil;
import org.jetbrains.kotlin.fileClasses.JvmFileClassUtil;
import org.jetbrains.kotlin.psi.*;
@@ -38,8 +39,10 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase {
@Nullable File javaFilesDir
) throws Exception {
boolean isIgnored = IGNORE_EXPECTED_FAILURES && InTextDirectivesUtils.isIgnoredTarget(getBackend(), wholeFile);
compile(files, javaFilesDir, !isIgnored);
try {
compile(files, javaFilesDir, !isIgnored);
blackBox(!isIgnored);
}
catch (Throwable t) {
@@ -52,7 +55,7 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase {
}
}
throw t;
throw new TestsRuntimeError(t);
}
doBytecodeListingTest(wholeFile);
compiler/tests-common/tests/org/jetbrains/kotlin/codegen/CodegenTestCase.java
+7 -3
View File
@@ -18,6 +18,8 @@ import kotlin.script.experimental.dependencies.ScriptDependencies;
import kotlin.text.Charsets;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.kotlin.TestsCompiletimeError;
import org.jetbrains.kotlin.TestsCompilerError;
import org.jetbrains.kotlin.backend.common.output.OutputFile;
import org.jetbrains.kotlin.backend.common.output.SimpleOutputFileCollection;
import org.jetbrains.kotlin.checkers.CheckerTestUtil;
@@ -491,7 +493,7 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
FqName facadeClassFqName = JvmFileClassUtil.getFileClassInfoNoResolve(myFiles.getPsiFile()).getFacadeClassFqName();
return generateClass(facadeClassFqName.asString());
}
@NotNull
protected Class<?> generateClass(@NotNull String name) {
return generateClass(name, true);
@@ -527,9 +529,9 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
DxChecker.check(classFileFactory);
}
}
catch (Throwable e) {
catch (TestsCompiletimeError e) {
if (reportProblems) {
e.printStackTrace();
e.getOriginal().printStackTrace();
System.err.println("Generating instructions as text...");
try {
if (classFileFactory == null) {
@@ -548,6 +550,8 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
} else {
fail("Compilation failure");
}
} catch (Throwable e) {
throw new TestsCompilerError(e);
}
}
return classFileFactory;
compiler/tests-common/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt
+7 -1
View File
@@ -17,6 +17,7 @@
package org.jetbrains.kotlin.codegen
import com.intellij.psi.search.GlobalSearchScope
import org.jetbrains.kotlin.TestsCompiletimeError
import org.jetbrains.kotlin.backend.jvm.JvmIrCodegenFactory
import org.jetbrains.kotlin.cli.common.output.writeAllTo
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
@@ -80,7 +81,12 @@ object GenerationUtils {
}
// For JVM-specific errors
AnalyzingUtils.throwExceptionOnErrors(state.collectedExtraJvmDiagnostics)
try {
AnalyzingUtils.throwExceptionOnErrors(state.collectedExtraJvmDiagnostics)
} catch (e: Throwable) {
throw TestsCompiletimeError(e)
}
return state
}
}
compiler/tests-common/tests/org/jetbrains/kotlin/resolve/lazy/JvmResolveUtil.kt
+11 -2
View File
@@ -18,6 +18,7 @@ package org.jetbrains.kotlin.resolve.lazy
import com.intellij.openapi.project.Project
import com.intellij.psi.search.GlobalSearchScope
import org.jetbrains.kotlin.TestsCompiletimeError
import org.jetbrains.kotlin.analyzer.AnalysisResult
import org.jetbrains.kotlin.cli.jvm.compiler.CliBindingTrace
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
@@ -57,11 +58,19 @@ object JvmResolveUtil {
trace: BindingTrace = CliBindingTrace()
): AnalysisResult {
for (file in files) {
AnalyzingUtils.checkForSyntacticErrors(file)
try {
AnalyzingUtils.checkForSyntacticErrors(file)
} catch (e: Exception) {
throw TestsCompiletimeError(e)
}
}
return analyze(project, files, configuration, packagePartProvider, trace).apply {
AnalyzingUtils.throwExceptionOnErrors(bindingContext)
try {
AnalyzingUtils.throwExceptionOnErrors(bindingContext)
} catch (e: Exception) {
throw TestsCompiletimeError(e)
}
}
}
compiler/tests-spec/testData/diagnostics/notLinked/contracts/declarations/contractBuilder/common/neg/5.exceptions.compiler.txt
Vendored
+2
View File
@@ -0,0 +1,2 @@
CASES: 4
java.lang.StackOverflowError
compiler/tests-spec/testData/psi/linked/when-expression/p-7/neg/1.2.exceptions.compiler.txt
Vendored
+1
View File
@@ -0,0 +1 @@
junit.framework.AssertionFailedError: Incomplete operation in parsed OK test, method getExpression in KtWhenConditionWithExpression returns null. Element text:
compiler/tests-spec/testData/psi/linked/when-expression/p-7/neg/1.2.kt
Vendored
+1
View File
@@ -6,6 +6,7 @@
* SENTENCE: [1] Type test condition: type checking operator followed by type.
* NUMBER: 2
* DESCRIPTION: 'When' with bound value and 'when condition' with type checking operator and non-type value.
* UNEXPECTED BEHAVIOUR
*/
fun case_2() {
compiler/tests-spec/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsTestSpec.kt
+13 -15
View File
@@ -5,6 +5,7 @@
package org.jetbrains.kotlin.checkers
import org.jetbrains.kotlin.TestExceptionsComparator
import org.jetbrains.kotlin.config.LanguageVersionSettings
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.psi.KtFile
@@ -12,12 +13,11 @@ import org.jetbrains.kotlin.resolve.BindingContext
import org.jetbrains.kotlin.spec.models.AbstractSpecTest
import org.jetbrains.kotlin.spec.SpecTestLinkedType
import org.jetbrains.kotlin.spec.parsers.CommonParser
import org.jetbrains.kotlin.spec.parsers.CommonPatterns.ls
import org.jetbrains.kotlin.spec.validators.*
import org.jetbrains.kotlin.test.*
import org.jetbrains.kotlin.utils.KotlinExceptionWithAttachments
import org.junit.Assert
import java.io.File
import java.util.regex.Matcher
import java.util.regex.Pattern
abstract class AbstractDiagnosticsTestSpec : AbstractDiagnosticsTest() {
@@ -84,32 +84,30 @@ abstract class AbstractDiagnosticsTestSpec : AbstractDiagnosticsTest() {
enableDescriptorsGenerationIfNeeded(testFilePath)
CommonParser.parseSpecTest(testFilePath, files.associate { Pair(it.ktFile!!.name, it.clearText) }).apply {
CommonParser.parseSpecTest(testFilePath, files.associate { Pair(it.fileName, it.clearText) }).apply {
specTest = first
testLinkedType = second
}
println(specTest)
try {
super.analyzeAndCheck(testDataFile, files)
} catch (e: KotlinExceptionWithAttachments) {
val matches = exceptionPattern.matcher(e.message)
if (!matches.find())
Assert.fail(SpecTestValidationFailedReason.UNKNOWN_FRONTEND_EXCEPTION.description)
val checkUnexpectedBehaviour: (Matcher?) -> Pair<Boolean, Set<Int>?> = l@{ matches ->
if (specTest.unexpectedBehavior) return@l Pair(true, null)
if (matches == null) return@l Pair(false, null)
val lineNumber = matches.group("lineNumber").toInt()
val symbolNumber = matches.group("symbolNumber").toInt()
val filename = matches.group("filename")
val fileContent = files.find { it.ktFile?.name == filename }!!.clearText
val exceptionPosition =
fileContent.lines().subList(0, lineNumber).joinToString("\n").length + symbolNumber
val exceptionPosition = fileContent.lines().subList(0, lineNumber).joinToString("\n").length + symbolNumber
val testCases = specTest.cases.byRanges[filename]
val isExpectedException = testCases!!.floorEntry(exceptionPosition).value.all { it.value.unexpectedBehavior }
val testCasesWithSamePosition = testCases!!.floorEntry(exceptionPosition).value
if (!isExpectedException)
Assert.fail()
return@l Pair(testCasesWithSamePosition.all { it.value.unexpectedBehavior }, testCasesWithSamePosition.keys.toSet())
}
TestExceptionsComparator(testDataFile).runAndCompareWithExpected(checkUnexpectedBehaviour) {
super.analyzeAndCheck(testDataFile, files)
}
}
compiler/tests-spec/tests/org/jetbrains/kotlin/codegen/AbstractBlackBoxCodegenTestSpec.kt
+5 -2
View File
@@ -6,11 +6,12 @@
package org.jetbrains.kotlin.codegen
import com.intellij.openapi.util.io.FileUtil
import org.jetbrains.kotlin.TestExceptionsComparator
import org.jetbrains.kotlin.spec.parsers.CommonParser
import org.jetbrains.kotlin.spec.parsers.CommonPatterns.packagePattern
import org.jetbrains.kotlin.spec.utils.GeneralConfiguration.TESTDATA_PATH
import org.jetbrains.kotlin.test.InTextDirectivesUtils
import java.io.File
import java.io.*
abstract class AbstractBlackBoxCodegenTestSpec : AbstractBlackBoxCodegenTest() {
companion object {
@@ -55,6 +56,8 @@ abstract class AbstractBlackBoxCodegenTestSpec : AbstractBlackBoxCodegenTest() {
includeHelpers(wholeFile, files)
super.doMultiFileTest(wholeFile, files, javaFilesDir)
TestExceptionsComparator(wholeFile).runAndCompareWithExpected {
super.doMultiFileTest(wholeFile, files, javaFilesDir)
}
}
}
compiler/tests-spec/tests/org/jetbrains/kotlin/parsing/AbstractParsingTestSpec.kt
+12 -9
View File
@@ -6,6 +6,7 @@
package org.jetbrains.kotlin.parsing
import com.intellij.openapi.util.io.FileUtil
import org.jetbrains.kotlin.TestExceptionsComparator
import org.jetbrains.kotlin.spec.parsers.CommonParser
import org.jetbrains.kotlin.spec.validators.*
import org.junit.Assert
@@ -13,22 +14,24 @@ import java.io.File
abstract class AbstractParsingTestSpec : AbstractParsingTest() {
override fun doParsingTest(filePath: String) {
val file = File(filePath)
val (specTest, testLinkedType) = CommonParser.parseSpecTest(
filePath,
mapOf("main.kt" to FileUtil.loadFile(File(filePath), true))
mapOf("main.kt" to FileUtil.loadFile(file, true))
)
println(specTest)
super.doParsingTest(filePath, CommonParser::testInfoFilter)
TestExceptionsComparator(file).runAndCompareWithExpected({ Pair(specTest.unexpectedBehavior, null) }) {
super.doParsingTest(filePath, CommonParser::testInfoFilter)
val psiTestValidator = ParsingTestTypeValidator(myFile, File(filePath), specTest)
try {
psiTestValidator.validatePathConsistency(testLinkedType)
psiTestValidator.validateTestType()
} catch (e: SpecTestValidationException) {
Assert.fail(e.description)
try {
val psiTestValidator = ParsingTestTypeValidator(myFile, File(filePath), specTest)
psiTestValidator.validatePathConsistency(testLinkedType)
psiTestValidator.validateTestType()
} catch (e: SpecTestValidationException) {
Assert.fail(e.description)
}
}
}
}
compiler/tests/org/jetbrains/kotlin/parsing/AbstractParsingTest.java
+17 -12
View File
@@ -28,6 +28,7 @@ import com.intellij.psi.util.PsiTreeUtil;
import com.intellij.util.PathUtil;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.kotlin.KtNodeTypes;
import org.jetbrains.kotlin.TestsCompilerError;
import org.jetbrains.kotlin.cli.common.script.CliScriptDefinitionProvider;
import org.jetbrains.kotlin.psi.*;
import org.jetbrains.kotlin.script.ScriptDefinitionProvider;
@@ -108,20 +109,24 @@ public abstract class AbstractParsingTest extends KtParsingTestCase {
String fileContent = loadFile(filePath);
myFileExt = FileUtilRt.getExtension(PathUtil.getFileName(filePath));
myFile = createFile(filePath, fileType, contentFilter != null ? contentFilter.invoke(fileContent) : fileContent);
myFile.acceptChildren(new KtVisitorVoid() {
@Override
public void visitKtElement(@NotNull KtElement element) {
element.acceptChildren(this);
try {
checkPsiGetters(element);
try {
myFile = createFile(filePath, fileType, contentFilter != null ? contentFilter.invoke(fileContent) : fileContent);
myFile.acceptChildren(new KtVisitorVoid() {
@Override
public void visitKtElement(@NotNull KtElement element) {
element.acceptChildren(this);
try {
checkPsiGetters(element);
}
catch (Throwable throwable) {
throw new TestsCompilerError(throwable);
}
}
catch (Throwable throwable) {
throw new RuntimeException(throwable);
}
}
});
});
} catch (Throwable throwable) {
throw new TestsCompilerError(throwable);
}
doCheckResult(myFullDataPath, filePath.replaceAll("\\.kts?", ".txt"), toParseTreeText(myFile, false, false).trim());
}