Implement tests exceptions fixation mechanism

compiler/tests-common/tests/org/jetbrains/kotlin/TestExceptionsComparator.kt

@@ -0,0 +1,107 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin
+
+import org.junit.Assert
+import org.jetbrains.kotlin.test.KotlinTestUtils
+import java.io.File
+import java.lang.Error
+import java.util.regex.Matcher
+import java.util.regex.Pattern
+
+enum class TestsExceptionFilePostfix(val text: String) {
+    COMPILER_ERROR("compiler"),
+    COMPILETIME_ERROR("compiletime"),
+    RUNTIME_ERROR("runtime"),
+    INFRASTRUCTURE_ERROR("infrastructure")
+}
+
+sealed class TestsError(val postfix: TestsExceptionFilePostfix) : Error() {
+    abstract val original: Throwable
+    override fun toString() = original.toString()
+}
+
+class TestsCompilerError(override val original: Throwable) : TestsError(TestsExceptionFilePostfix.COMPILER_ERROR)
+class TestsInfrastructureError(override val original: Throwable) : TestsError(TestsExceptionFilePostfix.INFRASTRUCTURE_ERROR)
+class TestsCompiletimeError(override val original: Throwable) : TestsError(TestsExceptionFilePostfix.COMPILETIME_ERROR)
+class TestsRuntimeError(override val original: Throwable) : TestsError(TestsExceptionFilePostfix.RUNTIME_ERROR)
+
+private enum class ExceptionType {
+    ANALYZING_EXPRESSION,
+    UNKNOWN
+}
+
+class TestExceptionsComparator(wholeFile: File) {
+    companion object {
+        private const val EXCEPTIONS_FILE_PREFIX = "exceptions"
+
+        private val exceptionMessagePatterns = mapOf(
+            ExceptionType.ANALYZING_EXPRESSION to
+                    Pattern.compile("""Exception while analyzing expression at \((?<lineNumber>\d+),(?<symbolNumber>\d+)\) in /(?<filename>.*?)$""")
+        )
+        private val ls = System.lineSeparator()
+    }
+
+    private val filePathPrefix = "${wholeFile.parent}/${wholeFile.nameWithoutExtension}.$EXCEPTIONS_FILE_PREFIX"
+
+    private fun analyze(e: Throwable): Matcher? {
+        for ((_, pattern) in exceptionMessagePatterns) {
+            if (e.message == null) continue
+            val matches = pattern.matcher(e.message)
+            if (matches.find()) return matches
+        }
+
+        return null
+    }
+
+    private fun getExceptionInfo(e: TestsError, cases: Set<Int>?): String {
+        val casesAsString = cases?.run { "CASES: " + joinToString() + ls } ?: ""
+
+        return when (e) {
+            is TestsRuntimeError ->
+                (e.original.cause ?: e.original).run {
+                    casesAsString + toString() + stackTrace[0]?.let { ls + it }
+                }
+            is TestsCompilerError, is TestsInfrastructureError -> casesAsString + (e.original.cause ?: e.original).toString()
+            is TestsCompiletimeError -> throw e.original
+        }
+    }
+
+    private fun validateExistingExceptionFiles(e: TestsError?) {
+        val postfixesOfFilesToCheck = TestsExceptionFilePostfix.values().toMutableSet().filter { it != e?.postfix }
+
+        postfixesOfFilesToCheck.forEach {
+            if (File("$filePathPrefix.${it.text}.txt").exists())
+                Assert.fail("No $it, but file $filePathPrefix.${it.text}.txt exists.")
+        }
+    }
+
+    fun runAndCompareWithExpected(checkUnexpectedBehaviour: ((Matcher?) -> Pair<Boolean, Set<Int>?>)? = null, runnable: () -> Unit) {
+        try {
+            runnable()
+        } catch (e: TestsError) {
+            val exceptionInfo = analyze(e.original)
+            val unexpectedBehaviourCheckResult = checkUnexpectedBehaviour?.invoke(exceptionInfo)
+
+            if (e is TestsCompilerError && unexpectedBehaviourCheckResult?.first == false)
+                throw e.original
+
+            val exceptionsFile = File("$filePathPrefix.${e.postfix.text}.txt")
+
+            try {
+                KotlinTestUtils.assertEqualsToFile(exceptionsFile, getExceptionInfo(e, unexpectedBehaviourCheckResult?.second))
+            } catch (t: AssertionError) {
+                e.original.printStackTrace()
+                throw t
+            }
+
+            e.original.printStackTrace()
+            validateExistingExceptionFiles(e)
+            return
+        }
+        validateExistingExceptionFiles(null)
+    }
+}

compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsTest.kt

@@ -10,6 +10,7 @@ import com.intellij.openapi.util.text.StringUtil
 import com.intellij.psi.PsiElement
 import com.intellij.psi.search.GlobalSearchScope
 import junit.framework.TestCase
+import org.jetbrains.kotlin.TestsCompilerError
 import org.jetbrains.kotlin.analyzer.AnalysisResult
 import org.jetbrains.kotlin.analyzer.common.CommonAnalyzerFacade
 import org.jetbrains.kotlin.builtins.jvm.JvmBuiltIns
@@ -64,6 +65,16 @@ import java.util.regex.Pattern
 
 abstract class AbstractDiagnosticsTest : BaseDiagnosticsTest() {
     override fun analyzeAndCheck(testDataFile: File, files: List<TestFile>) {
+        try {
+            analyzeAndCheckUnhandled(testDataFile, files)
+        } catch (t: AssertionError) {
+            throw t
+        } catch (t: Throwable) {
+            throw TestsCompilerError(t)
+        }
+    }
+
+    private fun analyzeAndCheckUnhandled(testDataFile: File, files: List<TestFile>) {
         val groupedByModule = files.groupBy(TestFile::module)
 
         var lazyOperationsLog: LazyOperationsLog? = null

compiler/tests-common/tests/org/jetbrains/kotlin/checkers/BaseDiagnosticsTest.kt

@@ -121,7 +121,7 @@ abstract class BaseDiagnosticsTest : KotlinMultiFileTestWithJava<TestModule, Tes
 
     inner class TestFile(
             val module: TestModule?,
-            fileName: String,
+            val fileName: String,
             textWithMarkers: String,
             val directives: Map<String, String>
     ) {

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractBlackBoxCodegenTest.java

@@ -9,6 +9,7 @@ import com.intellij.openapi.util.io.FileUtil;
 import kotlin.io.FilesKt;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.jetbrains.kotlin.TestsRuntimeError;
 import org.jetbrains.kotlin.backend.common.CodegenUtil;
 import org.jetbrains.kotlin.fileClasses.JvmFileClassUtil;
 import org.jetbrains.kotlin.psi.*;
@@ -38,8 +39,10 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase {
         @Nullable File javaFilesDir
     ) throws Exception {
         boolean isIgnored = IGNORE_EXPECTED_FAILURES && InTextDirectivesUtils.isIgnoredTarget(getBackend(), wholeFile);
+
+        compile(files, javaFilesDir, !isIgnored);
+
         try {
-            compile(files, javaFilesDir, !isIgnored);
             blackBox(!isIgnored);
         }
         catch (Throwable t) {
@@ -52,7 +55,7 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase {
                 }
             }
 
-            throw t;
+            throw new TestsRuntimeError(t);
         }
 
         doBytecodeListingTest(wholeFile);

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/CodegenTestCase.java

@@ -18,6 +18,8 @@ import kotlin.script.experimental.dependencies.ScriptDependencies;
 import kotlin.text.Charsets;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.jetbrains.kotlin.TestsCompiletimeError;
+import org.jetbrains.kotlin.TestsCompilerError;
 import org.jetbrains.kotlin.backend.common.output.OutputFile;
 import org.jetbrains.kotlin.backend.common.output.SimpleOutputFileCollection;
 import org.jetbrains.kotlin.checkers.CheckerTestUtil;
@@ -491,7 +493,7 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
         FqName facadeClassFqName = JvmFileClassUtil.getFileClassInfoNoResolve(myFiles.getPsiFile()).getFacadeClassFqName();
         return generateClass(facadeClassFqName.asString());
     }
-    
+
     @NotNull
     protected Class<?> generateClass(@NotNull String name) {
         return generateClass(name, true);
@@ -527,9 +529,9 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
                     DxChecker.check(classFileFactory);
                 }
             }
-            catch (Throwable e) {
+            catch (TestsCompiletimeError e) {
                 if (reportProblems) {
-                    e.printStackTrace();
+                    e.getOriginal().printStackTrace();
                     System.err.println("Generating instructions as text...");
                     try {
                         if (classFileFactory == null) {
@@ -548,6 +550,8 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
                 } else {
                     fail("Compilation failure");
                 }
+            } catch (Throwable e) {
+                throw new TestsCompilerError(e);
             }
         }
         return classFileFactory;

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt

@@ -17,6 +17,7 @@
 package org.jetbrains.kotlin.codegen
 
 import com.intellij.psi.search.GlobalSearchScope
+import org.jetbrains.kotlin.TestsCompiletimeError
 import org.jetbrains.kotlin.backend.jvm.JvmIrCodegenFactory
 import org.jetbrains.kotlin.cli.common.output.writeAllTo
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
@@ -80,7 +81,12 @@ object GenerationUtils {
         }
 
         // For JVM-specific errors
-        AnalyzingUtils.throwExceptionOnErrors(state.collectedExtraJvmDiagnostics)
+        try {
+            AnalyzingUtils.throwExceptionOnErrors(state.collectedExtraJvmDiagnostics)
+        } catch (e: Throwable) {
+            throw TestsCompiletimeError(e)
+        }
+
         return state
     }
 }

compiler/tests-common/tests/org/jetbrains/kotlin/resolve/lazy/JvmResolveUtil.kt

@@ -18,6 +18,7 @@ package org.jetbrains.kotlin.resolve.lazy
 
 import com.intellij.openapi.project.Project
 import com.intellij.psi.search.GlobalSearchScope
+import org.jetbrains.kotlin.TestsCompiletimeError
 import org.jetbrains.kotlin.analyzer.AnalysisResult
 import org.jetbrains.kotlin.cli.jvm.compiler.CliBindingTrace
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
@@ -57,11 +58,19 @@ object JvmResolveUtil {
         trace: BindingTrace = CliBindingTrace()
     ): AnalysisResult {
         for (file in files) {
-            AnalyzingUtils.checkForSyntacticErrors(file)
+            try {
+                AnalyzingUtils.checkForSyntacticErrors(file)
+            } catch (e: Exception) {
+                throw TestsCompiletimeError(e)
+            }
         }
 
         return analyze(project, files, configuration, packagePartProvider, trace).apply {
-            AnalyzingUtils.throwExceptionOnErrors(bindingContext)
+            try {
+                AnalyzingUtils.throwExceptionOnErrors(bindingContext)
+            } catch (e: Exception) {
+                throw TestsCompiletimeError(e)
+            }
         }
     }
 

compiler/tests-spec/testData/diagnostics/notLinked/contracts/declarations/contractBuilder/common/neg/5.exceptions.compiler.txt

@@ -0,0 +1,2 @@
+CASES: 4
+java.lang.StackOverflowError

compiler/tests-spec/testData/psi/linked/when-expression/p-7/neg/1.2.exceptions.compiler.txt

@@ -0,0 +1 @@
+junit.framework.AssertionFailedError: Incomplete operation in parsed OK test, method getExpression in KtWhenConditionWithExpression returns null. Element text:

compiler/tests-spec/testData/psi/linked/when-expression/p-7/neg/1.2.kt

@@ -6,6 +6,7 @@
  * SENTENCE: [1] Type test condition: type checking operator followed by type.
  * NUMBER: 2
  * DESCRIPTION: 'When' with bound value and 'when condition' with type checking operator and non-type value.
+ * UNEXPECTED BEHAVIOUR
  */
 
 fun case_2() {

compiler/tests-spec/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsTestSpec.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.kotlin.checkers
 
+import org.jetbrains.kotlin.TestExceptionsComparator
 import org.jetbrains.kotlin.config.LanguageVersionSettings
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.psi.KtFile
@@ -12,12 +13,11 @@ import org.jetbrains.kotlin.resolve.BindingContext
 import org.jetbrains.kotlin.spec.models.AbstractSpecTest
 import org.jetbrains.kotlin.spec.SpecTestLinkedType
 import org.jetbrains.kotlin.spec.parsers.CommonParser
-import org.jetbrains.kotlin.spec.parsers.CommonPatterns.ls
 import org.jetbrains.kotlin.spec.validators.*
 import org.jetbrains.kotlin.test.*
-import org.jetbrains.kotlin.utils.KotlinExceptionWithAttachments
 import org.junit.Assert
 import java.io.File
+import java.util.regex.Matcher
 import java.util.regex.Pattern
 
 abstract class AbstractDiagnosticsTestSpec : AbstractDiagnosticsTest() {
@@ -84,32 +84,30 @@ abstract class AbstractDiagnosticsTestSpec : AbstractDiagnosticsTest() {
 
         enableDescriptorsGenerationIfNeeded(testFilePath)
 
-        CommonParser.parseSpecTest(testFilePath, files.associate { Pair(it.ktFile!!.name, it.clearText) }).apply {
+        CommonParser.parseSpecTest(testFilePath, files.associate { Pair(it.fileName, it.clearText) }).apply {
             specTest = first
             testLinkedType = second
         }
 
         println(specTest)
 
-        try {
+        val checkUnexpectedBehaviour: (Matcher?) -> Pair<Boolean, Set<Int>?> = l@{ matches ->
-            super.analyzeAndCheck(testDataFile, files)
+            if (specTest.unexpectedBehavior) return@l Pair(true, null)
-        } catch (e: KotlinExceptionWithAttachments) {
+            if (matches == null) return@l Pair(false, null)
-            val matches = exceptionPattern.matcher(e.message)
-
-            if (!matches.find())
-                Assert.fail(SpecTestValidationFailedReason.UNKNOWN_FRONTEND_EXCEPTION.description)
 
             val lineNumber = matches.group("lineNumber").toInt()
             val symbolNumber = matches.group("symbolNumber").toInt()
             val filename = matches.group("filename")
             val fileContent = files.find { it.ktFile?.name == filename }!!.clearText
-            val exceptionPosition =
+            val exceptionPosition = fileContent.lines().subList(0, lineNumber).joinToString("\n").length + symbolNumber
-                fileContent.lines().subList(0, lineNumber).joinToString("\n").length + symbolNumber
             val testCases = specTest.cases.byRanges[filename]
-            val isExpectedException = testCases!!.floorEntry(exceptionPosition).value.all { it.value.unexpectedBehavior }
+            val testCasesWithSamePosition = testCases!!.floorEntry(exceptionPosition).value
 
-            if (!isExpectedException)
+            return@l Pair(testCasesWithSamePosition.all { it.value.unexpectedBehavior }, testCasesWithSamePosition.keys.toSet())
-                Assert.fail()
+        }
+
+        TestExceptionsComparator(testDataFile).runAndCompareWithExpected(checkUnexpectedBehaviour) {
+            super.analyzeAndCheck(testDataFile, files)
         }
     }
 

compiler/tests-spec/tests/org/jetbrains/kotlin/codegen/AbstractBlackBoxCodegenTestSpec.kt

@@ -6,11 +6,12 @@
 package org.jetbrains.kotlin.codegen
 
 import com.intellij.openapi.util.io.FileUtil
+import org.jetbrains.kotlin.TestExceptionsComparator
 import org.jetbrains.kotlin.spec.parsers.CommonParser
 import org.jetbrains.kotlin.spec.parsers.CommonPatterns.packagePattern
 import org.jetbrains.kotlin.spec.utils.GeneralConfiguration.TESTDATA_PATH
 import org.jetbrains.kotlin.test.InTextDirectivesUtils
-import java.io.File
+import java.io.*
 
 abstract class AbstractBlackBoxCodegenTestSpec : AbstractBlackBoxCodegenTest() {
     companion object {
@@ -55,6 +56,8 @@ abstract class AbstractBlackBoxCodegenTestSpec : AbstractBlackBoxCodegenTest() {
 
         includeHelpers(wholeFile, files)
 
-        super.doMultiFileTest(wholeFile, files, javaFilesDir)
+        TestExceptionsComparator(wholeFile).runAndCompareWithExpected {
+            super.doMultiFileTest(wholeFile, files, javaFilesDir)
+        }
     }
 }

compiler/tests-spec/tests/org/jetbrains/kotlin/parsing/AbstractParsingTestSpec.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.parsing
 
 import com.intellij.openapi.util.io.FileUtil
+import org.jetbrains.kotlin.TestExceptionsComparator
 import org.jetbrains.kotlin.spec.parsers.CommonParser
 import org.jetbrains.kotlin.spec.validators.*
 import org.junit.Assert
@@ -13,22 +14,24 @@ import java.io.File
 
 abstract class AbstractParsingTestSpec : AbstractParsingTest() {
     override fun doParsingTest(filePath: String) {
+        val file = File(filePath)
         val (specTest, testLinkedType) = CommonParser.parseSpecTest(
             filePath,
-            mapOf("main.kt" to FileUtil.loadFile(File(filePath), true))
+            mapOf("main.kt" to FileUtil.loadFile(file, true))
         )
 
         println(specTest)
 
-        super.doParsingTest(filePath, CommonParser::testInfoFilter)
+        TestExceptionsComparator(file).runAndCompareWithExpected({ Pair(specTest.unexpectedBehavior, null) }) {
+            super.doParsingTest(filePath, CommonParser::testInfoFilter)
 
-        val psiTestValidator = ParsingTestTypeValidator(myFile, File(filePath), specTest)
+            try {
-
+                val psiTestValidator = ParsingTestTypeValidator(myFile, File(filePath), specTest)
-        try {
+                psiTestValidator.validatePathConsistency(testLinkedType)
-            psiTestValidator.validatePathConsistency(testLinkedType)
+                psiTestValidator.validateTestType()
-            psiTestValidator.validateTestType()
+            } catch (e: SpecTestValidationException) {
-        } catch (e: SpecTestValidationException) {
+                Assert.fail(e.description)
-            Assert.fail(e.description)
+            }
         }
     }
 }

compiler/tests/org/jetbrains/kotlin/parsing/AbstractParsingTest.java

@@ -28,6 +28,7 @@ import com.intellij.psi.util.PsiTreeUtil;
 import com.intellij.util.PathUtil;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.kotlin.KtNodeTypes;
+import org.jetbrains.kotlin.TestsCompilerError;
 import org.jetbrains.kotlin.cli.common.script.CliScriptDefinitionProvider;
 import org.jetbrains.kotlin.psi.*;
 import org.jetbrains.kotlin.script.ScriptDefinitionProvider;
@@ -108,20 +109,24 @@ public abstract class AbstractParsingTest extends KtParsingTestCase {
         String fileContent = loadFile(filePath);
 
         myFileExt = FileUtilRt.getExtension(PathUtil.getFileName(filePath));
-        myFile = createFile(filePath, fileType, contentFilter != null ? contentFilter.invoke(fileContent) : fileContent);
 
-        myFile.acceptChildren(new KtVisitorVoid() {
+        try {
-            @Override
+            myFile = createFile(filePath, fileType, contentFilter != null ? contentFilter.invoke(fileContent) : fileContent);
-            public void visitKtElement(@NotNull KtElement element) {
+            myFile.acceptChildren(new KtVisitorVoid() {
-                element.acceptChildren(this);
+                @Override
-                try {
+                public void visitKtElement(@NotNull KtElement element) {
-                    checkPsiGetters(element);
+                    element.acceptChildren(this);
+                    try {
+                        checkPsiGetters(element);
+                    }
+                    catch (Throwable throwable) {
+                        throw new TestsCompilerError(throwable);
+                    }
                 }
-                catch (Throwable throwable) {
+            });
-                    throw new RuntimeException(throwable);
+        } catch (Throwable throwable) {
-                }
+            throw new TestsCompilerError(throwable);
-            }
+        }
-        });
 
         doCheckResult(myFullDataPath, filePath.replaceAll("\\.kts?", ".txt"), toParseTreeText(myFile, false, false).trim());
     }