Add annotation to prevent test invocation twice

Cause: it helps to fix double inversion of muted non-flaky tests result
(KTI-216).

compiler/tests-common/tests/org/jetbrains/kotlin/test/KotlinTestUtils.java

@@ -69,6 +69,7 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.StringWriter;
+import java.lang.annotation.Annotation;
 import java.lang.reflect.Method;
 import java.nio.charset.Charset;
 import java.util.*;
@@ -755,7 +756,7 @@ public class KotlinTestUtils {
     }
 
     private static void runTestImpl(@NotNull DoTest test, @Nullable TestCase testCase, String testDataFilePath) throws Exception {
-        if (testCase != null) {
+        if (testCase != null && !isRunTestOverridden(testCase)) {
             Function0<Unit> wrapWithMuteInDatabase = MuteWithDatabaseKt.wrapWithMuteInDatabase(testCase, () -> {
                 try {
                     test.invoke(testDataFilePath);
@@ -770,11 +771,20 @@ public class KotlinTestUtils {
                 return;
             }
         }
+        MuteWithFileKt.testWithMuteInFile(test, testCase).invoke(testDataFilePath);
+    }
 
-        DoTest wrappedTest = testCase != null ?
+    private static boolean isRunTestOverridden(TestCase testCase) {
-                             MuteWithFileKt.testWithMuteInFile(test, testCase) :
+        Class<?> type = testCase.getClass();
-                             MuteWithFileKt.testWithMuteInFile(test, "");
+        while (type != null) {
-        wrappedTest.invoke(testDataFilePath);
+            for (Annotation annotation : type.getDeclaredAnnotations()) {
+                if (annotation.annotationType().equals(WithMutedInDatabaseRunTest.class)) {
+                    return true;
+                }
+            }
+            type = type.getSuperclass();
+        }
+        return false;
     }
 
     private static DoTest testWithCustomIgnoreDirective(DoTest test, TargetBackend targetBackend, String ignoreDirective) throws Exception {

compiler/tests-common/tests/org/jetbrains/kotlin/test/muteWithDatabase.kt

@@ -300,3 +300,5 @@ fun isIgnoredInDatabaseWithLog(testCase: TestCase): Boolean {
 fun TestCase.runTest(test: () -> Unit) {
     (wrapWithMuteInDatabase(this, test) ?: test).invoke()
 }
+
+annotation class WithMutedInDatabaseRunTest

compiler/tests-common/tests/org/jetbrains/kotlin/test/muteWithFile.kt

@@ -17,8 +17,8 @@ private val AUTOMATICALLY_GENERATE_FAIL_FILE_FOR_FAILED_TESTS: Boolean = false
 annotation class MuteExtraSuffix(val value: String = "")
 
 @Throws(Exception::class)
-fun testWithMuteInFile(test: DoTest, testCase: TestCase): DoTest {
+fun testWithMuteInFile(test: DoTest, testCase: TestCase?): DoTest {
-    val extraSuffix = testCase.javaClass.getAnnotation(MuteExtraSuffix::class.java)?.value ?: ""
+    val extraSuffix = testCase?.javaClass?.getAnnotation(MuteExtraSuffix::class.java)?.value ?: ""
     return testWithMuteInFile(test, extraSuffix)
 }
 

compiler/tests/org/jetbrains/kotlin/util/KotlinVersionsTest.kt

@@ -20,7 +20,8 @@ import com.intellij.openapi.util.io.FileUtil
 import org.jetbrains.kotlin.codegen.forTestCompile.ForTestCompileRuntime
 import org.jetbrains.kotlin.config.KotlinCompilerVersion
 import org.jetbrains.kotlin.config.LanguageVersion
-import org.jetbrains.kotlin.test.isIgnoredInDatabaseWithLog
+import org.jetbrains.kotlin.test.WithMutedInDatabaseRunTest
+import org.jetbrains.kotlin.test.runTest
 import org.jetbrains.kotlin.test.testFramework.KtUsefulTestCase
 import org.jetbrains.kotlin.utils.addIfNotNull
 import org.junit.Assert
@@ -29,6 +30,7 @@ import org.xml.sax.helpers.DefaultHandler
 import java.io.File
 import javax.xml.parsers.SAXParserFactory
 
+@WithMutedInDatabaseRunTest
 class KotlinVersionsTest : KtUsefulTestCase() {
     fun testVersionsAreConsistent() {
         val versionPattern = "(\\d+)\\.(\\d+)(\\.(\\d+)|-SNAPSHOT)?".toRegex()
@@ -143,7 +145,7 @@ class KotlinVersionsTest : KtUsefulTestCase() {
 
     private fun Collection<Any>.areEqual(): Boolean = all(first()::equals)
 
-    override fun shouldRunTest(): Boolean {
+    override fun runTest() {
-        return super.shouldRunTest() && !isIgnoredInDatabaseWithLog(this)
+        runTest { super.runTest() }
     }
 }

idea/idea-completion/tests/org/jetbrains/kotlin/idea/completion/test/KotlinCompletionTestCase.java

@@ -10,7 +10,9 @@ import com.intellij.codeInsight.completion.CompletionTestCase;
 import com.intellij.openapi.vfs.newvfs.impl.VfsRootAccess;
 import com.intellij.util.ArrayUtil;
 import org.jetbrains.kotlin.test.KotlinTestUtils;
+import org.jetbrains.kotlin.test.WithMutedInDatabaseRunTest;
 
+@WithMutedInDatabaseRunTest
 abstract public class KotlinCompletionTestCase extends CompletionTestCase {
     @Override
     protected void setUp() throws Exception {

idea/idea-maven/test/org/jetbrains/kotlin/idea/maven/MavenImportingTestCase.java

@@ -51,14 +51,15 @@ import org.jetbrains.jps.model.java.JavaSourceRootProperties;
 import org.jetbrains.jps.model.java.JavaSourceRootType;
 import org.jetbrains.jps.model.module.JpsModuleSourceRootType;
 import org.jetbrains.kotlin.idea.test.KotlinSdkCreationChecker;
+import org.jetbrains.kotlin.test.KotlinTestUtils;
+import org.jetbrains.kotlin.test.WithMutedInDatabaseRunTest;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import static org.jetbrains.kotlin.test.MuteWithDatabaseKt.isIgnoredInDatabaseWithLog;
+@WithMutedInDatabaseRunTest
-
 public abstract class MavenImportingTestCase extends MavenTestCase {
     protected MavenProjectsTree myProjectsTree;
     protected MavenProjectsManager myProjectsManager;
@@ -103,8 +104,8 @@ public abstract class MavenImportingTestCase extends MavenTestCase {
     }
 
     @Override
-    protected boolean shouldRunTest() {
+    protected void runTest() throws Throwable {
-        return super.shouldRunTest() && !isIgnoredInDatabaseWithLog(this);
+        KotlinTestUtils.runTestWithThrowable(this, () -> super.runTest());
     }
 
     protected void assertModules(String... expectedNames) {

idea/idea-test-framework/test/org/jetbrains/kotlin/idea/test/KotlinCodeInsightTestCase.java

@@ -18,13 +18,17 @@ package org.jetbrains.kotlin.idea.test;
 
 import com.intellij.codeInsight.CodeInsightTestCase;
 import com.intellij.openapi.vfs.newvfs.impl.VfsRootAccess;
+import kotlin.Unit;
 import org.jetbrains.kotlin.test.KotlinTestUtils;
+import org.jetbrains.kotlin.test.MuteWithDatabaseKt;
+import org.jetbrains.kotlin.test.WithMutedInDatabaseRunTest;
 
 import static org.jetbrains.kotlin.test.MuteWithDatabaseKt.isIgnoredInDatabaseWithLog;
 
 /**
  * Please use KotlinLightCodeInsightFixtureTestCase as the base class for all new tests.
  */
+@WithMutedInDatabaseRunTest
 @Deprecated
 public abstract class KotlinCodeInsightTestCase extends CodeInsightTestCase {
     @Override
@@ -40,7 +44,7 @@ public abstract class KotlinCodeInsightTestCase extends CodeInsightTestCase {
     }
 
     @Override
-    protected boolean shouldRunTest() {
+    protected void runTest() throws Throwable {
-        return super.shouldRunTest() && !isIgnoredInDatabaseWithLog(this);
+        KotlinTestUtils.runTestWithThrowable(this, () -> super.runTest());
     }
 }

idea/idea-test-framework/test/org/jetbrains/kotlin/idea/test/KotlinLightCodeInsightFixtureTestCaseBase.java

@@ -29,6 +29,7 @@ import org.jetbrains.annotations.NonNls;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.kotlin.test.KotlinTestUtils;
+import org.jetbrains.kotlin.test.WithMutedInDatabaseRunTest;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -37,8 +38,7 @@ import java.io.OutputStreamWriter;
 import java.nio.charset.Charset;
 import java.util.Collection;
 
-import static org.jetbrains.kotlin.test.MuteWithDatabaseKt.isIgnoredInDatabaseWithLog;
+@WithMutedInDatabaseRunTest
-
 public abstract class KotlinLightCodeInsightFixtureTestCaseBase extends LightCodeInsightFixtureTestCase {
     @NotNull
     @Override
@@ -60,11 +60,6 @@ public abstract class KotlinLightCodeInsightFixtureTestCaseBase extends LightCod
     protected final Collection<File> myFilesToDelete = new THashSet<>();
     private final TempFiles myTempFiles = new TempFiles(myFilesToDelete);
 
-    @Override
-    protected boolean shouldRunTest() {
-        return super.shouldRunTest() && !isIgnoredInDatabaseWithLog(this);
-    }
-
     @Override
     protected void tearDown() throws Exception {
         myTempFiles.deleteAll();

idea/idea-test-framework/test/org/jetbrains/kotlin/idea/test/KotlinLightCodeInsightFixtureTestCaseBase.java.191

@@ -30,6 +30,7 @@ import org.jetbrains.annotations.NonNls;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.kotlin.test.KotlinTestUtils;
+import org.jetbrains.kotlin.test.WithMutedInDatabaseRunTest;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -38,8 +39,7 @@ import java.io.OutputStreamWriter;
 import java.nio.charset.Charset;
 import java.util.Collection;
 
-import static org.jetbrains.kotlin.test.MuteWithDatabaseKt.isIgnoredInDatabaseWithLog;
+@WithMutedInDatabaseRunTest
-
 public abstract class KotlinLightCodeInsightFixtureTestCaseBase extends LightCodeInsightFixtureTestCase {
     @NotNull
     @Override
@@ -65,11 +65,6 @@ public abstract class KotlinLightCodeInsightFixtureTestCaseBase extends LightCod
     protected final Collection<File> myFilesToDelete = new THashSet<>();
     private final TempFiles myTempFiles = new TempFiles(myFilesToDelete);
 
-    @Override
-    protected boolean shouldRunTest() {
-        return super.shouldRunTest() && !isIgnoredInDatabaseWithLog(this);
-    }
-
     @Override
     protected void tearDown() throws Exception {
         myTempFiles.deleteAll();

idea/jvm-debugger/jvm-debugger-test/test/org/jetbrains/kotlin/idea/debugger/test/KotlinDescriptorTestCase.kt

@@ -33,11 +33,8 @@ import org.jetbrains.kotlin.idea.debugger.test.util.KotlinOutputChecker
 import org.jetbrains.kotlin.idea.debugger.test.util.LogPropagator
 import org.jetbrains.kotlin.idea.test.ConfigLibraryUtil
 import org.jetbrains.kotlin.idea.test.PluginTestCaseBase
-import org.jetbrains.kotlin.test.Directives
+import org.jetbrains.kotlin.test.*
 import org.jetbrains.kotlin.test.KotlinBaseTest.TestFile
-import org.jetbrains.kotlin.test.KotlinTestUtils
-import org.jetbrains.kotlin.test.TestMetadata
-import org.jetbrains.kotlin.test.isIgnoredInDatabaseWithLog
 import org.jetbrains.kotlin.test.testFramework.runWriteAction
 import org.junit.ComparisonFailure
 import java.io.File
@@ -48,6 +45,7 @@ internal const val TEST_LIBRARY_NAME = "TestLibrary"
 
 class TestFiles(val originalFile: File, val wholeFile: TestFile, files: List<TestFile>) : List<TestFile> by files
 
+@WithMutedInDatabaseRunTest
 abstract class KotlinDescriptorTestCase : DescriptorTestCase() {
     private lateinit var testAppDirectory: File
     private lateinit var sourcesOutputDirectory: File
@@ -233,8 +231,8 @@ abstract class KotlinDescriptorTestCase : DescriptorTestCase() {
         return super.getData(dataId)
     }
 
-    override fun shouldRunTest(): Boolean {
+    override fun runTest() {
-        return super.shouldRunTest() && !isIgnoredInDatabaseWithLog(this)
+        runTest { super.runTest() }
     }
 
     protected fun getTestDirectoryPath(): String = javaClass.getAnnotation(TestMetadata::class.java).value

idea/tests/org/jetbrains/kotlin/idea/caches/resolve/MultiModuleHighlightingTest.kt

@@ -42,10 +42,12 @@ import org.jetbrains.kotlin.samWithReceiver.SamWithReceiverCommandLineProcessor.
 import org.jetbrains.kotlin.test.JUnit3WithIdeaConfigurationRunner
 import org.jetbrains.kotlin.test.MockLibraryUtil
 import org.jetbrains.kotlin.test.TestJdkKind.FULL_JDK
+import org.jetbrains.kotlin.test.WithMutedInDatabaseRunTest
 import org.jetbrains.kotlin.test.runTest
 import org.junit.Assert.assertNotEquals
 import org.junit.runner.RunWith
 
+@WithMutedInDatabaseRunTest
 @RunWith(JUnit3WithIdeaConfigurationRunner::class)
 open class MultiModuleHighlightingTest : AbstractMultiModuleHighlightingTest() {
     override fun getTestDataPath() = PluginTestCaseBase.getTestDataPathBase() + "/multiModuleHighlighting/"

idea/tests/org/jetbrains/kotlin/idea/configuration/AbstractConfigureKotlinTest.kt

@@ -24,11 +24,14 @@ import org.jetbrains.kotlin.idea.test.PluginTestCaseBase
 import org.jetbrains.kotlin.idea.test.PluginTestCaseBase.*
 import org.jetbrains.kotlin.idea.util.getProjectJdkTableSafe
 import org.jetbrains.kotlin.test.KotlinTestUtils
+import org.jetbrains.kotlin.test.WithMutedInDatabaseRunTest
 import org.jetbrains.kotlin.test.isIgnoredInDatabaseWithLog
+import org.jetbrains.kotlin.test.runTest
 import org.jetbrains.kotlin.utils.PathUtil
 import java.io.File
 import java.nio.file.Path
 
+@WithMutedInDatabaseRunTest
 abstract class AbstractConfigureKotlinTest : PlatformTestCase() {
     override fun setUp() {
         super.setUp()
@@ -132,8 +135,8 @@ abstract class AbstractConfigureKotlinTest : PlatformTestCase() {
         UsefulTestCase.assertDoesntExist(File(JS_CONFIGURATOR.getDefaultPathToJarFile(project)))
     }
 
-    override fun shouldRunTest(): Boolean {
+    override fun runTest() {
-        return super.shouldRunTest() && !isIgnoredInDatabaseWithLog(this)
+        return runTest { super.runTest() }
     }
 
     companion object {

idea/tests/org/jetbrains/kotlin/idea/decompiler/navigation/NavigateFromJSLibraryTest.kt

@@ -10,9 +10,11 @@ import junit.framework.TestCase
 import org.jetbrains.kotlin.idea.test.PluginTestCaseBase
 import org.jetbrains.kotlin.idea.test.SdkAndMockLibraryProjectDescriptor
 import org.jetbrains.kotlin.test.JUnit3WithIdeaConfigurationRunner
+import org.jetbrains.kotlin.test.WithMutedInDatabaseRunTest
 import org.jetbrains.kotlin.test.runTest
 import org.junit.runner.RunWith
 
+@WithMutedInDatabaseRunTest
 @RunWith(JUnit3WithIdeaConfigurationRunner::class)
 class NavigateFromJSLibrarySourcesTest : AbstractNavigateFromLibrarySourcesTest() {
     fun testIcon() {

idea/tests/org/jetbrains/kotlin/idea/decompiler/navigation/NavigateFromLibrarySourcesTest.kt

@@ -15,10 +15,12 @@ import org.jetbrains.kotlin.idea.test.SdkAndMockLibraryProjectDescriptor
 import org.jetbrains.kotlin.psi.KtClass
 import org.jetbrains.kotlin.psi.KtClassOrObject
 import org.jetbrains.kotlin.test.JUnit3WithIdeaConfigurationRunner
+import org.jetbrains.kotlin.test.WithMutedInDatabaseRunTest
 import org.jetbrains.kotlin.test.runTest
 import org.junit.runner.RunWith
 import kotlin.test.assertTrue
 
+@WithMutedInDatabaseRunTest
 @RunWith(JUnit3WithIdeaConfigurationRunner::class)
 class NavigateFromLibrarySourcesTest : AbstractNavigateFromLibrarySourcesTest() {
     fun testJdkClass() {

idea/tests/org/jetbrains/kotlin/idea/inspections/CoroutineNonBlockingontextDetectionTest.kt

@@ -11,7 +11,6 @@ import org.jetbrains.kotlin.idea.test.KotlinLightCodeInsightFixtureTestCase
 import org.jetbrains.kotlin.idea.test.KotlinWithJdkAndRuntimeLightProjectDescriptor
 import org.jetbrains.kotlin.idea.test.PluginTestCaseBase
 import org.jetbrains.kotlin.test.JUnit3WithIdeaConfigurationRunner
-import org.jetbrains.kotlin.test.runTest
 import org.junit.runner.RunWith
 
 @RunWith(JUnit3WithIdeaConfigurationRunner::class)
@@ -47,9 +46,7 @@ class CoroutineNonBlockingContextDetectionTest : KotlinLightCodeInsightFixtureTe
     }
 
     private fun doTest(fileName: String) {
-        runTest {
+        myFixture.configureByFile(fileName)
-            myFixture.configureByFile(fileName)
+        myFixture.testHighlighting(true, false, false, fileName)
-            myFixture.testHighlighting(true, false, false, fileName)
-        }
     }
 }

jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/BaseKotlinJpsBuildTestCase.kt

@@ -19,8 +19,10 @@ package org.jetbrains.kotlin.jps.build
 import org.jetbrains.jps.builders.JpsBuildTestCase
 import org.jetbrains.jps.model.library.JpsLibrary
 import org.jetbrains.kotlin.compilerRunner.JpsKotlinCompilerRunner
+import org.jetbrains.kotlin.test.WithMutedInDatabaseRunTest
 import org.jetbrains.kotlin.test.runTest
 
+@WithMutedInDatabaseRunTest
 abstract class BaseKotlinJpsBuildTestCase : JpsBuildTestCase() {
     @Throws(Exception::class)
     override fun setUp() {

plugins/uast-kotlin/tests/org/jetbrains/uast/test/env/kotlin/AbstractTestWithCoreEnvironment.kt

@@ -14,6 +14,7 @@ import com.intellij.psi.PsiManager
 import com.intellij.rt.execution.junit.FileComparisonFailure
 import junit.framework.TestCase
 import org.jetbrains.kotlin.test.IdeaSystemPropertiesForParallelRunConfigurator
+import org.jetbrains.kotlin.test.WithMutedInDatabaseRunTest
 import org.jetbrains.kotlin.test.runTest
 import org.jetbrains.uast.UastContext
 import org.jetbrains.uast.UastLanguagePlugin
@@ -21,6 +22,7 @@ import org.jetbrains.uast.evaluation.UEvaluatorExtension
 import org.jetbrains.uast.java.JavaUastLanguagePlugin
 import java.io.File
 
+@WithMutedInDatabaseRunTest
 abstract class AbstractTestWithCoreEnvironment : TestCase() {
 
     companion object {