Switch off logging for failing IrBlackBox tests

Logging of tests that are expected to fail is controlled by
kotlin.suppress.expected.test.failures project property.

compiler/build.gradle.kts

@@ -92,6 +92,7 @@ projectTest {
     dependsOn(*testDistProjects.map { "$it:dist" }.toTypedArray())
     workingDir = rootDir
     systemProperty("kotlin.test.script.classpath", testSourceSet.output.classesDirs.joinToString(File.pathSeparator))
+    systemProperty("kotlin.suppress.expected.test.failures", project.findProperty("kotlin.suppress.expected.test.failures") ?: false)
     doFirst {
         systemProperty("kotlin.ant.classpath", antLauncherJar.asPath)
         systemProperty("kotlin.ant.launcher.class", "org.apache.tools.ant.Main")

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractAsmLikeInstructionListingTest.kt

@@ -23,7 +23,7 @@ abstract class AbstractAsmLikeInstructionListingTest : CodegenTestCase() {
         val LOCAL_VARIABLE_TABLE_DIRECTIVE = "// LOCAL_VARIABLE_TABLE"
     }
 
-    override fun doMultiFileTest(wholeFile: File, files: List<TestFile>, javaFilesDir: File?) {
+    override fun doMultiFileTest(wholeFile: File, files: List<TestFile>, javaFilesDir: File?, reportFailures: Boolean) {
         val txtFile = File(wholeFile.parentFile, wholeFile.nameWithoutExtension + ".txt")
         compile(files, javaFilesDir)
 

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractBlackBoxAgainstJavaCodegenTest.kt

@@ -22,12 +22,12 @@ import org.jetbrains.kotlin.test.ConfigurationKind
 import java.io.File
 
 abstract class AbstractBlackBoxAgainstJavaCodegenTest : AbstractBlackBoxCodegenTest() {
-    override fun doMultiFileTest(wholeFile: File, files: MutableList<TestFile>, javaFilesDir: File?) {
+    override fun doMultiFileTest(wholeFile: File, files: MutableList<TestFile>, javaFilesDir: File?, reportFailures: Boolean) {
         javaClassesOutputDirectory = javaFilesDir!!.let { directory ->
             CodegenTestUtil.compileJava(CodegenTestUtil.findJavaSourcesInDirectory(directory), emptyList(), extractJavacOptions(files))
         }
 
-        super.doMultiFileTest(wholeFile, files, null)
+        super.doMultiFileTest(wholeFile, files, null, reportFailures)
     }
 
     override fun updateConfiguration(configuration: CompilerConfiguration) {

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractBlackBoxCodegenTest.java

@@ -31,18 +31,21 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase {
     protected void doMultiFileTest(
         @NotNull File wholeFile,
         @NotNull List<TestFile> files,
-        @Nullable File javaFilesDir
+        @Nullable File javaFilesDir,
+        boolean reportFailures
     ) throws Exception {
         try {
-            compile(files, javaFilesDir);
+            compile(files, javaFilesDir, reportFailures);
-            blackBox();
+            blackBox(reportFailures);
         }
         catch (Throwable t) {
-            try {
+            if (reportFailures) {
-                // To create .txt file in case of failure
+                try {
-                doBytecodeListingTest(wholeFile);
+                    // To create .txt file in case of failure
-            }
+                    doBytecodeListingTest(wholeFile);
-            catch (Throwable ignored) {
+                }
+                catch (Throwable ignored) {
+                }
             }
 
             throw t;
@@ -89,9 +92,9 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase {
         assertEqualsToFile(expectedFile, text, s -> s.replace("COROUTINES_PACKAGE", coroutinesPackage));
     }
 
-    protected void blackBox() {
+    protected void blackBox(boolean reportProblems) {
         // If there are many files, the first 'box(): String' function will be executed.
-        GeneratedClassLoader generatedClassLoader = generateAndCreateClassLoader();
+        GeneratedClassLoader generatedClassLoader = generateAndCreateClassLoader(reportProblems);
         for (KtFile firstFile : myFiles.getPsiFiles()) {
             String className = getFacadeFqName(firstFile, classFileFactory.getGenerationState().getBindingContext());
             if (className == null) continue;
@@ -104,7 +107,9 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase {
                 }
             }
             catch (Throwable e) {
-                System.out.println(generateToText());
+                if (reportProblems) {
+                    System.out.println(generateToText());
+                }
                 throw ExceptionUtilsKt.rethrow(e);
             }
             finally {

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractBlackBoxInlineCodegenTest.kt

@@ -19,15 +19,17 @@ package org.jetbrains.kotlin.codegen
 import java.io.File
 
 abstract class AbstractBlackBoxInlineCodegenTest : AbstractBlackBoxCodegenTest() {
-    override fun doMultiFileTest(wholeFile: File, files: List<TestFile>, javaFilesDir: File?) {
+    override fun doMultiFileTest(wholeFile: File, files: List<TestFile>, javaFilesDir: File?, reportFailures: Boolean) {
-        super.doMultiFileTest(wholeFile, files, javaFilesDir)
+        super.doMultiFileTest(wholeFile, files, javaFilesDir, reportFailures)
         try {
             InlineTestUtil.checkNoCallsToInline(initializedClassLoader.allGeneratedFiles.filterClassFiles(), myFiles.psiFiles)
             SMAPTestUtil.checkSMAP(files, generateClassesInFile().getClassFiles(), false)
         }
         catch (e: Throwable) {
-            println(generateToText())
+            if (reportFailures) {
-            throw e
+                println(generateToText())
+                throw e
+            }
         }
     }
 }

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractBytecodeListingTest.kt

@@ -14,7 +14,7 @@ import org.jetbrains.org.objectweb.asm.Opcodes.*
 import java.io.File
 
 abstract class AbstractBytecodeListingTest : CodegenTestCase() {
-    override fun doMultiFileTest(wholeFile: File, files: List<TestFile>, javaFilesDir: File?) {
+    override fun doMultiFileTest(wholeFile: File, files: List<TestFile>, javaFilesDir: File?, reportFailures: Boolean) {
 
         compile(files, javaFilesDir)
         val actualTxt = BytecodeListingTextCollectingVisitor.getText(classFileFactory, withSignatures = isWithSignatures(wholeFile))

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractBytecodeTextTest.kt

@@ -20,7 +20,7 @@ import java.util.regex.Pattern
 abstract class AbstractBytecodeTextTest : CodegenTestCase() {
 
     @Throws(Exception::class)
-    override fun doMultiFileTest(wholeFile: File, files: List<CodegenTestCase.TestFile>, javaFilesDir: File?) {
+    override fun doMultiFileTest(wholeFile: File, files: List<CodegenTestCase.TestFile>, javaFilesDir: File?, reportFailures: Boolean) {
         createEnvironmentWithMockJdkAndIdeaAnnotations(ConfigurationKind.ALL, files, TestJdkKind.MOCK_JDK, javaFilesDir)
         loadMultiFiles(files)
 

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractCheckLocalVariablesTableTest.kt

@@ -32,7 +32,7 @@ import java.util.regex.Pattern
 
 abstract class AbstractCheckLocalVariablesTableTest : CodegenTestCase() {
 
-    override fun doMultiFileTest(wholeFile: File, files: List<CodegenTestCase.TestFile>, javaFilesDir: File?) {
+    override fun doMultiFileTest(wholeFile: File, files: List<CodegenTestCase.TestFile>, javaFilesDir: File?, reportFailures: Boolean) {
         compile(files, javaFilesDir)
 
         try {
@@ -52,8 +52,10 @@ abstract class AbstractCheckLocalVariablesTableTest : CodegenTestCase() {
 
             doCompare(wholeFile, files.single().content, actualLocalVariables)
         } catch (e: Throwable) {
-            println(classFileFactory.createText())
+            if (reportFailures) {
-            throw e
+                println(classFileFactory.createText())
+                throw e
+            }
         }
     }
 

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractCompileKotlinAgainstInlineKotlinTest.kt

@@ -19,7 +19,7 @@ package org.jetbrains.kotlin.codegen
 import java.io.File
 
 abstract class AbstractCompileKotlinAgainstInlineKotlinTest : AbstractCompileKotlinAgainstKotlinTest() {
-    override fun doMultiFileTest(wholeFile: File, files: List<TestFile>, javaFilesDir: File?) {
+    override fun doMultiFileTest(wholeFile: File, files: List<TestFile>, javaFilesDir: File?, reportFailures: Boolean) {
         val (factory1, factory2) = doTwoFileTest(files.filter { it.name.endsWith(".kt") })
         try {
             val allGeneratedFiles = factory1.asList() + factory2.asList()
@@ -28,8 +28,10 @@ abstract class AbstractCompileKotlinAgainstInlineKotlinTest : AbstractCompileKot
             SMAPTestUtil.checkSMAP(files, allGeneratedFiles.filterClassFiles(), true)
         }
         catch (e: Throwable) {
-            println("FIRST:\n\n${factory1.createText()}\n\nSECOND:\n\n${factory2.createText()}")
+            if (reportFailures) {
-            throw e
+                println("FIRST:\n\n${factory1.createText()}\n\nSECOND:\n\n${factory2.createText()}")
+                throw e
+            }
         }
     }
 }

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractCompileKotlinAgainstKotlinTest.java

@@ -49,7 +49,12 @@ public abstract class AbstractCompileKotlinAgainstKotlinTest extends CodegenTest
     }
 
     @Override
-    protected void doMultiFileTest(@NotNull File wholeFile, @NotNull List<TestFile> files, @Nullable File javaFilesDir) throws Exception {
+    protected void doMultiFileTest(
+            @NotNull File wholeFile,
+            @NotNull List<TestFile> files,
+            @Nullable File javaFilesDir,
+            boolean reportFailures
+    ) throws Exception {
         assert javaFilesDir == null : ".java files are not supported yet in this test";
         doTwoFileTest(files);
     }

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractDumpDeclarationsTest.kt

@@ -15,7 +15,7 @@ abstract class AbstractDumpDeclarationsTest : CodegenTestCase() {
 
     private lateinit var dumpToFile: File
 
-    override fun doMultiFileTest(wholeFile: File, files: List<TestFile>, javaFilesDir: File?) {
+    override fun doMultiFileTest(wholeFile: File, files: List<TestFile>, javaFilesDir: File?, reportFailures: Boolean) {
         val expectedResult = KotlinTestUtils.replaceExtension(wholeFile, "json")
         dumpToFile = KotlinTestUtils.tmpDirForTest(this).resolve(name + ".json")
         compile(files, null)

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractKapt3BuilderModeBytecodeShapeTest.kt

@@ -29,7 +29,7 @@ abstract class AbstractKapt3BuilderModeBytecodeShapeTest : CodegenTestCase() {
         }
     }
 
-    override fun doMultiFileTest(wholeFile: File, files: MutableList<TestFile>, javaFilesDir: File?) {
+    override fun doMultiFileTest(wholeFile: File, files: MutableList<TestFile>, javaFilesDir: File?, reportFailures: Boolean) {
         val txtFile = File(wholeFile.parentFile, wholeFile.nameWithoutExtension + ".txt")
         compile(files, javaFilesDir)
         KotlinTestUtils.assertEqualsToFile(txtFile, BytecodeListingTextCollectingVisitor.getText(classFileFactory))

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractLightAnalysisModeTest.kt

@@ -26,7 +26,7 @@ abstract class AbstractLightAnalysisModeTest : CodegenTestCase() {
         }
     }
 
-    override fun doMultiFileTest(wholeFile: File, files: List<CodegenTestCase.TestFile>, javaFilesDir: File?) {
+    override fun doMultiFileTest(wholeFile: File, files: List<CodegenTestCase.TestFile>, javaFilesDir: File?, reportFailures: Boolean) {
         for (file in files) {
             if (file.content.contains("// IGNORE_LIGHT_ANALYSIS")) {
                 return

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractLineNumberTest.kt

@@ -29,7 +29,7 @@ import kotlin.collections.ArrayList
 abstract class AbstractLineNumberTest : CodegenTestCase() {
 
     override fun doMultiFileTest(
-        wholeFile: File, files: MutableList<CodegenTestCase.TestFile>, javaFilesDir: File?
+        wholeFile: File, files: MutableList<CodegenTestCase.TestFile>, javaFilesDir: File?, reportFailures: Boolean
     ) {
         val isCustomTest = wholeFile.parentFile.name.equals("custom", ignoreCase = true)
         if (!isCustomTest) {
@@ -49,8 +49,10 @@ abstract class AbstractLineNumberTest : CodegenTestCase() {
                 KtUsefulTestCase.assertSameElements(actualLineNumbers, expectedLineNumbers)
             }
         } catch (e: Throwable) {
-            println(classFileFactory.createText())
+            if (reportFailures) {
-            throw e
+                println(classFileFactory.createText())
+                throw e
+            }
         }
     }
 

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractScriptCodegenTest.java

@@ -20,6 +20,7 @@ import com.intellij.openapi.util.Pair;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.kotlin.name.FqName;
 import org.jetbrains.kotlin.test.ConfigurationKind;
+import org.jetbrains.kotlin.test.TargetBackend;
 import org.jetbrains.kotlin.utils.ExceptionUtilsKt;
 
 import java.lang.reflect.Constructor;
@@ -33,7 +34,7 @@ public abstract class AbstractScriptCodegenTest extends CodegenTestCase {
     }
 
     @Override
-    protected void doTest(@NotNull String filename) {
+    protected void doTest(@NotNull String filename, TargetBackend targetBackend, boolean reportFailures) {
         loadFileByFullPath(filename);
 
         try {

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractTopLevelMembersInvocationTest.java

@@ -23,10 +23,7 @@ import kotlin.sequences.SequencesKt;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles;
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment;
-import org.jetbrains.kotlin.test.CompilerTestUtil;
+import org.jetbrains.kotlin.test.*;
-import org.jetbrains.kotlin.test.ConfigurationKind;
-import org.jetbrains.kotlin.test.KotlinTestUtils;
-import org.jetbrains.kotlin.test.TestJdkKind;
 
 import java.io.File;
 import java.util.Collections;
@@ -34,7 +31,7 @@ import java.util.List;
 
 public abstract class AbstractTopLevelMembersInvocationTest extends AbstractBytecodeTextTest {
     @Override
-    public void doTest(@NotNull String filename) throws Exception {
+    public void doTest(@NotNull String filename, TargetBackend targetBackend, boolean reportFailures) throws Exception {
         File root = new File(filename);
         List<String> sourceFiles = SequencesKt.toList(SequencesKt.map(
                 SequencesKt.filter(FilesKt.walkTopDown(root).maxDepth(1), File::isFile),

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/CodegenTestCase.java

@@ -34,10 +34,7 @@ import org.jetbrains.kotlin.fileClasses.JvmFileClassUtil;
 import org.jetbrains.kotlin.name.FqName;
 import org.jetbrains.kotlin.psi.KtFile;
 import org.jetbrains.kotlin.script.ScriptDependenciesProvider;
-import org.jetbrains.kotlin.test.ConfigurationKind;
+import org.jetbrains.kotlin.test.*;
-import org.jetbrains.kotlin.test.InTextDirectivesUtils;
-import org.jetbrains.kotlin.test.KotlinTestUtils;
-import org.jetbrains.kotlin.test.TestJdkKind;
 import org.jetbrains.kotlin.test.clientserver.TestProxy;
 import org.jetbrains.kotlin.test.testFramework.KtUsefulTestCase;
 import org.jetbrains.kotlin.utils.ExceptionUtilsKt;
@@ -397,13 +394,18 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
 
     @NotNull
     protected GeneratedClassLoader generateAndCreateClassLoader() {
+        return generateAndCreateClassLoader(true);
+    }
+
+    @NotNull
+    protected GeneratedClassLoader generateAndCreateClassLoader(boolean reportProblems) {
         if (initializedClassLoader != null) {
             fail("Double initialization of class loader in same test");
         }
 
         initializedClassLoader = createClassLoader();
 
-        if (!verifyAllFilesWithAsm(generateClassesInFile(), initializedClassLoader)) {
+        if (!verifyAllFilesWithAsm(generateClassesInFile(reportProblems), initializedClassLoader, reportProblems)) {
             fail("Verification failed: see exceptions above");
         }
 
@@ -486,11 +488,16 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
         FqName facadeClassFqName = JvmFileClassUtil.getFileClassInfoNoResolve(myFiles.getPsiFile()).getFacadeClassFqName();
         return generateClass(facadeClassFqName.asString());
     }
-
+    
     @NotNull
     protected Class<?> generateClass(@NotNull String name) {
+        return generateClass(name, true);
+    }
+
+    @NotNull
+    protected Class<?> generateClass(@NotNull String name, boolean reportProblems) {
         try {
-            return generateAndCreateClassLoader().loadClass(name);
+            return generateAndCreateClassLoader(reportProblems).loadClass(name);
         }
         catch (ClassNotFoundException e) {
             fail("No class file was generated for: " + name);
@@ -500,6 +507,11 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
 
     @NotNull
     protected ClassFileFactory generateClassesInFile() {
+        return generateClassesInFile(true);
+    }
+
+    @NotNull
+    protected ClassFileFactory generateClassesInFile(boolean reportProblems) {
         if (classFileFactory == null) {
             try {
                 GenerationState generationState = GenerationUtils.compileFiles(
@@ -513,22 +525,26 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
                 }
             }
             catch (Throwable e) {
-                e.printStackTrace();
+                if (reportProblems) {
-                System.err.println("Generating instructions as text...");
+                    e.printStackTrace();
-                try {
+                    System.err.println("Generating instructions as text...");
-                    if (classFileFactory == null) {
+                    try {
-                        System.err.println("Cannot generate text: exception was thrown during generation");
+                        if (classFileFactory == null) {
+                            System.err.println("Cannot generate text: exception was thrown during generation");
+                        }
+                        else {
+                            System.err.println(classFileFactory.createText());
+                        }
                     }
-                    else {
+                    catch (Throwable e1) {
-                        System.err.println(classFileFactory.createText());
+                        System.err.println("Exception thrown while trying to generate text, the actual exception follows:");
+                        e1.printStackTrace();
+                        System.err.println("-----------------------------------------------------------------------------");
                     }
+                    fail("See exceptions above");
+                } else {
+                    fail("Compilation failure");
                 }
-                catch (Throwable e1) {
-                    System.err.println("Exception thrown while trying to generate text, the actual exception follows:");
-                    e1.printStackTrace();
-                    System.err.println("-----------------------------------------------------------------------------");
-                }
-                fail("See exceptions above");
             }
         }
         return classFileFactory;
@@ -538,15 +554,15 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
         return true;
     }
 
-    protected static boolean verifyAllFilesWithAsm(ClassFileFactory factory, ClassLoader loader) {
+    protected static boolean verifyAllFilesWithAsm(ClassFileFactory factory, ClassLoader loader, boolean reportProblems) {
         boolean noErrors = true;
         for (OutputFile file : ClassFileUtilsKt.getClassFiles(factory)) {
-            noErrors &= verifyWithAsm(file, loader);
+            noErrors &= verifyWithAsm(file, loader, reportProblems);
         }
         return noErrors;
     }
 
-    private static boolean verifyWithAsm(@NotNull OutputFile file, ClassLoader loader) {
+    private static boolean verifyWithAsm(@NotNull OutputFile file, ClassLoader loader, boolean reportProblems) {
         ClassNode classNode = new ClassNode();
         new ClassReader(file.asByteArray()).accept(classNode, 0);
 
@@ -560,20 +576,22 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
                 analyzer.analyze(classNode.name, method);
             }
             catch (Throwable e) {
-                System.err.println(file.asText());
+                if (reportProblems) {
-                System.err.println(classNode.name + "::" + method.name + method.desc);
+                    System.err.println(file.asText());
+                    System.err.println(classNode.name + "::" + method.name + method.desc);
 
-                //noinspection InstanceofCatchParameter
+                    //noinspection InstanceofCatchParameter
-                if (e instanceof AnalyzerException) {
+                    if (e instanceof AnalyzerException) {
-                    // Print the erroneous instruction
+                        // Print the erroneous instruction
-                    TraceMethodVisitor tmv = new TraceMethodVisitor(new Textifier());
+                        TraceMethodVisitor tmv = new TraceMethodVisitor(new Textifier());
-                    ((AnalyzerException) e).node.accept(tmv);
+                        ((AnalyzerException) e).node.accept(tmv);
-                    PrintWriter pw = new PrintWriter(System.err);
+                        PrintWriter pw = new PrintWriter(System.err);
-                    tmv.p.print(pw);
+                        tmv.p.print(pw);
-                    pw.flush();
+                        pw.flush();
+                    }
+
+                    e.printStackTrace();
                 }
-
-                e.printStackTrace();
                 noErrors = false;
             }
         }
@@ -631,6 +649,14 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
     protected void compile(
             @NotNull List<TestFile> files,
             @Nullable File javaSourceDir
+    ) {
+        compile(files, javaSourceDir, true);
+    }
+
+    protected void compile(
+            @NotNull List<TestFile> files,
+            @Nullable File javaSourceDir,
+            boolean reportProblems
     ) {
         configurationKind = extractConfigurationKind(files);
         boolean loadAndroidAnnotations = files.stream().anyMatch(it ->
@@ -659,7 +685,7 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
 
         loadMultiFiles(files);
 
-        generateClassesInFile();
+        generateClassesInFile(reportProblems);
 
         if (javaSourceDir != null) {
             // If there are Java files, they should be compiled against the class files produced by Kotlin, so we dump them to the disk
@@ -759,14 +785,14 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
         }
     }
 
-    protected void doTest(String filePath) throws Exception {
+    protected void doTest(String filePath, TargetBackend targetBackend, boolean reportFailures) throws Exception {
         File file = new File(filePath);
         String expectedText = KotlinTestUtils.doLoadFile(file);
         Ref<File> javaFilesDir = Ref.create();
 
         List<TestFile> testFiles = createTestFiles(file, expectedText, javaFilesDir, "");
 
-        doMultiFileTest(file, testFiles, javaFilesDir.get());
+        doMultiFileTest(file, testFiles, javaFilesDir.get(), reportFailures);
     }
 
     protected void doTestWithCoroutinesPackageReplacement(String filePath, String packageName) throws Exception {
@@ -778,7 +804,7 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
 
         List<TestFile> testFiles = createTestFiles(file, expectedText, javaFilesDir, coroutinesPackage);
 
-        doMultiFileTest(file, testFiles, javaFilesDir.get());
+        doMultiFileTest(file, testFiles, javaFilesDir.get(), true);
     }
 
     @NotNull
@@ -813,7 +839,8 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
     protected void doMultiFileTest(
         @NotNull File wholeFile,
         @NotNull List<TestFile> files,
-        @Nullable File javaFilesDir
+        @Nullable File javaFilesDir,
+        boolean reportFailures
     ) throws Exception {
         throw new UnsupportedOperationException("Multi-file test cases are not supported in this test");
     }

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/defaultConstructor/AbstractDefaultArgumentsReflectionTest.java

@@ -19,6 +19,7 @@ package org.jetbrains.kotlin.codegen.defaultConstructor;
 import com.intellij.openapi.util.io.FileUtil;
 import org.jetbrains.kotlin.codegen.CodegenTestCase;
 import org.jetbrains.kotlin.test.ConfigurationKind;
+import org.jetbrains.kotlin.test.TargetBackend;
 
 import java.io.File;
 import java.io.IOException;
@@ -35,7 +36,7 @@ public abstract class AbstractDefaultArgumentsReflectionTest extends CodegenTest
     }
 
     @Override
-    protected void doTest(String path) throws IOException {
+    protected void doTest(String path, TargetBackend targetBackend, boolean reportFailures) throws IOException {
         loadFileByFullPath(path);
 
         String fileText = FileUtil.loadFile(new File(path), true);

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/flags/AbstractWriteFlagsTest.java

@@ -52,7 +52,7 @@ public abstract class AbstractWriteFlagsTest extends CodegenTestCase {
 
     @Override
     protected void doMultiFileTest(
-            @NotNull File wholeFile, @NotNull List<TestFile> files, @Nullable File javaFilesDir
+            @NotNull File wholeFile, @NotNull List<TestFile> files, @Nullable File javaFilesDir, boolean reportFailures
     ) throws Exception {
         compile(files, null);
 

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/ir/AbstractIrBlackBoxAgainstJavaCodegenTest.kt

@@ -10,7 +10,6 @@ import org.jetbrains.kotlin.codegen.AbstractBlackBoxAgainstJavaCodegenTest
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.JVMConfigurationKeys
 import org.jetbrains.kotlin.test.ConfigurationKind
-import java.io.File
 
 abstract class AbstractIrBlackBoxAgainstJavaCodegenTest : AbstractBlackBoxAgainstJavaCodegenTest() {
 

compiler/tests-common/tests/org/jetbrains/kotlin/ir/AbstractIrGeneratorTestCase.kt

@@ -42,7 +42,7 @@ import java.io.PrintWriter
 import java.util.*
 
 abstract class AbstractIrGeneratorTestCase : CodegenTestCase() {
-    override fun doMultiFileTest(wholeFile: File, files: List<TestFile>, javaFilesDir: File?) {
+    override fun doMultiFileTest(wholeFile: File, files: List<TestFile>, javaFilesDir: File?, reportFailures: Boolean) {
         setupEnvironment(files, javaFilesDir)
 
         loadMultiFiles(files)

compiler/tests-common/tests/org/jetbrains/kotlin/jvm/compiler/AbstractWriteSignatureTest.kt

@@ -16,13 +16,15 @@ import java.util.regex.MatchResult
 abstract class AbstractWriteSignatureTest : CodegenTestCase() {
 
 
-    override fun doMultiFileTest(wholeFile: File, files: MutableList<TestFile>, javaFilesDir: File?) {
+    override fun doMultiFileTest(wholeFile: File, files: MutableList<TestFile>, javaFilesDir: File?, reportFailures: Boolean) {
         compile(files, javaFilesDir)
         try {
             parseExpectations(wholeFile).check()
         } catch (e: Throwable) {
-            println(classFileFactory.createText())
+            if (reportFailures) {
-            throw e
+                println(classFileFactory.createText())
+                throw e
+            }
         }
     }
 

compiler/tests-common/tests/org/jetbrains/kotlin/test/KotlinTestUtils.java

@@ -106,6 +106,9 @@ public class KotlinTestUtils {
     private static final boolean DONT_IGNORE_TESTS_WORKING_ON_COMPATIBLE_BACKEND =
             Boolean.getBoolean("org.jetbrains.kotlin.dont.ignore.tests.working.on.compatible.backend");
 
+    public static final boolean SUPPRESS_EXPECTED_FAILURES =
+            Boolean.getBoolean("kotlin.suppress.expected.test.failures");
+
 
     private static final boolean AUTOMATICALLY_UNMUTE_PASSED_TESTS = false;
     private static final boolean AUTOMATICALLY_MUTE_FAILED_TESTS = false;
@@ -1022,12 +1025,26 @@ public class KotlinTestUtils {
         void invoke(String filePath) throws Exception;
     }
 
+    // Sometimes we need to pass additional parameters to the test.
+    // A two-argument function conflicts with ParsingTestGenerated, so let's have three.
+    public interface DoTest3 {
+        void invoke(String filePath, TargetBackend targetBackend, boolean reportFailures) throws Exception;
+    }
+
     // In this test runner version the `testDataFile` parameter is annotated by `TestDataFile`.
     // So only file paths passed to this parameter will be used in navigation actions, like "Navigate to testdata" and "Related Symbol..."
     public static void runTest(DoTest test, TargetBackend targetBackend, @TestDataFile String testDataFile) throws Exception {
         runTest0(test, targetBackend, testDataFile);
     }
 
+    public static void runTest(DoTest3 test, TargetBackend targetBackend, @TestDataFile String testDataFile) throws Exception {
+        runTest3(test, targetBackend, testDataFile);
+    }
+
+    public static void runTest0(DoTest test, TargetBackend targetBackend, String testDataFilePath) throws Exception {
+        runTest3((filePath, targetBackend2, reportFailures) -> test.invoke(filePath), targetBackend, testDataFilePath);
+    }
+
     // In this test runner version, NONE of the parameters are annotated by `TestDataFile`.
     // So DevKit will use test name to determine related files in navigation actions, like "Navigate to testdata" and "Related Symbol..."
     //
@@ -1036,7 +1053,7 @@ public class KotlinTestUtils {
     // Cons:
     // * sometimes, for too common/general names, it shows many variants to navigate
     // * it adds an additional step for navigation -- you must choose an exact file to navigate
-    public static void runTest0(DoTest test, TargetBackend targetBackend, String testDataFilePath) throws Exception {
+    private static void runTest3(DoTest3 test, TargetBackend targetBackend, String testDataFilePath) throws Exception {
         File testDataFile = new File(testDataFilePath);
 
         boolean isIgnored = isIgnoredTarget(targetBackend, testDataFile);
@@ -1048,8 +1065,10 @@ public class KotlinTestUtils {
             isIgnored &= isIgnoredTarget(targetBackend.getCompatibleWith(), testDataFile);
         }
 
+        boolean reportFailures = !isIgnored || !SUPPRESS_EXPECTED_FAILURES;
+
         try {
-            test.invoke(testDataFilePath);
+            test.invoke(testDataFilePath, targetBackend, reportFailures);
         }
         catch (Throwable e) {
 

compiler/tests-spec/tests/org/jetbrains/kotlin/codegen/AbstractBlackBoxCodegenTestSpec.kt

@@ -45,7 +45,7 @@ abstract class AbstractBlackBoxCodegenTestSpec : AbstractBlackBoxCodegenTest() {
         }
     }
 
-    override fun doMultiFileTest(wholeFile: File, files: MutableList<TestFile>, javaFilesDir: File?) {
+    override fun doMultiFileTest(wholeFile: File, files: MutableList<TestFile>, javaFilesDir: File?, reportFailures: Boolean) {
         val (specTest, _) = CommonParser.parseSpecTest(
             wholeFile.canonicalPath,
             mapOf("main.kt" to FileUtil.loadFile(wholeFile, true))
@@ -55,6 +55,6 @@ abstract class AbstractBlackBoxCodegenTestSpec : AbstractBlackBoxCodegenTest() {
 
         includeHelpers(wholeFile, files)
 
-        super.doMultiFileTest(wholeFile, files, javaFilesDir)
+        super.doMultiFileTest(wholeFile, files, javaFilesDir, reportFailures)
     }
 }

compiler/tests/org/jetbrains/kotlin/codegen/AbstractCustomScriptCodegenTest.kt

@@ -41,7 +41,7 @@ abstract class AbstractCustomScriptCodegenTest : CodegenTestCase() {
         configuration.addJvmClasspathRoots(additionalDependencies.orEmpty())
     }
 
-    override fun doMultiFileTest(wholeFile: File, files: MutableList<TestFile>, javaFilesDir: File?) {
+    override fun doMultiFileTest(wholeFile: File, files: MutableList<TestFile>, javaFilesDir: File?, reportFailures: Boolean) {
         if (files.size > 1) {
             throw UnsupportedOperationException("Multiple files are not yet supported in this test")
         }
@@ -78,8 +78,10 @@ abstract class AbstractCustomScriptCodegenTest : CodegenTestCase() {
             val expectedFields = extractAllKeyValPairs(content, "expected:")
             checkExpectedFields(expectedFields, scriptClass, scriptInstance)
         } catch (e: Throwable) {
-            println(generateToText())
+            if (reportFailures) {
-            throw e
+                println(generateToText())
+                throw e
+            }
         }
     }
 

compiler/tests/org/jetbrains/kotlin/codegen/Java9CodegenTest.kt

@@ -25,7 +25,7 @@ class Java9CodegenTest : AbstractBlackBoxCodegenTest() {
 
     override fun getPrefix(): String = "java9/box"
 
-    override fun blackBox() {
+    override fun blackBox(reportFailures: Boolean) {
         val tmpdir = KotlinTestUtils.tmpDirForTest(this)
         generateClassesInFile().writeAll(tmpdir, null)
 
@@ -49,6 +49,6 @@ class Java9CodegenTest : AbstractBlackBoxCodegenTest() {
 
     fun testVarHandle() {
         loadFile()
-        blackBox()
+        blackBox(true)
     }
 }