Switch off logging for failing IrBlackBox tests

Logging of tests that are expected to fail is controlled by
kotlin.suppress.expected.test.failures project property.

compiler/build.gradle.kts

@@ -92,6 +92,7 @@ projectTest {
     dependsOn(*testDistProjects.map { "$it:dist" }.toTypedArray())
     workingDir = rootDir
     systemProperty("kotlin.test.script.classpath", testSourceSet.output.classesDirs.joinToString(File.pathSeparator))
+    systemProperty("kotlin.suppress.expected.test.failures", project.findProperty("kotlin.suppress.expected.test.failures") ?: false)
     doFirst {
         systemProperty("kotlin.ant.classpath", antLauncherJar.asPath)
         systemProperty("kotlin.ant.launcher.class", "org.apache.tools.ant.Main")

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractBlackBoxCodegenTest.java

@@ -14,6 +14,7 @@ import org.jetbrains.kotlin.fileClasses.JvmFileClassUtil;
 import org.jetbrains.kotlin.psi.*;
 import org.jetbrains.kotlin.resolve.BindingContext;
 import org.jetbrains.kotlin.test.InTextDirectivesUtils;
+import org.jetbrains.kotlin.test.TargetBackend;
 import org.jetbrains.kotlin.utils.ExceptionUtilsKt;
 
 import java.io.File;
@@ -27,22 +28,28 @@ import static org.jetbrains.kotlin.test.clientserver.TestProcessServerKt.getGene
 
 public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase {
 
+    public static final boolean IGNORE_EXPECTED_FAILURES =
+            Boolean.getBoolean("kotlin.suppress.expected.test.failures");
+
     @Override
     protected void doMultiFileTest(
         @NotNull File wholeFile,
         @NotNull List<TestFile> files,
         @Nullable File javaFilesDir
     ) throws Exception {
+        boolean isIgnored = IGNORE_EXPECTED_FAILURES && InTextDirectivesUtils.isIgnoredTarget(getBackend(), wholeFile);
         try {
-            compile(files, javaFilesDir);
+            compile(files, javaFilesDir, !isIgnored);
-            blackBox();
+            blackBox(!isIgnored);
         }
         catch (Throwable t) {
-            try {
+            if (!isIgnored) {
-                // To create .txt file in case of failure
+                try {
-                doBytecodeListingTest(wholeFile);
+                    // To create .txt file in case of failure
-            }
+                    doBytecodeListingTest(wholeFile);
-            catch (Throwable ignored) {
+                }
+                catch (Throwable ignored) {
+                }
             }
 
             throw t;
@@ -89,9 +96,9 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase {
         assertEqualsToFile(expectedFile, text, s -> s.replace("COROUTINES_PACKAGE", coroutinesPackage));
     }
 
-    protected void blackBox() {
+    protected void blackBox(boolean reportProblems) {
         // If there are many files, the first 'box(): String' function will be executed.
-        GeneratedClassLoader generatedClassLoader = generateAndCreateClassLoader();
+        GeneratedClassLoader generatedClassLoader = generateAndCreateClassLoader(reportProblems);
         for (KtFile firstFile : myFiles.getPsiFiles()) {
             String className = getFacadeFqName(firstFile, classFileFactory.getGenerationState().getBindingContext());
             if (className == null) continue;
@@ -104,7 +111,9 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase {
                 }
             }
             catch (Throwable e) {
-                System.out.println(generateToText());
+                if (reportProblems) {
+                    System.out.println(generateToText());
+                }
                 throw ExceptionUtilsKt.rethrow(e);
             }
             finally {
@@ -123,4 +132,8 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase {
         }
         return null;
     }
+
+    protected TargetBackend getBackend() {
+        return TargetBackend.JVM;
+    }
 }

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/CodegenTestCase.java

@@ -397,13 +397,18 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
 
     @NotNull
     protected GeneratedClassLoader generateAndCreateClassLoader() {
+        return generateAndCreateClassLoader(true);
+    }
+
+    @NotNull
+    protected GeneratedClassLoader generateAndCreateClassLoader(boolean reportProblems) {
         if (initializedClassLoader != null) {
             fail("Double initialization of class loader in same test");
         }
 
         initializedClassLoader = createClassLoader();
 
-        if (!verifyAllFilesWithAsm(generateClassesInFile(), initializedClassLoader)) {
+        if (!verifyAllFilesWithAsm(generateClassesInFile(reportProblems), initializedClassLoader, reportProblems)) {
             fail("Verification failed: see exceptions above");
         }
 
@@ -489,8 +494,13 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
     
     @NotNull
     protected Class<?> generateClass(@NotNull String name) {
+        return generateClass(name, true);
+    }
+
+    @NotNull
+    protected Class<?> generateClass(@NotNull String name, boolean reportProblems) {
         try {
-            return generateAndCreateClassLoader().loadClass(name);
+            return generateAndCreateClassLoader(reportProblems).loadClass(name);
         }
         catch (ClassNotFoundException e) {
             fail("No class file was generated for: " + name);
@@ -500,6 +510,11 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
 
     @NotNull
     protected ClassFileFactory generateClassesInFile() {
+        return generateClassesInFile(true);
+    }
+
+    @NotNull
+    protected ClassFileFactory generateClassesInFile(boolean reportProblems) {
         if (classFileFactory == null) {
             try {
                 GenerationState generationState = GenerationUtils.compileFiles(
@@ -513,22 +528,26 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
                 }
             }
             catch (Throwable e) {
-                e.printStackTrace();
+                if (reportProblems) {
-                System.err.println("Generating instructions as text...");
+                    e.printStackTrace();
-                try {
+                    System.err.println("Generating instructions as text...");
-                    if (classFileFactory == null) {
+                    try {
-                        System.err.println("Cannot generate text: exception was thrown during generation");
+                        if (classFileFactory == null) {
+                            System.err.println("Cannot generate text: exception was thrown during generation");
+                        }
+                        else {
+                            System.err.println(classFileFactory.createText());
+                        }
                     }
-                    else {
+                    catch (Throwable e1) {
-                        System.err.println(classFileFactory.createText());
+                        System.err.println("Exception thrown while trying to generate text, the actual exception follows:");
+                        e1.printStackTrace();
+                        System.err.println("-----------------------------------------------------------------------------");
                     }
+                    fail("See exceptions above");
+                } else {
+                    fail("Compilation failure");
                 }
-                catch (Throwable e1) {
-                    System.err.println("Exception thrown while trying to generate text, the actual exception follows:");
-                    e1.printStackTrace();
-                    System.err.println("-----------------------------------------------------------------------------");
-                }
-                fail("See exceptions above");
             }
         }
         return classFileFactory;
@@ -538,15 +557,15 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
         return true;
     }
 
-    protected static boolean verifyAllFilesWithAsm(ClassFileFactory factory, ClassLoader loader) {
+    protected static boolean verifyAllFilesWithAsm(ClassFileFactory factory, ClassLoader loader, boolean reportProblems) {
         boolean noErrors = true;
         for (OutputFile file : ClassFileUtilsKt.getClassFiles(factory)) {
-            noErrors &= verifyWithAsm(file, loader);
+            noErrors &= verifyWithAsm(file, loader, reportProblems);
         }
         return noErrors;
     }
 
-    private static boolean verifyWithAsm(@NotNull OutputFile file, ClassLoader loader) {
+    private static boolean verifyWithAsm(@NotNull OutputFile file, ClassLoader loader, boolean reportProblems) {
         ClassNode classNode = new ClassNode();
         new ClassReader(file.asByteArray()).accept(classNode, 0);
 
@@ -560,20 +579,22 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
                 analyzer.analyze(classNode.name, method);
             }
             catch (Throwable e) {
-                System.err.println(file.asText());
+                if (reportProblems) {
-                System.err.println(classNode.name + "::" + method.name + method.desc);
+                    System.err.println(file.asText());
+                    System.err.println(classNode.name + "::" + method.name + method.desc);
 
-                //noinspection InstanceofCatchParameter
+                    //noinspection InstanceofCatchParameter
-                if (e instanceof AnalyzerException) {
+                    if (e instanceof AnalyzerException) {
-                    // Print the erroneous instruction
+                        // Print the erroneous instruction
-                    TraceMethodVisitor tmv = new TraceMethodVisitor(new Textifier());
+                        TraceMethodVisitor tmv = new TraceMethodVisitor(new Textifier());
-                    ((AnalyzerException) e).node.accept(tmv);
+                        ((AnalyzerException) e).node.accept(tmv);
-                    PrintWriter pw = new PrintWriter(System.err);
+                        PrintWriter pw = new PrintWriter(System.err);
-                    tmv.p.print(pw);
+                        tmv.p.print(pw);
-                    pw.flush();
+                        pw.flush();
+                    }
+
+                    e.printStackTrace();
                 }
-
-                e.printStackTrace();
                 noErrors = false;
             }
         }
@@ -631,6 +652,14 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
     protected void compile(
             @NotNull List<TestFile> files,
             @Nullable File javaSourceDir
+    ) {
+        compile(files, javaSourceDir, true);
+    }
+
+    protected void compile(
+            @NotNull List<TestFile> files,
+            @Nullable File javaSourceDir,
+            boolean reportProblems
     ) {
         configurationKind = extractConfigurationKind(files);
         boolean loadAndroidAnnotations = files.stream().anyMatch(it ->
@@ -659,7 +688,7 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
 
         loadMultiFiles(files);
 
-        generateClassesInFile();
+        generateClassesInFile(reportProblems);
 
         if (javaSourceDir != null) {
             // If there are Java files, they should be compiled against the class files produced by Kotlin, so we dump them to the disk

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/ir/AbstractIrBlackBoxAgainstJavaCodegenTest.kt

@@ -10,6 +10,7 @@ import org.jetbrains.kotlin.codegen.AbstractBlackBoxAgainstJavaCodegenTest
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.JVMConfigurationKeys
 import org.jetbrains.kotlin.test.ConfigurationKind
+import org.jetbrains.kotlin.test.TargetBackend
 import java.io.File
 
 abstract class AbstractIrBlackBoxAgainstJavaCodegenTest : AbstractBlackBoxAgainstJavaCodegenTest() {
@@ -22,4 +23,6 @@ abstract class AbstractIrBlackBoxAgainstJavaCodegenTest : AbstractBlackBoxAgains
     override fun extractConfigurationKind(files: MutableList<TestFile>): ConfigurationKind {
         return ConfigurationKind.ALL
     }
+
+    override fun getBackend() = TargetBackend.JVM_IR
 }

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/ir/AbstractIrBlackBoxCodegenTest.kt

@@ -20,6 +20,7 @@ import org.jetbrains.kotlin.codegen.AbstractBlackBoxCodegenTest
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.JVMConfigurationKeys
 import org.jetbrains.kotlin.test.ConfigurationKind
+import org.jetbrains.kotlin.test.TargetBackend
 
 abstract class AbstractIrBlackBoxCodegenTest : AbstractBlackBoxCodegenTest() {
     override fun updateConfiguration(configuration: CompilerConfiguration) {
@@ -31,4 +32,6 @@ abstract class AbstractIrBlackBoxCodegenTest : AbstractBlackBoxCodegenTest() {
     override fun extractConfigurationKind(files: MutableList<TestFile>): ConfigurationKind {
         return ConfigurationKind.ALL
     }
+
+    override fun getBackend() = TargetBackend.JVM_IR
 }

compiler/tests/org/jetbrains/kotlin/codegen/Java9CodegenTest.kt

@@ -25,7 +25,7 @@ class Java9CodegenTest : AbstractBlackBoxCodegenTest() {
 
     override fun getPrefix(): String = "java9/box"
 
-    override fun blackBox() {
+    override fun blackBox(reportFailures: Boolean) {
         val tmpdir = KotlinTestUtils.tmpDirForTest(this)
         generateClassesInFile().writeAll(tmpdir, null)
 
@@ -49,6 +49,6 @@ class Java9CodegenTest : AbstractBlackBoxCodegenTest() {
 
     fun testVarHandle() {
         loadFile()
-        blackBox()
+        blackBox(true)
     }
 }

plugins/android-extensions/android-extensions-compiler/test/org/jetbrains/android/synthetic/test/AbstractAndroidBoxTest.kt

@@ -106,6 +106,6 @@ abstract class AbstractAndroidBoxTest : AbstractBlackBoxCodegenTest() {
                 ArrayUtil.toStringArray(files),
                 KotlinTestUtils.getHomeDirectory() + "/plugins/android-extensions/android-extensions-compiler/testData"
         )
-        blackBox()
+        blackBox(true)
     }
 }