Fail spec box tests if they have unexpected behaviour and passed

compiler/tests-common/tests/org/jetbrains/kotlin/TestExceptionsComparator.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2010-2018 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
@@ -98,10 +98,13 @@ class TestExceptionsComparator(wholeFile: File) {
         }
     }
 
+    fun run(expectedException: TestsExceptionType?, runnable: () -> Unit) =
+        run(expectedException, mapOf(), null, runnable)
+
     fun run(
         expectedException: TestsExceptionType?,
-        exceptionByCases: Map<Int, TestsExceptionType?> = mapOf(),
+        exceptionByCases: Map<Int, TestsExceptionType?>,
-        computeExceptionPoint: ((Matcher?) -> Set<Int>?)? = null,
+        computeExceptionPoint: ((Matcher?) -> Set<Int>?)?,
         runnable: () -> Unit
     ) {
         try {

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractBlackBoxCodegenTest.java

@@ -27,14 +27,17 @@ import static org.jetbrains.kotlin.test.clientserver.TestProcessServerKt.getBoxM
 import static org.jetbrains.kotlin.test.clientserver.TestProcessServerKt.getGeneratedClass;
 
 public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase {
-    @Override
+    protected void doMultiFileTest(
-    protected void doMultiFileTest(@NotNull File wholeFile, @NotNull List<TestFile> files) throws Exception {
+            @NotNull File wholeFile,
+            @NotNull List<TestFile> files,
+            boolean unexpectedBehaviour
+    ) throws Exception {
         boolean isIgnored = InTextDirectivesUtils.isIgnoredTarget(getBackend(), wholeFile);
 
         compile(files, !isIgnored);
 
         try {
-            blackBox(!isIgnored);
+            blackBox(!isIgnored, unexpectedBehaviour);
         }
         catch (Throwable t) {
             if (!isIgnored) {
@@ -52,6 +55,14 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase {
         doBytecodeListingTest(wholeFile);
     }
 
+    @Override
+    protected void doMultiFileTest(
+        @NotNull File wholeFile,
+        @NotNull List<TestFile> files
+    ) throws Exception {
+        doMultiFileTest(wholeFile, files, false);
+    }
+
     private void doBytecodeListingTest(@NotNull File wholeFile) throws Exception {
         if (!InTextDirectivesUtils.isDirectiveDefined(FileUtil.loadFile(wholeFile), "CHECK_BYTECODE_LISTING")) return;
 
@@ -90,7 +101,7 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase {
         assertEqualsToFile(expectedFile, text, s -> s.replace("COROUTINES_PACKAGE", coroutinesPackage));
     }
 
-    protected void blackBox(boolean reportProblems) {
+    protected void blackBox(boolean reportProblems, boolean unexpectedBehaviour) {
         // If there are many files, the first 'box(): String' function will be executed.
         GeneratedClassLoader generatedClassLoader = generateAndCreateClassLoader(reportProblems);
         for (KtFile firstFile : myFiles.getPsiFiles()) {
@@ -100,7 +111,7 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase {
             try {
                 Method method = getBoxMethodOrNull(aClass);
                 if (method != null) {
-                    callBoxMethodAndCheckResult(generatedClassLoader, aClass, method);
+                    callBoxMethodAndCheckResult(generatedClassLoader, aClass, method, unexpectedBehaviour);
                     return;
                 }
             }
@@ -117,6 +128,10 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase {
         fail("Can't find box method!");
     }
 
+    protected void blackBox(boolean reportProblems) {
+        blackBox(reportProblems, false);
+    }
+
     @Nullable
     private static String getFacadeFqName(@NotNull KtFile file) {
         return CodegenUtil.getMemberDeclarationsToGenerate(file).isEmpty()

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/CodegenTestCase.java

@@ -838,8 +838,17 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
         callBoxMethodAndCheckResult(classLoader, aClass, method);
     }
 
-    protected void callBoxMethodAndCheckResult(URLClassLoader classLoader, Class<?> aClass, Method method)
+    private void callBoxMethodAndCheckResult(URLClassLoader classLoader, Class<?> aClass, Method method)
             throws IOException, IllegalAccessException, InvocationTargetException {
+        callBoxMethodAndCheckResult(classLoader, aClass, method, false);
+    }
+
+    protected void callBoxMethodAndCheckResult(
+            URLClassLoader classLoader,
+            Class<?> aClass,
+            Method method,
+            boolean unexpectedBehaviour
+    ) throws IOException, IllegalAccessException, InvocationTargetException {
         String result;
         if (BOX_IN_SEPARATE_PROCESS_PORT != null) {
             result = invokeBoxInSeparateProcess(classLoader, aClass);
@@ -859,7 +868,11 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
                 }
             }
         }
-        assertEquals("OK", result);
+        if (unexpectedBehaviour) {
+            assertNotSame("OK", result);
+        } else {
+            assertEquals("OK", result);
+        }
     }
 
     @NotNull

compiler/tests-spec/tests/org/jetbrains/kotlin/spec/codegen/AbstractBlackBoxCodegenTestSpec.kt

@@ -3,18 +3,19 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
-package org.jetbrains.kotlin.codegen
+package org.jetbrains.kotlin.spec.codegen
 
 import com.intellij.openapi.util.io.FileUtil
 import org.jetbrains.kotlin.TestExceptionsComparator
-import org.jetbrains.kotlin.spec.models.AbstractSpecTest
+import org.jetbrains.kotlin.codegen.AbstractBlackBoxCodegenTest
-import org.jetbrains.kotlin.spec.parsers.CommonParser
-import org.jetbrains.kotlin.spec.parsers.CommonPatterns.packagePattern
 import org.jetbrains.kotlin.spec.utils.GeneralConfiguration.TESTDATA_PATH
-import org.jetbrains.kotlin.spec.validators.BlackBoxTestTypeValidator
+import org.jetbrains.kotlin.spec.utils.models.AbstractSpecTest
-import org.jetbrains.kotlin.spec.validators.SpecTestValidationException
+import org.jetbrains.kotlin.spec.utils.parsers.CommonParser
+import org.jetbrains.kotlin.spec.utils.parsers.CommonPatterns.packagePattern
+import org.jetbrains.kotlin.spec.utils.validators.BlackBoxTestTypeValidator
+import org.jetbrains.kotlin.spec.utils.validators.SpecTestValidationException
 import org.junit.Assert
-import java.io.File
+import java.io.*
 
 abstract class AbstractBlackBoxCodegenTestSpec : AbstractBlackBoxCodegenTest() {
     companion object {
@@ -61,12 +62,12 @@ abstract class AbstractBlackBoxCodegenTestSpec : AbstractBlackBoxCodegenTest() {
 
         includeHelpers(wholeFile, files, specTest)
 
+        val runTest = { super.doMultiFileTest(wholeFile, files, specTest.unexpectedBehavior) }
+
         if (specTest.exception == null) {
-            super.doMultiFileTest(wholeFile, files)
+            runTest()
         } else {
-            TestExceptionsComparator(wholeFile).run(specTest.exception) {
+            TestExceptionsComparator(wholeFile).run(specTest.exception, runTest)
-                super.doMultiFileTest(wholeFile, files)
-            }
         }
     }
 }