From 9487d291da23694da3720f87dea3529e6a1e3ec5 Mon Sep 17 00:00:00 2001 From: "victor.petukhov" Date: Mon, 19 Aug 2019 11:54:34 +0300 Subject: [PATCH] Fail spec box tests if they have `unexpected behaviour` and passed --- .../kotlin/TestExceptionsComparator.kt | 9 ++++--- .../codegen/AbstractBlackBoxCodegenTest.java | 25 +++++++++++++++---- .../kotlin/codegen/CodegenTestCase.java | 17 +++++++++++-- .../AbstractBlackBoxCodegenTestSpec.kt | 23 +++++++++-------- 4 files changed, 53 insertions(+), 21 deletions(-) diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/TestExceptionsComparator.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/TestExceptionsComparator.kt index 32054371d1d..8d8a7475519 100644 --- a/compiler/tests-common/tests/org/jetbrains/kotlin/TestExceptionsComparator.kt +++ b/compiler/tests-common/tests/org/jetbrains/kotlin/TestExceptionsComparator.kt @@ -1,5 +1,5 @@ /* - * Copyright 2010-2018 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ @@ -98,10 +98,13 @@ class TestExceptionsComparator(wholeFile: File) { } } + fun run(expectedException: TestsExceptionType?, runnable: () -> Unit) = + run(expectedException, mapOf(), null, runnable) + fun run( expectedException: TestsExceptionType?, - exceptionByCases: Map = mapOf(), - computeExceptionPoint: ((Matcher?) -> Set?)? = null, + exceptionByCases: Map, + computeExceptionPoint: ((Matcher?) -> Set?)?, runnable: () -> Unit ) { try { diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractBlackBoxCodegenTest.java b/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractBlackBoxCodegenTest.java index 8eb9744c5f2..d5c390c9ae9 100644 --- a/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractBlackBoxCodegenTest.java +++ b/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/AbstractBlackBoxCodegenTest.java @@ -27,14 +27,17 @@ import static org.jetbrains.kotlin.test.clientserver.TestProcessServerKt.getBoxM import static org.jetbrains.kotlin.test.clientserver.TestProcessServerKt.getGeneratedClass; public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase { - @Override - protected void doMultiFileTest(@NotNull File wholeFile, @NotNull List files) throws Exception { + protected void doMultiFileTest( + @NotNull File wholeFile, + @NotNull List files, + boolean unexpectedBehaviour + ) throws Exception { boolean isIgnored = InTextDirectivesUtils.isIgnoredTarget(getBackend(), wholeFile); compile(files, !isIgnored); try { - blackBox(!isIgnored); + blackBox(!isIgnored, unexpectedBehaviour); } catch (Throwable t) { if (!isIgnored) { @@ -52,6 +55,14 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase { doBytecodeListingTest(wholeFile); } + @Override + protected void doMultiFileTest( + @NotNull File wholeFile, + @NotNull List files + ) throws Exception { + doMultiFileTest(wholeFile, files, false); + } + private void doBytecodeListingTest(@NotNull File wholeFile) throws Exception { if (!InTextDirectivesUtils.isDirectiveDefined(FileUtil.loadFile(wholeFile), "CHECK_BYTECODE_LISTING")) return; @@ -90,7 +101,7 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase { assertEqualsToFile(expectedFile, text, s -> s.replace("COROUTINES_PACKAGE", coroutinesPackage)); } - protected void blackBox(boolean reportProblems) { + protected void blackBox(boolean reportProblems, boolean unexpectedBehaviour) { // If there are many files, the first 'box(): String' function will be executed. GeneratedClassLoader generatedClassLoader = generateAndCreateClassLoader(reportProblems); for (KtFile firstFile : myFiles.getPsiFiles()) { @@ -100,7 +111,7 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase { try { Method method = getBoxMethodOrNull(aClass); if (method != null) { - callBoxMethodAndCheckResult(generatedClassLoader, aClass, method); + callBoxMethodAndCheckResult(generatedClassLoader, aClass, method, unexpectedBehaviour); return; } } @@ -117,6 +128,10 @@ public abstract class AbstractBlackBoxCodegenTest extends CodegenTestCase { fail("Can't find box method!"); } + protected void blackBox(boolean reportProblems) { + blackBox(reportProblems, false); + } + @Nullable private static String getFacadeFqName(@NotNull KtFile file) { return CodegenUtil.getMemberDeclarationsToGenerate(file).isEmpty() diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/CodegenTestCase.java b/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/CodegenTestCase.java index c03bdf26e53..bb2ef898ddb 100644 --- a/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/CodegenTestCase.java +++ b/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/CodegenTestCase.java @@ -838,8 +838,17 @@ public abstract class CodegenTestCase extends KtUsefulTestCase { callBoxMethodAndCheckResult(classLoader, aClass, method); } - protected void callBoxMethodAndCheckResult(URLClassLoader classLoader, Class aClass, Method method) + private void callBoxMethodAndCheckResult(URLClassLoader classLoader, Class aClass, Method method) throws IOException, IllegalAccessException, InvocationTargetException { + callBoxMethodAndCheckResult(classLoader, aClass, method, false); + } + + protected void callBoxMethodAndCheckResult( + URLClassLoader classLoader, + Class aClass, + Method method, + boolean unexpectedBehaviour + ) throws IOException, IllegalAccessException, InvocationTargetException { String result; if (BOX_IN_SEPARATE_PROCESS_PORT != null) { result = invokeBoxInSeparateProcess(classLoader, aClass); @@ -859,7 +868,11 @@ public abstract class CodegenTestCase extends KtUsefulTestCase { } } } - assertEquals("OK", result); + if (unexpectedBehaviour) { + assertNotSame("OK", result); + } else { + assertEquals("OK", result); + } } @NotNull diff --git a/compiler/tests-spec/tests/org/jetbrains/kotlin/spec/codegen/AbstractBlackBoxCodegenTestSpec.kt b/compiler/tests-spec/tests/org/jetbrains/kotlin/spec/codegen/AbstractBlackBoxCodegenTestSpec.kt index 50abb0dbbd3..28406467845 100644 --- a/compiler/tests-spec/tests/org/jetbrains/kotlin/spec/codegen/AbstractBlackBoxCodegenTestSpec.kt +++ b/compiler/tests-spec/tests/org/jetbrains/kotlin/spec/codegen/AbstractBlackBoxCodegenTestSpec.kt @@ -3,18 +3,19 @@ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ -package org.jetbrains.kotlin.codegen +package org.jetbrains.kotlin.spec.codegen import com.intellij.openapi.util.io.FileUtil import org.jetbrains.kotlin.TestExceptionsComparator -import org.jetbrains.kotlin.spec.models.AbstractSpecTest -import org.jetbrains.kotlin.spec.parsers.CommonParser -import org.jetbrains.kotlin.spec.parsers.CommonPatterns.packagePattern +import org.jetbrains.kotlin.codegen.AbstractBlackBoxCodegenTest import org.jetbrains.kotlin.spec.utils.GeneralConfiguration.TESTDATA_PATH -import org.jetbrains.kotlin.spec.validators.BlackBoxTestTypeValidator -import org.jetbrains.kotlin.spec.validators.SpecTestValidationException +import org.jetbrains.kotlin.spec.utils.models.AbstractSpecTest +import org.jetbrains.kotlin.spec.utils.parsers.CommonParser +import org.jetbrains.kotlin.spec.utils.parsers.CommonPatterns.packagePattern +import org.jetbrains.kotlin.spec.utils.validators.BlackBoxTestTypeValidator +import org.jetbrains.kotlin.spec.utils.validators.SpecTestValidationException import org.junit.Assert -import java.io.File +import java.io.* abstract class AbstractBlackBoxCodegenTestSpec : AbstractBlackBoxCodegenTest() { companion object { @@ -61,12 +62,12 @@ abstract class AbstractBlackBoxCodegenTestSpec : AbstractBlackBoxCodegenTest() { includeHelpers(wholeFile, files, specTest) + val runTest = { super.doMultiFileTest(wholeFile, files, specTest.unexpectedBehavior) } + if (specTest.exception == null) { - super.doMultiFileTest(wholeFile, files) + runTest() } else { - TestExceptionsComparator(wholeFile).run(specTest.exception) { - super.doMultiFileTest(wholeFile, files) - } + TestExceptionsComparator(wholeFile).run(specTest.exception, runTest) } } }