Add black box spec tests validation

compiler/tests-common/tests/org/jetbrains/kotlin/TestExceptionsComparator.kt

@@ -67,8 +67,7 @@ class TestExceptionsComparator(wholeFile: File) {
                 (e.original.cause ?: e.original).run {
                     casesAsString + toString() + stackTrace[0]?.let { ls + it }
                 }
-            is TestsCompilerError, is TestsInfrastructureError -> casesAsString + (e.original.cause ?: e.original).toString()
+            is TestsCompilerError, is TestsCompiletimeError, is TestsInfrastructureError -> casesAsString + (e.original.cause ?: e.original).toString()
-            is TestsCompiletimeError -> throw e.original
         }
     }
 

compiler/tests-common/tests/org/jetbrains/kotlin/codegen/CodegenTestCase.java

@@ -546,10 +546,11 @@ public abstract class CodegenTestCase extends KtUsefulTestCase {
                         e1.printStackTrace();
                         System.err.println("-----------------------------------------------------------------------------");
                     }
-                    fail("See exceptions above");
+                    System.err.println("See exceptions above");
                 } else {
-                    fail("Compilation failure");
+                    System.err.println("Compilation failure");
                 }
+                throw e;
             } catch (Throwable e) {
                 throw new TestsCompilerError(e);
             }

compiler/tests-spec/tests/org/jetbrains/kotlin/codegen/AbstractBlackBoxCodegenTestSpec.kt

@@ -10,7 +10,10 @@ import org.jetbrains.kotlin.TestExceptionsComparator
 import org.jetbrains.kotlin.spec.parsers.CommonParser
 import org.jetbrains.kotlin.spec.parsers.CommonPatterns.packagePattern
 import org.jetbrains.kotlin.spec.utils.GeneralConfiguration.TESTDATA_PATH
+import org.jetbrains.kotlin.spec.validators.BlackBoxTestTypeValidator
+import org.jetbrains.kotlin.spec.validators.SpecTestValidationException
 import org.jetbrains.kotlin.test.InTextDirectivesUtils
+import org.junit.Assert
 import java.io.*
 
 abstract class AbstractBlackBoxCodegenTestSpec : AbstractBlackBoxCodegenTest() {
@@ -47,11 +50,19 @@ abstract class AbstractBlackBoxCodegenTestSpec : AbstractBlackBoxCodegenTest() {
     }
 
     override fun doMultiFileTest(wholeFile: File, files: MutableList<TestFile>, javaFilesDir: File?) {
-        val (specTest, _) = CommonParser.parseSpecTest(
+        val (specTest, testLinkedType) = CommonParser.parseSpecTest(
             wholeFile.canonicalPath,
             mapOf("main.kt" to FileUtil.loadFile(wholeFile, true))
         )
 
+        val validator = BlackBoxTestTypeValidator(wholeFile, specTest)
+
+        try {
+            validator.validatePathConsistency(testLinkedType)
+        } catch (e: SpecTestValidationException) {
+            Assert.fail(e.description)
+        }
+
         println(specTest)
 
         includeHelpers(wholeFile, files)

compiler/tests-spec/tests/org/jetbrains/kotlin/spec/parsers/CommonParser.kt

@@ -21,7 +21,10 @@ import java.io.File
 import java.util.regex.Pattern
 
 object CommonParser {
-    fun String.withUnderscores() = replace(" ", "_").toUpperCase()
+    fun String.withUnderscores() = replace(" ", "_")
+        .replace(File.separator, "_")
+        .toUpperCase()
+
     fun String.splitByComma() = split(Regex(""",\s*"""))
     fun String.splitByPathSeparator() = split(File.separator)
     fun String.withSpaces() = replace("_", " ")

compiler/tests-spec/tests/org/jetbrains/kotlin/spec/validators/BlackBoxTestTypeValidator.kt

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.spec.validators
+
+import org.jetbrains.kotlin.spec.models.AbstractSpecTest
+import java.io.File
+
+class BlackBoxTestTypeValidator(
+    testDataFile: File,
+    private val testInfo: AbstractSpecTest
+) : AbstractTestValidator(testInfo, testDataFile) {
+    override fun computeTestTypes() = mapOf(1 to testInfo.testType)
+}