Support filtering diagnostics in tests

compiler/frontend/src/org/jetbrains/jet/checkers/CheckerTestUtil.java

@@ -345,7 +345,7 @@ public class CheckerTestUtil {
         @NotNull
         @Override
         public Severity getSeverity() {
-            throw new IllegalStateException();
+            return Severity.ERROR;
         }
 
         @NotNull
@@ -372,7 +372,7 @@ public class CheckerTestUtil {
         }
     }
 
-    private static class SyntaxErrorDiagnosticFactory extends AbstractDiagnosticFactory {
+    public static class SyntaxErrorDiagnosticFactory extends AbstractDiagnosticFactory {
         public static final SyntaxErrorDiagnosticFactory INSTANCE = new SyntaxErrorDiagnosticFactory();
 
         private SyntaxErrorDiagnosticFactory() {}

compiler/tests/org/jetbrains/jet/JetTestUtils.java

@@ -18,6 +18,7 @@ package org.jetbrains.jet;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.intellij.openapi.Disposable;
 import com.intellij.openapi.editor.Document;
@@ -92,6 +93,9 @@ public class JetTestUtils {
     private static final Pattern KT_FILES = Pattern.compile(".*?.kt");
     private static List<File> filesToDelete = new ArrayList<File>();
 
+    public static final Pattern FILE_PATTERN = Pattern.compile("//\\s*FILE:\\s*(.*)$", Pattern.MULTILINE);
+    public static final Pattern DIRECTIVE_PATTERN = Pattern.compile("^//\\s*!(\\w+):\\s*(.*)$", Pattern.MULTILINE);
+
     public static final BindingTrace DUMMY_TRACE = new BindingTrace() {
 
 
@@ -308,8 +312,6 @@ public class JetTestUtils {
         }
     }
 
-    public static final Pattern FILE_PATTERN = Pattern.compile("//\\s*FILE:\\s*(.*)$", Pattern.MULTILINE);
-
     public static JetFile createFile(@NonNls String name, String text, @NotNull Project project) {
         LightVirtualFile virtualFile = new LightVirtualFile(name, JetLanguage.INSTANCE, text);
         virtualFile.setCharset(CharsetToolkit.UTF8_CHARSET);
@@ -376,15 +378,17 @@ public class JetTestUtils {
     }
 
     public interface TestFileFactory<F> {
-        F create(String fileName, String text);
+        F create(String fileName, String text, Map<String, String> directives);
     }
 
     public static <F> List<F> createTestFiles(String testFileName, String expectedText, TestFileFactory<F> factory) {
-        List<F> testFileFiles = Lists.newArrayList();
+        Map<String, String> directives = parseDirectives(expectedText);
+
+        List<F> testFiles = Lists.newArrayList();
         Matcher matcher = FILE_PATTERN.matcher(expectedText);
         if (!matcher.find()) {
             // One file
-            testFileFiles.add(factory.create(testFileName, expectedText));
+            testFiles.add(factory.create(testFileName, expectedText, directives));
         }
         else {
             int processedChars = 0;
@@ -405,7 +409,7 @@ public class JetTestUtils {
                 String fileText = expectedText.substring(start, end);
                 processedChars = end;
 
-                testFileFiles.add(factory.create(fileName, fileText));
+                testFiles.add(factory.create(fileName, fileText, directives));
 
                 if (!nextFileExists) break;
             }
@@ -414,7 +418,24 @@ public class JetTestUtils {
                                                              " to " +
                                                              (expectedText.length() - 1);
         }
-        return testFileFiles;
+        return testFiles;
+    }
+
+    private static Map<String, String> parseDirectives(String expectedText) {
+        Map<String, String> directives = Maps.newHashMap();
+        Matcher directiveMatcher = DIRECTIVE_PATTERN.matcher(expectedText);
+        int start = 0;
+        while (directiveMatcher.find()) {
+            if (directiveMatcher.start() != start) {
+                Assert.fail("Directives should only occur at the beginning of a file: " + directiveMatcher.group());
+            }
+            String name = directiveMatcher.group(1);
+            String value = directiveMatcher.group(2);
+            String oldValue = directives.put(name, value);
+            Assert.assertNull("Directive overwritten: " + name + " old value: " + oldValue + " new value: " + value, oldValue);
+            start = directiveMatcher.end() + 1;
+        }
+        return directives;
     }
 
     public static List<String> loadBeforeAfterText(String filePath) {
@@ -429,7 +450,7 @@ public class JetTestUtils {
 
         List<String> files = createTestFiles("", content, new TestFileFactory<String>() {
             @Override
-            public String create(String fileName, String text) {
+            public String create(String fileName, String text, Map<String, String> directives) {
                 int firstLineEnd = text.indexOf('\n');
                 return StringUtil.trimTrailing(text.substring(firstLineEnd + 1));
             }

compiler/tests/org/jetbrains/jet/checkers/AbstractJetDiagnosticsTest.java

@@ -16,12 +16,16 @@
 
 package org.jetbrains.jet.checkers;
 
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.google.common.io.Files;
 import com.intellij.lang.java.JavaLanguage;
+import com.intellij.openapi.util.Condition;
+import com.intellij.openapi.util.Conditions;
 import com.intellij.openapi.util.TextRange;
 import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.psi.PsiFileFactory;
+import com.intellij.util.containers.ContainerUtil;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.jet.ConfigurationKind;
@@ -29,20 +33,35 @@ import org.jetbrains.jet.JetLiteFixture;
 import org.jetbrains.jet.JetTestUtils;
 import org.jetbrains.jet.TestJdkKind;
 import org.jetbrains.jet.cli.jvm.compiler.JetCoreEnvironment;
-import org.jetbrains.jet.lang.diagnostics.Diagnostic;
+import org.jetbrains.jet.lang.diagnostics.*;
-import org.jetbrains.jet.lang.diagnostics.DiagnosticUtils;
 import org.jetbrains.jet.lang.psi.JetFile;
 import org.jetbrains.jet.lang.resolve.BindingContext;
 import org.jetbrains.jet.utils.ExceptionUtils;
+import org.junit.Assert;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 public abstract class AbstractJetDiagnosticsTest extends JetLiteFixture {
 
+    public static final String DIAGNOSTICS_DIRECTIVE = "DIAGNOSTICS";
+    public static final Pattern DIAGNOSTICS_PATTERN = Pattern.compile("([\\+\\-!])(\\w+)\\s*");
+    public static final ImmutableSet<AbstractDiagnosticFactory> DIAGNOSTICS_TO_INCLUDE_ANYWAY =
+            ImmutableSet.of(
+                    Errors.UNRESOLVED_REFERENCE,
+                    Errors.UNRESOLVED_REFERENCE_WRONG_RECEIVER,
+                    CheckerTestUtil.SyntaxErrorDiagnosticFactory.INSTANCE,
+                    CheckerTestUtil.DebugInfoDiagnosticFactory.ELEMENT_WITH_ERROR_TYPE,
+                    CheckerTestUtil.DebugInfoDiagnosticFactory.MISSING_UNRESOLVED,
+                    CheckerTestUtil.DebugInfoDiagnosticFactory.UNRESOLVED_WITH_TARGET
+            );
+
     @Override
     protected JetCoreEnvironment createEnvironment() {
         File javaFilesDir = new File(FileUtil.getTempDirectory(), "java-files");
@@ -83,18 +102,22 @@ public abstract class AbstractJetDiagnosticsTest extends JetLiteFixture {
         List<TestFile> testFiles =
                 JetTestUtils.createTestFiles(file.getName(), expectedText, new JetTestUtils.TestFileFactory<TestFile>() {
                     @Override
-                    public TestFile create(String fileName, String text) {
+                    public TestFile create(String fileName, String text, Map<String, String> directives) {
                         if (fileName.endsWith(".java")) {
                             writeJavaFile(fileName, text, javaFilesDir);
                         }
-                        return new TestFile(fileName, text);
+                        return new TestFile(fileName, text, parseDiagnosticFilterDirective(directives));
                     }
                 });
 
         analyzeAndCheck(file, expectedText, testFiles);
     }
 
-    protected abstract void analyzeAndCheck(File testDataFile, String expectedText, List<TestFile> files);
+    protected abstract void analyzeAndCheck(
+            File testDataFile,
+            String expectedText,
+            List<TestFile> files
+    );
 
     protected static List<JetFile> getJetFiles(List<TestFile> testFiles) {
         List<JetFile> jetFiles = Lists.newArrayList();
@@ -106,13 +129,81 @@ public abstract class AbstractJetDiagnosticsTest extends JetLiteFixture {
         return jetFiles;
     }
 
+    public static Condition<Diagnostic> parseDiagnosticFilterDirective(Map<String, String> diagnostics) {
+        String directives = diagnostics.get(DIAGNOSTICS_DIRECTIVE);
+        if (directives == null) {
+            return Conditions.alwaysTrue();
+        }
+        Condition<Diagnostic> condition = Conditions.alwaysTrue();
+        Matcher matcher = DIAGNOSTICS_PATTERN.matcher(directives);
+        if (!matcher.find()) {
+            Assert.fail("Wrong syntax in the '// DIAGNOSTICS: ...' directive:\n" +
+                        "found: '" + directives + "'\n" +
+                        "Must be '([+-!]DIAGNOSTIC_FACTORY_NAME|ERROR|WARNING|INFO)+'\n" +
+                        "where '+' means 'include'\n" +
+                        "      '-' means 'exclude'\n" +
+                        "      '!' means 'exclude everything but this'\n" +
+                        "directives are applied in the order of appearance, i.e. !FOO +BAR means inluce only FOO and BAR");
+        }
+        boolean first = true;
+        do {
+            String operation = matcher.group(1);
+            final String name = matcher.group(2);
+
+            Condition<Diagnostic> newCondition;
+            if (ImmutableSet.of("ERROR", "WARNING", "INFO").contains(name)) {
+                final Severity severity = Severity.valueOf(name);
+                newCondition = new Condition<Diagnostic>() {
+                    @Override
+                    public boolean value(Diagnostic diagnostic) {
+                        return diagnostic.getSeverity() == severity;
+                    }
+                };
+            }
+            else {
+                newCondition = new Condition<Diagnostic>() {
+                    @Override
+                    public boolean value(Diagnostic diagnostic) {
+                        return name.equals(diagnostic.getFactory().getName());
+                    }
+                };
+            }
+            if ("!".equals(operation)) {
+                if (!first) {
+                    Assert.fail("'" + operation + name + "' appears in a position rather than the first one, " +
+                                "which effectively cancels all the previous filters in this directive");
+                }
+                condition = newCondition;
+            }
+            else if ("+".equals(operation)) {
+                condition = Conditions.or(condition, newCondition);
+            }
+            else if ("-".equals(operation)) {
+                condition = Conditions.and(condition, Conditions.not(newCondition));
+            }
+            first = false;
+        }
+        while (matcher.find());
+        // We always include UNRESOLVED_REFERENCE and SYNTAX_ERROR because they are too likely to indicate erroneous test data
+        return Conditions.or(
+                condition,
+                new Condition<Diagnostic>() {
+                    @Override
+                    public boolean value(Diagnostic diagnostic) {
+                        return DIAGNOSTICS_TO_INCLUDE_ANYWAY.contains(diagnostic.getFactory());
+                    }
+                });
+    }
+
     protected class TestFile {
         private final List<CheckerTestUtil.DiagnosedRange> diagnosedRanges = Lists.newArrayList();
         private final String expectedText;
         private final String clearText;
         private final JetFile jetFile;
+        private final Condition<Diagnostic> whatDiagnosticsToConsider;
 
-        public TestFile(String fileName, String textWithMarkers) {
+        public TestFile(String fileName, String textWithMarkers, Condition<Diagnostic> whatDiagnosticsToConsider) {
+            this.whatDiagnosticsToConsider = whatDiagnosticsToConsider;
             if (fileName.endsWith(".java")) {
                 PsiFileFactory.getInstance(getProject()).createFileFromText(fileName, JavaLanguage.INSTANCE, textWithMarkers);
                 // TODO: check there's not syntax errors
@@ -142,7 +233,10 @@ public abstract class AbstractJetDiagnosticsTest extends JetLiteFixture {
             }
 
             final boolean[] ok = { true };
-            List<Diagnostic> diagnostics = CheckerTestUtil.getDiagnosticsIncludingSyntaxErrors(bindingContext, jetFile);
+            List<Diagnostic> diagnostics = ContainerUtil.filter(
+                    CheckerTestUtil.getDiagnosticsIncludingSyntaxErrors(bindingContext, jetFile),
+                    whatDiagnosticsToConsider
+            );
             CheckerTestUtil.diagnosticsDiff(diagnosedRanges, diagnostics, new CheckerTestUtil.DiagnosticDiffCallbacks() {
 
                 @Override

compiler/tests/org/jetbrains/jet/jvm/compiler/AbstractLoadJavaTest.java

@@ -49,6 +49,7 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 import static org.jetbrains.jet.jvm.compiler.LoadDescriptorUtil.*;
 import static org.jetbrains.jet.lang.resolve.java.DescriptorSearchRule.INCLUDE_KOTLIN_SOURCES;
@@ -89,7 +90,7 @@ public abstract class AbstractLoadJavaTest extends TestCaseWithTmpdir {
                 new File(javaFileName).getName(), FileUtil.loadFile(new File(javaFileName)),
                 new JetTestUtils.TestFileFactory<File>() {
                     @Override
-                    public File create(String fileName, String text) {
+                    public File create(String fileName, String text, Map<String, String> directives) {
                         File targetFile = new File(srcDir, fileName);
                         try {
                             FileUtil.writeToFile(targetFile, text);

compiler/tests/org/jetbrains/jet/lang/resolve/lazy/AbstractLazyResolveNamespaceComparingTest.java

@@ -34,6 +34,7 @@ import org.junit.Assert;
 import java.io.File;
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 import static org.jetbrains.jet.test.util.DescriptorValidator.ValidationVisitor.ALLOW_ERROR_TYPES;
 import static org.jetbrains.jet.test.util.DescriptorValidator.ValidationVisitor.FORBID_ERROR_TYPES;
@@ -62,7 +63,7 @@ public abstract class AbstractLazyResolveNamespaceComparingTest extends KotlinTe
                 .createTestFiles(testFileName, FileUtil.loadFile(new File(testFileName), true),
                                  new JetTestUtils.TestFileFactory<JetFile>() {
                                      @Override
-                                     public JetFile create(String fileName, String text) {
+                                     public JetFile create(String fileName, String text, Map<String, String> directives) {
                                          return JetPsiFactory.createFile(getProject(), fileName, text);
                                      }
                                  });

compiler/tests/org/jetbrains/jet/lang/resolve/lazy/AbstractLazyResolveTest.java

@@ -32,6 +32,7 @@ import org.jetbrains.jet.resolve.JetExpectedResolveDataUtil;
 
 import java.io.File;
 import java.util.List;
+import java.util.Map;
 
 public abstract class AbstractLazyResolveTest extends JetLiteFixture {
     private ExpectedResolveData expectedResolveData;
@@ -66,7 +67,7 @@ public abstract class AbstractLazyResolveTest extends JetLiteFixture {
 
         List<JetFile> files = JetTestUtils.createTestFiles("file.kt", text, new JetTestUtils.TestFileFactory<JetFile>() {
             @Override
-            public JetFile create(String fileName, String text) {
+            public JetFile create(String fileName, String text, Map<String, String> directives) {
                 return expectedResolveData.createFileFromMarkedUpText(fileName, text);
             }
         });

compiler/tests/org/jetbrains/jet/resolve/ExtensibleResolveTestCase.java

@@ -25,6 +25,7 @@ import org.jetbrains.jet.lang.psi.JetFile;
 
 import java.io.File;
 import java.util.List;
+import java.util.Map;
 
 public abstract class ExtensibleResolveTestCase extends JetLiteFixture {
     private ExpectedResolveData expectedResolveData;
@@ -53,7 +54,7 @@ public abstract class ExtensibleResolveTestCase extends JetLiteFixture {
         String text = JetTestUtils.doLoadFile(file);
         List<JetFile> files = JetTestUtils.createTestFiles("file.kt", text, new JetTestUtils.TestFileFactory<JetFile>() {
             @Override
-            public JetFile create(String fileName, String text) {
+            public JetFile create(String fileName, String text, Map<String, String> directives) {
                 return expectedResolveData.createFileFromMarkedUpText(fileName, text);
             }
         });