diff --git a/compiler/integration-tests/data/compilationFailed/hello.compile.expected b/compiler/integration-tests/data/compilationFailed/hello.compile.expected
new file mode 100644
index 00000000000..6319b1c5a44
--- /dev/null
+++ b/compiler/integration-tests/data/compilationFailed/hello.compile.expected
@@ -0,0 +1,4 @@
+OUT ERROR: [TestData]\hello.kt: (4, 5) Unresolved reference: a
+OUT WARNING: [TestData]\hello.kt: (4, 5) The expression is unused
+ERR exec() finished with COMPILATION_ERROR return code
+Return code: 1
diff --git a/compiler/integration-tests/data/compilationFailed/hello.kt b/compiler/integration-tests/data/compilationFailed/hello.kt
new file mode 100644
index 00000000000..139b0ddc047
--- /dev/null
+++ b/compiler/integration-tests/data/compilationFailed/hello.kt
@@ -0,0 +1,5 @@
+package Hello
+
+fun main(args : Array<String>) {
+    a
+}
diff --git a/compiler/integration-tests/src/org/jetbrains/kotlin/CompilerSmokeTest.java b/compiler/integration-tests/src/org/jetbrains/kotlin/CompilerSmokeTest.java
index f8dd4c30491..b0ec6652e4b 100644
--- a/compiler/integration-tests/src/org/jetbrains/kotlin/CompilerSmokeTest.java
+++ b/compiler/integration-tests/src/org/jetbrains/kotlin/CompilerSmokeTest.java
@@ -43,4 +43,11 @@ public class CompilerSmokeTest extends KotlinIntegrationTestBase {
         assertEquals("compilation failed", 0, runCompiler("Smoke.compile", "-module", "Smoke.kts", "-jar", jar));
         runJava("Smoke.run", "-cp", jar + File.pathSeparator + getKotlinRuntimePath(), "Smoke.namespace", "1", "2", "3");
     }
+
+    @Test
+    public void compilationFailed() throws Exception {
+        final String jar = tempDir.getAbsolutePath() + File.separator + "smoke.jar";
+
+        assertEquals("compilation failed", 0, runCompiler("hello.compile", "-src", "hello.kt", "-jar", jar));
+    }
 }
diff --git a/compiler/integration-tests/src/org/jetbrains/kotlin/KotlinIntegrationTestBase.java b/compiler/integration-tests/src/org/jetbrains/kotlin/KotlinIntegrationTestBase.java
index 58bc3bdd0f4..465888216fe 100644
--- a/compiler/integration-tests/src/org/jetbrains/kotlin/KotlinIntegrationTestBase.java
+++ b/compiler/integration-tests/src/org/jetbrains/kotlin/KotlinIntegrationTestBase.java
@@ -101,28 +101,30 @@ public abstract class KotlinIntegrationTestBase {
             assertEquals("Non-zero exit code", 0, exitCode);
         }
         else {
-            check(logName, executionLog);
+            check(logName, executionLog.toString());
         }
 
         return exitCode;
     }
 
-    protected void check(String baseName, StringBuilder content) throws IOException {
+    protected void check(String baseName, String content) throws IOException {
         final File actualFile = new File(testDataDir, baseName + ".actual");
         final File expectedFile = new File(testDataDir, baseName + ".expected");
 
+        final String normalizedContent = StringUtil.replace(content, testDataDir.getAbsolutePath(), "[TestData]", true);
+
         if (!expectedFile.isFile()) {
-            Files.write(content, actualFile, Charsets.UTF_8);
+            Files.write(normalizedContent, actualFile, Charsets.UTF_8);
             fail("No .expected file " + expectedFile);
         }
         else {
             final String goldContent = Files.toString(expectedFile, UTF_8);
             try {
-                assertEquals(goldContent, content.toString());
+                assertEquals(goldContent, normalizedContent);
                 actualFile.delete();
             }
             catch (ComparisonFailure e) {
-                Files.write(content, actualFile, Charsets.UTF_8);
+                Files.write(normalizedContent, actualFile, Charsets.UTF_8);
                 throw e;
             }
         }