diff --git a/build-tools/build.xml b/build-tools/build.xml
index b3458ea1767..50d59149b9c 100644
--- a/build-tools/build.xml
+++ b/build-tools/build.xml
@@ -61,6 +61,7 @@
                     <antcall target = "zip"/>
                     <verify-exists file="${output}/${output.name}.zip" type="file"/>
 
+                    <!-- Sometimes deleting "dist/kotlinc/lib/alt/kotlin-jdk-headers.jar" fails -->
                     <delete dir  = "${kotlin-home}" failonerror="false"/>
                     <mkdir  dir  = "${kotlin-home}"/>
                     <unzip  src  = "${output}/${output.name}.zip" dest="${output}"/>
@@ -309,7 +310,7 @@
                 <then>
                     <!-- Batch tests -->
                     <if>
-                        <matches string="@{classpath}" pattern="\.jar$"/>
+                        <matches string="@{classpath}" pattern="/.jar$"/>
                         <then>
                             <!-- Batch tests from a jar -->
                             <echo>Running JUnit: classpath = "@{classpath}" (Jar)</echo>
@@ -434,6 +435,24 @@
                     <fail message="kotlinc-jar: compilation should have failed!"/>
                 </then>
             </if>
+
+            <var name="failed" value="false"/>
+            <trycatch property="error-message" reference="bar">
+                <try>
+                    <!-- Should fail -->
+                    <kotlinc-jar module = "${basedir}/build-tools/test/compilation-fail/Smoke.kts"/>
+                </try>
+                <catch>
+                    <var name="failed" value="true"/>
+                    <echo>"ERROR:" was expected here</echo>
+                </catch>
+            </trycatch>
+            <if>
+                <isfalse value="${failed}"/>
+                <then>
+                    <fail message="kotlinc-jar: compilation should have failed!"/>
+                </then>
+            </if>
         </sequential>
     </macrodef>
 
diff --git a/build-tools/test/compilation-fail/Smoke.kts b/build-tools/test/compilation-fail/Smoke.kts
new file mode 100644
index 00000000000..fdef116a2a5
--- /dev/null
+++ b/build-tools/test/compilation-fail/Smoke.kts
@@ -0,0 +1,3 @@
+import kotlin.modules.*
+
+fun proj
\ No newline at end of file