diff --git a/compiler/testData/compileKotlinAgainstCustomBinaries/brokenJarWithNoClassForObjectProperty/broken.jar b/compiler/testData/compileKotlinAgainstCustomBinaries/brokenJarWithNoClassForObjectProperty/broken.jar
deleted file mode 100644
index b42c5758714..00000000000
Binary files a/compiler/testData/compileKotlinAgainstCustomBinaries/brokenJarWithNoClassForObjectProperty/broken.jar and /dev/null differ
diff --git a/compiler/testData/compileKotlinAgainstCustomBinaries/brokenJarWithNoClassForObjectProperty/library/source.kt b/compiler/testData/compileKotlinAgainstCustomBinaries/brokenJarWithNoClassForObjectProperty/library/source.kt
new file mode 100644
index 00000000000..ecbd62b4e7e
--- /dev/null
+++ b/compiler/testData/compileKotlinAgainstCustomBinaries/brokenJarWithNoClassForObjectProperty/library/source.kt
@@ -0,0 +1,3 @@
+package test
+
+public object Lol
diff --git a/compiler/testData/compileKotlinAgainstCustomBinaries/brokenJarWithNoClassForObjectProperty/source.kt.txt b/compiler/testData/compileKotlinAgainstCustomBinaries/brokenJarWithNoClassForObjectProperty/source.kt.txt
deleted file mode 100644
index ff74958b0c5..00000000000
--- a/compiler/testData/compileKotlinAgainstCustomBinaries/brokenJarWithNoClassForObjectProperty/source.kt.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-package test
-
-// To regenerate this test, run CompileBinaryTestData.kt
-
-// Manual instructions:
-// 1. Compile this file
-// 2. Delete test/Lol.class
-// 3. Pack test/ into broken.jar
-
-public object Lol
diff --git a/compiler/testData/compileKotlinAgainstCustomBinaries/missingEnumReferencedInAnnotation/MissingEnum.jar b/compiler/testData/compileKotlinAgainstCustomBinaries/missingEnumReferencedInAnnotation/MissingEnum.jar
deleted file mode 100644
index 826bfcc401c..00000000000
Binary files a/compiler/testData/compileKotlinAgainstCustomBinaries/missingEnumReferencedInAnnotation/MissingEnum.jar and /dev/null differ
diff --git a/compiler/testData/compileKotlinAgainstCustomBinaries/missingEnumReferencedInAnnotation/MissingEnum.kt.txt b/compiler/testData/compileKotlinAgainstCustomBinaries/missingEnumReferencedInAnnotation/MissingEnum.kt.txt
deleted file mode 100644
index e5dcc495aa1..00000000000
--- a/compiler/testData/compileKotlinAgainstCustomBinaries/missingEnumReferencedInAnnotation/MissingEnum.kt.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-package test
-
-// To regenerate this test, run CompileBinaryTestData.kt
-
-// Manual instructions:
-// 1. Compile this file
-// 2. Delete test/E.class
-// 3. Pack test/ into jar
-
-enum class E { ENTRY }
-
-annotation class Anno(val e: E)
-
-Anno(E.ENTRY) open class Class
diff --git a/compiler/testData/compileKotlinAgainstCustomBinaries/missingEnumReferencedInAnnotation/library/source.kt b/compiler/testData/compileKotlinAgainstCustomBinaries/missingEnumReferencedInAnnotation/library/source.kt
new file mode 100644
index 00000000000..1408b6d3618
--- /dev/null
+++ b/compiler/testData/compileKotlinAgainstCustomBinaries/missingEnumReferencedInAnnotation/library/source.kt
@@ -0,0 +1,7 @@
+package test
+
+enum class E { ENTRY }
+
+annotation class Anno(val e: E)
+
+Anno(E.ENTRY) open class Class
diff --git a/compiler/tests/org/jetbrains/jet/jvm/compiler/CompileKotlinAgainstCustomBinariesTest.java b/compiler/tests/org/jetbrains/jet/jvm/compiler/CompileKotlinAgainstCustomBinariesTest.java
index 5e730f7c540..280d04736e8 100644
--- a/compiler/tests/org/jetbrains/jet/jvm/compiler/CompileKotlinAgainstCustomBinariesTest.java
+++ b/compiler/tests/org/jetbrains/jet/jvm/compiler/CompileKotlinAgainstCustomBinariesTest.java
@@ -35,11 +35,16 @@ import org.jetbrains.jet.lang.types.ErrorUtils;
 import org.jetbrains.jet.test.TestCaseWithTmpdir;
 import org.jetbrains.jet.test.util.DescriptorValidator;
 import org.jetbrains.jet.test.util.NamespaceComparator;
+import org.jetbrains.jet.utils.ExceptionUtils;
 
+import java.io.BufferedOutputStream;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.util.*;
-import java.util.regex.Pattern;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.zip.ZipOutputStream;
 
 import static org.jetbrains.jet.test.util.NamespaceComparator.validateAndCompareNamespaceWithFile;
 
@@ -101,8 +106,34 @@ public class CompileKotlinAgainstCustomBinariesTest extends TestCaseWithTmpdir {
     }
 
     @NotNull
-    private List<File> findAllJars() {
-        return FileUtil.findFilesByMask(Pattern.compile("^.*\\.jar$"), getTestDataDirectory());
+    private static File copyJarFileWithoutEntry(@NotNull File jarPath, @NotNull String entryToDelete) {
+        try {
+            File outputFile = new File(jarPath.getParentFile(), FileUtil.getNameWithoutExtension(jarPath) + "-after.jar");
+
+            @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
+            JarFile jar = new JarFile(jarPath);
+            ZipOutputStream output = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(outputFile)));
+            try {
+                for (Enumeration<JarEntry> enumeration = jar.entries(); enumeration.hasMoreElements(); ) {
+                    JarEntry jarEntry = enumeration.nextElement();
+                    if (entryToDelete.equals(jarEntry.getName())) {
+                        continue;
+                    }
+                    output.putNextEntry(jarEntry);
+                    output.write(FileUtil.loadBytes(jar.getInputStream(jarEntry)));
+                    output.closeEntry();
+                }
+            }
+            finally {
+                output.close();
+                jar.close();
+            }
+
+            return outputFile;
+        }
+        catch (IOException e) {
+            throw ExceptionUtils.rethrow(e);
+        }
     }
 
 
@@ -126,7 +157,7 @@ public class CompileKotlinAgainstCustomBinariesTest extends TestCaseWithTmpdir {
         Collection<DeclarationDescriptor> allDescriptors = analyzeAndGetAllDescriptors(new Function0<List<File>>() {
             @Override
             public List<File> invoke() {
-                return findAllJars();
+                return Collections.singletonList(copyJarFileWithoutEntry(compileLibrary("library"), "test/Lol.class"));
             }
         });
         assertEquals(allDescriptors.size(), 1);
@@ -150,7 +181,7 @@ public class CompileKotlinAgainstCustomBinariesTest extends TestCaseWithTmpdir {
         doTestWithTxt(new Function0<List<File>>() {
             @Override
             public List<File> invoke() {
-                return findAllJars();
+                return Collections.singletonList(copyJarFileWithoutEntry(compileLibrary("library"), "test/E.class"));
             }
         });
     }
diff --git a/generators/src/org/jetbrains/jet/generators/tests/CompileBinaryTestData.kt b/generators/src/org/jetbrains/jet/generators/tests/CompileBinaryTestData.kt
index c9ad2dabe4c..ec3dfd40cd0 100644
--- a/generators/src/org/jetbrains/jet/generators/tests/CompileBinaryTestData.kt
+++ b/generators/src/org/jetbrains/jet/generators/tests/CompileBinaryTestData.kt
@@ -45,20 +45,6 @@ Retention(RetentionPolicy.RUNTIME)
 private annotation class GenScript(val dir: String, val source: String, val binary: String)
 
 private object BinaryTestData {
-    GenScript("compiler/testData/compileKotlinAgainstBinariesCustom/brokenJarWithNoClassForObjectProperty", "source.kt", "broken.jar")
-    fun genBrokenJarWithNoClassForObjectProperty() {
-        rm("test/Lol.class")
-        jar("broken.jar", "test")
-        rm("test")
-    }
-
-    GenScript("compiler/testData/compileKotlinAgainstCustomBinaries/missingEnumReferencedInAnnotation", "MissingEnum.kt", "MissingEnum.jar")
-    fun genMissingEnumReferencedInAnnotation() {
-        rm("test/E.class")
-        jar("MissingEnum.jar", "test")
-        rm("test")
-    }
-
     GenScript("idea/testData/completion/basic/custom/", "TopLevelNonImportedExtFunSource.kt", "TopLevelNonImportedExtFun.jar")
     fun genTopLevelNonImportedExtFun() {
         jar("TopLevelNonImportedExtFun.jar", "abc")