diff --git a/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateLibraries/DuplicateLibTest-1.jar b/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateLibraries/DuplicateLibTest-1.jar
deleted file mode 100644
index 8f3b55848e5..00000000000
Binary files a/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateLibraries/DuplicateLibTest-1.jar and /dev/null differ
diff --git a/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateLibraries/DuplicateLibTest-2.jar b/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateLibraries/DuplicateLibTest-2.jar
deleted file mode 100644
index 8f3b55848e5..00000000000
Binary files a/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateLibraries/DuplicateLibTest-2.jar and /dev/null differ
diff --git a/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateLibraries/DuplicateTest.kt.txt b/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateLibraries/DuplicateTest.kt.txt
deleted file mode 100644
index 77f7b2f7d21..00000000000
--- a/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateLibraries/DuplicateTest.kt.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-package testing
-
-// To regenerate this test, run CompileBinaryTestData.kt
-// Or manually, compile this file into DuplicateLibTest-1.jar and copy it to DuplicateLibTest-2.jar
-
-object TopLevelObject
-
-class Outer {
-    inner class Inner
-    class Nested
-}
diff --git a/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateLibraries/library-1/source.kt b/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateLibraries/library-1/source.kt
new file mode 100644
index 00000000000..96fd72e944e
--- /dev/null
+++ b/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateLibraries/library-1/source.kt
@@ -0,0 +1,8 @@
+package testing
+
+object TopLevelObject
+
+class Outer {
+    inner class Inner
+    class Nested
+}
diff --git a/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateLibraries/library-2/source.kt b/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateLibraries/library-2/source.kt
new file mode 100644
index 00000000000..96fd72e944e
--- /dev/null
+++ b/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateLibraries/library-2/source.kt
@@ -0,0 +1,8 @@
+package testing
+
+object TopLevelObject
+
+class Outer {
+    inner class Inner
+    class Nested
+}
diff --git a/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateObjectInBinaryAndSources/library/source.kt b/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateObjectInBinaryAndSources/library/source.kt
new file mode 100644
index 00000000000..ecbd62b4e7e
--- /dev/null
+++ b/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateObjectInBinaryAndSources/library/source.kt
@@ -0,0 +1,3 @@
+package test
+
+public object Lol
diff --git a/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateObjectInBinaryAndSources/objectBinaries.jar b/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateObjectInBinaryAndSources/objectBinaries.jar
deleted file mode 100644
index 72beb54369a..00000000000
Binary files a/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateObjectInBinaryAndSources/objectBinaries.jar and /dev/null differ
diff --git a/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateObjectInBinaryAndSources/source.kt.txt b/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateObjectInBinaryAndSources/source.kt.txt
deleted file mode 100644
index b54fd6a5d52..00000000000
--- a/compiler/testData/compileKotlinAgainstCustomBinaries/duplicateObjectInBinaryAndSources/source.kt.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-package test
-
-// To regenerate this test, run CompileBinaryTestData.kt
-// Or manually compile this file into jar
-
-public object Lol
diff --git a/compiler/tests/org/jetbrains/jet/jvm/compiler/CompileKotlinAgainstCustomBinariesTest.java b/compiler/tests/org/jetbrains/jet/jvm/compiler/CompileKotlinAgainstCustomBinariesTest.java
index 7fceff4e089..5e730f7c540 100644
--- a/compiler/tests/org/jetbrains/jet/jvm/compiler/CompileKotlinAgainstCustomBinariesTest.java
+++ b/compiler/tests/org/jetbrains/jet/jvm/compiler/CompileKotlinAgainstCustomBinariesTest.java
@@ -22,6 +22,7 @@ import jet.Function0;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.jet.ConfigurationKind;
 import org.jetbrains.jet.JetTestUtils;
+import org.jetbrains.jet.MockLibraryUtil;
 import org.jetbrains.jet.TestJdkKind;
 import org.jetbrains.jet.cli.jvm.compiler.JetCoreEnvironment;
 import org.jetbrains.jet.lang.descriptors.DeclarationDescriptor;
@@ -37,10 +38,7 @@ import org.jetbrains.jet.test.util.NamespaceComparator;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
 import java.util.regex.Pattern;
 
 import static org.jetbrains.jet.test.util.NamespaceComparator.validateAndCompareNamespaceWithFile;
@@ -53,6 +51,11 @@ public class CompileKotlinAgainstCustomBinariesTest extends TestCaseWithTmpdir {
         return new File(TEST_DATA_PATH, getTestName(true));
     }
 
+    @NotNull
+    private File compileLibrary(@NotNull String sourcePath) {
+        return MockLibraryUtil.compileLibraryToJar(new File(getTestDataDirectory(), sourcePath).getPath());
+    }
+
     private void doTestWithTxt(@NotNull Function0<List<File>> classPathProducer) throws Exception {
         File ktFile = new File(getTestDataDirectory(), getTestName(false) + ".kt");
 
@@ -107,7 +110,7 @@ public class CompileKotlinAgainstCustomBinariesTest extends TestCaseWithTmpdir {
         Collection<DeclarationDescriptor> allDescriptors = analyzeAndGetAllDescriptors(new Function0<List<File>>() {
             @Override
             public List<File> invoke() {
-                return findAllJars();
+                return Collections.singletonList(compileLibrary("library"));
             }
         });
         assertEquals(allDescriptors.size(), 2);
@@ -138,7 +141,7 @@ public class CompileKotlinAgainstCustomBinariesTest extends TestCaseWithTmpdir {
         doTestWithTxt(new Function0<List<File>>() {
             @Override
             public List<File> invoke() {
-                return findAllJars();
+                return Arrays.asList(compileLibrary("library-1"), compileLibrary("library-2"));
             }
         });
     }
diff --git a/generators/src/org/jetbrains/jet/generators/tests/CompileBinaryTestData.kt b/generators/src/org/jetbrains/jet/generators/tests/CompileBinaryTestData.kt
index 01b3a5c59f1..c9ad2dabe4c 100644
--- a/generators/src/org/jetbrains/jet/generators/tests/CompileBinaryTestData.kt
+++ b/generators/src/org/jetbrains/jet/generators/tests/CompileBinaryTestData.kt
@@ -52,19 +52,6 @@ private object BinaryTestData {
         rm("test")
     }
 
-    GenScript("compiler/testData/compileKotlinAgainstBinariesCustom/duplicateObjectInBinaryAndSources", "source.kt", "objectBinaries.jar")
-    fun genDuplicateObjectInBinaryAndSources() {
-        jar("objectBinaries.jar", "test")
-        rm("test")
-    }
-
-    GenScript("compiler/testData/compileKotlinAgainstCustomBinaries/duplicateLibraries", "DuplicateTest.kt", "DuplicateLibTest-1.jar")
-    fun genDuplicateLibraries() {
-        jar("DuplicateLibTest-1.jar", "testing")
-        cp("DuplicateLibTest-1.jar", "DuplicateLibTest-2.jar")
-        rm("testing")
-    }
-
     GenScript("compiler/testData/compileKotlinAgainstCustomBinaries/missingEnumReferencedInAnnotation", "MissingEnum.kt", "MissingEnum.jar")
     fun genMissingEnumReferencedInAnnotation() {
         rm("test/E.class")