Split resulting test jar into small ones (1000 files per jar)

Dex on teamcity has memory problems with large jar

compiler/android-tests/android-module/ant.properties

@@ -1,18 +0,0 @@
-# This file is used to override default values used by the Ant build system.
-#
-# This file must be checked into Version Control Systems, as it is
-# integral to the build system of your project.
-
-# This file is only used by the Ant script.
-
-# You can use this to override default values such as
-#  'source.dir' for the location of your java source folder and
-#  'out.dir' for the location of your output folder.
-
-# You can also use it define how the release builds are signed by declaring
-# the following properties:
-#  'key.store' for the location of your keystore and
-#  'key.alias' for the name of the key to use.
-# The password will be asked during the build when you use the 'release' target.
-
-tested.project.dir=tested-module

compiler/android-tests/android-module/build.gradle

@@ -64,12 +64,27 @@ android {
 
     dexOptions {
         dexInProcess false
-        javaMaxHeapSize "700m"
+        javaMaxHeapSize "300m"
         //default is 4 and Total Memory = maxProcessCount * javaMaxHeapSize
-        maxProcessCount 2
+        maxProcessCount 1
+        additionalParameters "--debug"
     }
 }
 
+task jarTestFolders() {
+    println "Jar folders..."
+    new File("${projectDir}/libs/").listFiles().each { File file ->
+        if (file.isDirectory()) {
+            println "Jar '${file.name}' folder..."
+            ant.jar(basedir: "libs/${file.name}/", destfile: "libs/" +  file.name  + ".jar")
+        }
+    }
+}
+
+tasks.withType(JavaCompile) {
+    compileTask -> compileTask.dependsOn jarTestFolders
+}
+
 dependencies {
     compile fileTree(dir: 'libs', include: ['*.jar'])
     androidTestCompile 'junit:junit:4.12'

compiler/android-tests/android-module/build.xml

@@ -1,19 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project name="testsForAndroid" default="help" basedir="">
-
-  <property file="local.properties"/>
-
-  <property file="ant.properties"/>
-
-  <loadproperties srcFile="project.properties"/>
-
-  <import file="custom_rules.xml" optional="true"/>
-
-  <import file="${sdk.dir}/tools/ant/build.xml"/>
-
-  <target name="pack_libraries">
-    <jar basedir="libs/codegen-test-output" destfile="libs/test.jar"/>
-    <delete dir="libs/codegen-test-output"/>
-  </target>
-
-</project>

compiler/android-tests/android-module/gradle.properties

@@ -0,0 +1,2 @@
+#don't try to download android specific tools within gradle: licence acceptance will be required
+android.builder.sdkDownload=false

compiler/android-tests/src/org/jetbrains/kotlin/android/tests/CodegenTestsOnAndroidRunner.java

@@ -20,7 +20,6 @@ import com.intellij.util.PlatformUtils;
 import junit.framework.TestCase;
 import junit.framework.TestSuite;
 import org.jetbrains.annotations.Nullable;
-import org.jetbrains.kotlin.android.tests.ant.AntRunner;
 import org.jetbrains.kotlin.android.tests.download.SDKDownloader;
 import org.jetbrains.kotlin.android.tests.emulator.Emulator;
 import org.jetbrains.kotlin.android.tests.gradle.GradleRunner;
@@ -83,8 +82,6 @@ public class CodegenTestsOnAndroidRunner {
         downloader.unzipAll();
         PermissionManager.setPermissions(pathManager);
 
-        AntRunner antRunner = new AntRunner(pathManager);
-        antRunner.packLibraries();
         Emulator emulator = new Emulator(pathManager, Emulator.ARM);
         GradleRunner gradleRunner = new GradleRunner(pathManager);
         gradleRunner.clean();

compiler/android-tests/src/org/jetbrains/kotlin/android/tests/PathManager.java

@@ -52,8 +52,8 @@ public class PathManager {
         return getAndroidSdkRoot() + "/build-tools";
     }
 
-    public String getOutputForCompiledFiles() {
+    public String getOutputForCompiledFiles(int index) {
-        return tmpFolder + "/libs/codegen-test-output";
+        return tmpFolder + "/libs/libtest" + index;
     }
 
     public String getLibsFolderInAndroidTestedModuleTmpFolder() {
@@ -84,10 +84,6 @@ public class PathManager {
         return getDependenciesRoot() + "/download";
     }
 
-    public String getAntBinDirectory() {
-        return rootFolder + "/dependencies/ant-1.8/bin";
-    }
-
     public String getAndroidModuleRoot() {
         return rootFolder + "/compiler/android-tests/android-module";
     }

compiler/android-tests/src/org/jetbrains/kotlin/android/tests/ant/AntRunner.java

@@ -1,105 +0,0 @@
-/*
- * Copyright 2010-2015 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.jetbrains.kotlin.android.tests.ant;
-
-import com.intellij.execution.configurations.GeneralCommandLine;
-import com.intellij.openapi.util.SystemInfo;
-import org.jetbrains.kotlin.android.tests.OutputUtils;
-import org.jetbrains.kotlin.android.tests.PathManager;
-import org.jetbrains.kotlin.android.tests.run.RunResult;
-import org.jetbrains.kotlin.android.tests.run.RunUtils;
-import org.jetbrains.kotlin.utils.ExceptionUtilsKt;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class AntRunner {
-    private final List<String> listOfAntCommands;
-
-    public AntRunner(PathManager pathManager) {
-        listOfAntCommands = new ArrayList<>();
-        String antCmdName = SystemInfo.isWindows ? "ant.bat" : "ant";
-        listOfAntCommands.add(pathManager.getAntBinDirectory() + "/" + antCmdName);
-        listOfAntCommands.add("-Dsdk.dir=" + pathManager.getAndroidSdkRoot());
-        listOfAntCommands.add("-buildfile");
-        listOfAntCommands.add(pathManager.getTmpFolder() + "/build.xml");
-    }
-
-    /* Pack compiled sources on first step to one jar file */
-    public void packLibraries() {
-        System.out.println("Pack libraries...");
-        RunResult result = RunUtils.execute(generateCommandLine("pack_libraries"));
-        OutputUtils.checkResult(result);
-    }
-
-    /* Clean output directory */
-    public void cleanOutput() {
-        System.out.println("Clearing output directory...");
-        RunResult result = RunUtils.execute(generateCommandLine("clean"));
-        OutputUtils.checkResult(result);
-    }
-
-    public void compileSources() {
-        System.out.println("Compiling sources...");
-        RunResult result = RunUtils.execute(generateCommandLine("debug"));
-        OutputUtils.checkResult(result);
-    }
-
-    public void installApplicationOnEmulator() {
-        System.out.println("Installing apk...");
-        RunResult result = RunUtils.execute(generateCommandLine("installt"));
-        String resultOutput = result.getOutput();
-        if (!isInstallSuccessful(resultOutput)) {
-            installApplicationOnEmulator();
-            return;
-        }
-        else {
-            if (result.getStatus()) {
-                System.out.println(resultOutput);
-            }
-        }
-        OutputUtils.checkResult(result);
-    }
-
-    public String runTestsOnEmulator() {
-        System.out.println("Running tests...");
-        RunResult result = RunUtils.execute(generateCommandLine("test"));
-        String resultOutput = result.getOutput();
-        OutputUtils.checkResult(result);
-        return resultOutput;
-    }
-
-    private static boolean isInstallSuccessful(String output) {
-        if (output.contains("Is the system running?")) {
-            System.out.println("Device not ready. Waiting for 20 sec.");
-            try {
-                Thread.sleep(20000);
-            }
-            catch (InterruptedException e) {
-                throw ExceptionUtilsKt.rethrow(e);
-            }
-            return false;
-        }
-        return true;
-    }
-
-    private GeneralCommandLine generateCommandLine(String taskName) {
-        GeneralCommandLine commandLine = new GeneralCommandLine(listOfAntCommands);
-        commandLine.addParameter(taskName);
-        return commandLine;
-    }
-}

compiler/android-tests/src/org/jetbrains/kotlin/android/tests/download/SDKDownloader.java

@@ -54,7 +54,7 @@ public class SDKDownloader {
         platformToolsZipPath = pathManager.getRootForDownload() + "/platform-tools.zip";
         skdToolsZipPath = pathManager.getRootForDownload() + "/tools.zip";
         buildToolsZipPath = pathManager.getRootForDownload() + "/build-tools.zip";
-        gradleZipPath = pathManager.getRootForDownload() + "/gradle.zip";
+        gradleZipPath = pathManager.getRootForDownload() + "/gradle" + GRADLE_VERSION + ".zip";
     }
 
     public void downloadPlatform() {

compiler/android-tests/src/org/jetbrains/kotlin/android/tests/gradle/GradleRunner.java

@@ -46,7 +46,9 @@ public class GradleRunner {
 
     public void build() {
         System.out.println("Building gradle project...");
-        RunResult result = RunUtils.execute(generateCommandLine("build"));
+        GeneralCommandLine build = generateCommandLine("build");
+        build.addParameter("--stacktrace");
+        RunResult result = RunUtils.execute(build);
         OutputUtils.checkResult(result);
     }
 

compiler/android-tests/src/org/jetbrains/kotlin/android/tests/run/PermissionManager.java

@@ -29,7 +29,6 @@ public class PermissionManager {
 
     public static void setPermissions(PathManager pathManager) {
         if (!SystemInfo.isWindows) {
-            RunUtils.execute(generateChmodCmd(pathManager.getAntBinDirectory() + "/ant"));
             setExecPermissionForSimpleNamedFiles(new File(pathManager.getToolsFolderInAndroidSdk()));
             setExecPermissionForSimpleNamedFiles(new File(pathManager.getToolsFolderInAndroidSdk() + "/bin64"));
             setExecPermissionForSimpleNamedFiles(new File(pathManager.getBuildToolsFolderInAndroidSdk() + "/" + SDKDownloader.BUILD_TOOLS));

compiler/android-tests/tests/org/jetbrains/kotlin/android/tests/CodegenTestsOnAndroidGenerator.java

@@ -55,6 +55,7 @@ public class CodegenTestsOnAndroidGenerator extends KtUsefulTestCase {
     private static final String generatorName = "CodegenTestsOnAndroidGenerator";
 
     private static int MODULE_INDEX = 1;
+    private int WRITED_FILES_COUNT = 0;
 
     private final List<String> generatedTestNames = Lists.newArrayList();
 
@@ -191,10 +192,15 @@ public class CodegenTestsOnAndroidGenerator extends KtUsefulTestCase {
         private void writeFiles(List<KtFile> filesToCompile) {
             if (filesToCompile.isEmpty()) return;
 
+            //1000 files per folder, each folder would be jared by build.gradle script
+            // We can't create one big jar with all test cause dex has problem with memory on teamcity
+            WRITED_FILES_COUNT += filesToCompile.size();
+            File outputDir = new File(pathManager.getOutputForCompiledFiles(WRITED_FILES_COUNT / 1000));
+
             System.out.println("Generating " + filesToCompile.size() + " files" +
                                (inheritMultifileParts
                                 ? " (JVM.INHERIT_MULTIFILE_PARTS)"
-                                : isFullJdkAndRuntime ? " (full jdk and runtime)" : "") + "...");
+                                : isFullJdkAndRuntime ? " (full jdk and runtime)" : "") + " into " + outputDir.getName() + "...");
             OutputFileCollection outputFiles;
             try {
                 outputFiles = GenerationUtils.compileFiles(filesToCompile, environment).getFactory();
@@ -203,7 +209,6 @@ public class CodegenTestsOnAndroidGenerator extends KtUsefulTestCase {
                 throw new RuntimeException(e);
             }
 
-            File outputDir = new File(pathManager.getOutputForCompiledFiles());
             if (!outputDir.exists()) {
                 outputDir.mkdirs();
             }