Release generated bytecode after each processed part

Part here means separate '.kt' file or multi-file facade
Now this memory optimization doesn't work with jar's,
because there is no simple way to write them incrementally.

compiler/backend/src/org/jetbrains/kotlin/codegen/ClassBuilderOnDemand.java

@@ -36,8 +36,12 @@ public class ClassBuilderOnDemand extends DelegatingClassBuilder {
 
     @Override
     public void done() {
-        if (classBuilder.isComputed()) {
+        if (isComputed()) {
             classBuilder.invoke().done();
         }
     }
+
+    public boolean isComputed() {
+        return classBuilder.isComputed();
+    }
 }

compiler/backend/src/org/jetbrains/kotlin/codegen/ClassFileFactory.java

@@ -92,6 +92,10 @@ public class ClassFileFactory implements OutputFileCollection {
         }
     }
 
+    public void releaseGeneratedOutput() {
+        generators.clear();
+    }
+
     private void writeModuleMappings() {
         final JvmPackageTable.PackageTable.Builder builder = JvmPackageTable.PackageTable.newBuilder();
         String outputFilePath = getMappingFileName(state.getModuleName());
@@ -144,6 +148,11 @@ public class ClassFileFactory implements OutputFileCollection {
     @Override
     public List<OutputFile> asList() {
         done();
+        return getCurrentOutput();
+    }
+
+    @NotNull
+    public List<OutputFile> getCurrentOutput() {
         return ContainerUtil.map(generators.keySet(), new Function<String, OutputFile>() {
             @Override
             public OutputFile fun(String relativeClassFilePath) {

compiler/backend/src/org/jetbrains/kotlin/codegen/MultifileClassCodegen.kt

@@ -127,6 +127,7 @@ class MultifileClassCodegen(
             ProgressIndicatorAndCompilationCanceledStatus.checkCanceled()
             try {
                 generatePart(file, generateCallableMemberTasks, partFqNames)
+                state.afterIndependentPart()
             }
             catch (e: ProcessCanceledException) {
                 throw e
@@ -292,6 +293,9 @@ class MultifileClassCodegen(
 
     private fun done() {
         classBuilder.done()
+        if (classBuilder.isComputed) {
+            state.afterIndependentPart()
+        }
     }
 
     companion object {

compiler/backend/src/org/jetbrains/kotlin/codegen/PackageCodegen.java

@@ -27,7 +27,6 @@ import org.jetbrains.kotlin.codegen.state.GenerationState;
 import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor;
 import org.jetbrains.kotlin.diagnostics.DiagnosticUtils;
 import org.jetbrains.kotlin.fileClasses.JvmFileClassInfo;
-import org.jetbrains.kotlin.load.kotlin.PackageParts;
 import org.jetbrains.kotlin.name.FqName;
 import org.jetbrains.kotlin.progress.ProgressIndicatorAndCompilationCanceledStatus;
 import org.jetbrains.kotlin.psi.*;
@@ -60,6 +59,7 @@ public class PackageCodegen {
             ProgressIndicatorAndCompilationCanceledStatus.checkCanceled();
             try {
                 generateFile(file);
+                state.afterIndependentPart();
             }
             catch (ProcessCanceledException e) {
                 throw e;

compiler/backend/src/org/jetbrains/kotlin/codegen/state/GenerationState.kt

@@ -67,7 +67,8 @@ class GenerationState @JvmOverloads constructor(
         val outDirectory: File? = null,
         val incrementalCompilationComponents: IncrementalCompilationComponents? = null,
         val generateOpenMultifileClasses: Boolean = false,
-        val progress: Progress = Progress.DEAF
+        val progress: Progress = Progress.DEAF,
+        private val onIndependentPartCompilationEnd: GenerationStateEventCallback = GenerationStateEventCallback.DO_NOTHING
 ) {
     abstract class GenerateClassFilter {
         abstract fun shouldAnnotateClass(processingClassOrObject: KtClassOrObject): Boolean
@@ -170,6 +171,10 @@ class GenerationState @JvmOverloads constructor(
         CodegenBinding.initTrace(this)
     }
 
+    fun afterIndependentPart() {
+        onIndependentPartCompilationEnd(this)
+    }
+
     private fun markUsed() {
         if (used) throw IllegalStateException("${GenerationState::class.java} cannot be used more than once")
 
@@ -197,3 +202,14 @@ private class LazyJvmDiagnostics(compute: () -> Diagnostics): Diagnostics {
 
     override fun iterator() = delegate.iterator()
 }
+
+interface GenerationStateEventCallback : (GenerationState) -> Unit {
+    companion object {
+        val DO_NOTHING = GenerationStateEventCallback {  }
+    }
+}
+
+fun GenerationStateEventCallback(block: (GenerationState) -> Unit): GenerationStateEventCallback =
+        object : GenerationStateEventCallback {
+            override fun invoke(s: GenerationState) = block(s)
+        }

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinToJVMBytecodeCompiler.kt

@@ -20,6 +20,7 @@ import com.intellij.openapi.util.io.JarUtil
 import org.jetbrains.kotlin.analyzer.AnalysisResult
 import org.jetbrains.kotlin.asJava.FilteredJvmDiagnostics
 import org.jetbrains.kotlin.backend.common.output.OutputFileCollection
+import org.jetbrains.kotlin.backend.common.output.SimpleOutputFileCollection
 import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
 import org.jetbrains.kotlin.cli.common.CompilerPluginContext
 import org.jetbrains.kotlin.cli.common.ExitCode
@@ -29,6 +30,7 @@ import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
 import org.jetbrains.kotlin.cli.jvm.config.*
 import org.jetbrains.kotlin.codegen.*
 import org.jetbrains.kotlin.codegen.state.GenerationState
+import org.jetbrains.kotlin.codegen.state.GenerationStateEventCallback
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.addKotlinSourceRoots
 import org.jetbrains.kotlin.fileClasses.JvmFileClassUtil
@@ -82,6 +84,20 @@ object KotlinToJVMBytecodeCompiler {
         }
     }
 
+    private fun createOutputFilesFlushingCallbackIfPossible(
+            configuration: CompilerConfiguration,
+            outputDir: File?,
+            jarPath: File?
+    ): GenerationStateEventCallback {
+        if (jarPath != null) return GenerationStateEventCallback.DO_NOTHING
+
+        return GenerationStateEventCallback { state ->
+            val currentOutput = SimpleOutputFileCollection(state.factory.currentOutput)
+            writeOutput(configuration, currentOutput, outputDir, jarPath = null, jarRuntime = false, mainClass = null)
+            state.factory.releaseGeneratedOutput()
+        }
+    }
+
     fun compileModules(
             environment: KotlinCoreEnvironment,
             configuration: CompilerConfiguration,
@@ -119,8 +135,13 @@ object KotlinToJVMBytecodeCompiler {
                     environment.project, getAbsolutePaths(directory, module), configuration) { s -> throw IllegalStateException("Should have been checked before: " + s) }
             if (!checkKotlinPackageUsage(environment, ktFiles)) return false
             val moduleOutputDirectory = File(module.getOutputDirectory())
+
+            val onIndependentPartCompilationEnd =
+                    createOutputFilesFlushingCallbackIfPossible(configuration, File(module.getOutputDirectory()), jarPath)
+
             val generationState = generate(environment, result, ktFiles, module, moduleOutputDirectory,
-                                           module.getModuleName())
+                                           module.getModuleName(),  onIndependentPartCompilationEnd)
+
             outputFiles.put(module, generationState.factory)
             generationStates.add(generationState);
         }
@@ -192,7 +213,9 @@ object KotlinToJVMBytecodeCompiler {
         }
 
         if (!checkKotlinPackageUsage(environment, environment.getSourceFiles())) return false
-        val generationState = analyzeAndGenerate(environment) ?: return false
+
+        val onIndependentPartCompilationEnd = createOutputFilesFlushingCallbackIfPossible(environment.configuration, outputDir, jar)
+        val generationState = analyzeAndGenerate(environment, onIndependentPartCompilationEnd) ?: return false
 
         val mainClass = findMainClass(generationState, environment.getSourceFiles())
 
@@ -247,7 +270,7 @@ object KotlinToJVMBytecodeCompiler {
             configuration: CompilerConfiguration,
             paths: KotlinPaths,
             environment: KotlinCoreEnvironment): Class<*>? {
-        val state = analyzeAndGenerate(environment) ?: return null
+        val state = analyzeAndGenerate(environment, GenerationStateEventCallback.DO_NOTHING) ?: return null
 
         val classLoader: GeneratedClassLoader
         try {
@@ -266,14 +289,17 @@ object KotlinToJVMBytecodeCompiler {
 
     }
 
-    fun analyzeAndGenerate(environment: KotlinCoreEnvironment): GenerationState? {
+    fun analyzeAndGenerate(
+            environment: KotlinCoreEnvironment,
+            onIndependentPartCompilationEnd: GenerationStateEventCallback
+    ): GenerationState? {
         val result = analyze(environment, null) ?: return null
 
         if (!result.shouldGenerateCode) return null
 
         result.throwIfError()
 
-        return generate(environment, result, environment.getSourceFiles(), null, null, null)
+        return generate(environment, result, environment.getSourceFiles(), null, null, null, onIndependentPartCompilationEnd)
     }
 
     private fun analyze(environment: KotlinCoreEnvironment, targetDescription: String?): AnalysisResult? {
@@ -331,7 +357,9 @@ object KotlinToJVMBytecodeCompiler {
             sourceFiles: List<KtFile>,
             module: Module?,
             outputDirectory: File?,
-            moduleName: String?): GenerationState {
+            moduleName: String?,
+            onIndependentPartCompilationEnd: GenerationStateEventCallback
+    ): GenerationState {
         val configuration = environment.configuration
         val incrementalCompilationComponents = configuration.get(JVMConfigurationKeys.INCREMENTAL_COMPILATION_COMPONENTS)
 
@@ -369,7 +397,8 @@ object KotlinToJVMBytecodeCompiler {
                 moduleName,
                 outputDirectory,
                 incrementalCompilationComponents,
-                configuration.get(JVMConfigurationKeys.MULTIFILE_FACADES_OPEN, false))
+                configuration.get(JVMConfigurationKeys.MULTIFILE_FACADES_OPEN, false),
+                onIndependentPartCompilationEnd = onIndependentPartCompilationEnd)
         ProgressIndicatorAndCompilationCanceledStatus.checkCanceled()
 
         val generationStart = PerformanceCounter.currentTime()

compiler/tests/org/jetbrains/kotlin/codegen/StdlibTest.java

@@ -29,6 +29,7 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinToJVMBytecodeCompiler;
 import org.jetbrains.kotlin.cli.jvm.config.JvmContentRootsKt;
 import org.jetbrains.kotlin.codegen.forTestCompile.ForTestCompileRuntime;
 import org.jetbrains.kotlin.codegen.state.GenerationState;
+import org.jetbrains.kotlin.codegen.state.GenerationStateEventCallback;
 import org.jetbrains.kotlin.config.CompilerConfiguration;
 import org.jetbrains.kotlin.config.ContentRootsKt;
 import org.jetbrains.kotlin.descriptors.ClassDescriptor;
@@ -61,7 +62,8 @@ public class StdlibTest extends KotlinTestWithEnvironment {
     }
 
     public void testStdlib() throws ClassNotFoundException {
-        GenerationState state = KotlinToJVMBytecodeCompiler.INSTANCE.analyzeAndGenerate(getEnvironment());
+        GenerationState state = KotlinToJVMBytecodeCompiler.INSTANCE.analyzeAndGenerate(
+                getEnvironment(), GenerationStateEventCallback.Companion.getDO_NOTHING());
         if (state == null) {
             fail("There were compilation errors");
         }