From bc5110550adb11520700cd28b569f97bd3e16834 Mon Sep 17 00:00:00 2001 From: Denis Zharkov Date: Thu, 18 Feb 2016 15:06:05 +0300 Subject: [PATCH] Release generated bytecode after each processed part Part here means separate '.kt' file or multi-file facade Now this memory optimization doesn't work with jar's, because there is no simple way to write them incrementally. --- .../kotlin/codegen/ClassBuilderOnDemand.java | 6 ++- .../kotlin/codegen/ClassFileFactory.java | 9 ++++ .../kotlin/codegen/MultifileClassCodegen.kt | 4 ++ .../kotlin/codegen/PackageCodegen.java | 2 +- .../kotlin/codegen/state/GenerationState.kt | 18 +++++++- .../compiler/KotlinToJVMBytecodeCompiler.kt | 43 ++++++++++++++++--- .../jetbrains/kotlin/codegen/StdlibTest.java | 4 +- 7 files changed, 75 insertions(+), 11 deletions(-) diff --git a/compiler/backend/src/org/jetbrains/kotlin/codegen/ClassBuilderOnDemand.java b/compiler/backend/src/org/jetbrains/kotlin/codegen/ClassBuilderOnDemand.java index 28b00db2d2f..72945b31e4d 100644 --- a/compiler/backend/src/org/jetbrains/kotlin/codegen/ClassBuilderOnDemand.java +++ b/compiler/backend/src/org/jetbrains/kotlin/codegen/ClassBuilderOnDemand.java @@ -36,8 +36,12 @@ public class ClassBuilderOnDemand extends DelegatingClassBuilder { @Override public void done() { - if (classBuilder.isComputed()) { + if (isComputed()) { classBuilder.invoke().done(); } } + + public boolean isComputed() { + return classBuilder.isComputed(); + } } diff --git a/compiler/backend/src/org/jetbrains/kotlin/codegen/ClassFileFactory.java b/compiler/backend/src/org/jetbrains/kotlin/codegen/ClassFileFactory.java index 7c23abec485..0d6bb0fcae9 100644 --- a/compiler/backend/src/org/jetbrains/kotlin/codegen/ClassFileFactory.java +++ b/compiler/backend/src/org/jetbrains/kotlin/codegen/ClassFileFactory.java @@ -92,6 +92,10 @@ public class ClassFileFactory implements OutputFileCollection { } } + public void releaseGeneratedOutput() { + generators.clear(); + } + private void writeModuleMappings() { final JvmPackageTable.PackageTable.Builder builder = JvmPackageTable.PackageTable.newBuilder(); String outputFilePath = getMappingFileName(state.getModuleName()); @@ -144,6 +148,11 @@ public class ClassFileFactory implements OutputFileCollection { @Override public List asList() { done(); + return getCurrentOutput(); + } + + @NotNull + public List getCurrentOutput() { return ContainerUtil.map(generators.keySet(), new Function() { @Override public OutputFile fun(String relativeClassFilePath) { diff --git a/compiler/backend/src/org/jetbrains/kotlin/codegen/MultifileClassCodegen.kt b/compiler/backend/src/org/jetbrains/kotlin/codegen/MultifileClassCodegen.kt index f7d863502bc..d4325a046f4 100644 --- a/compiler/backend/src/org/jetbrains/kotlin/codegen/MultifileClassCodegen.kt +++ b/compiler/backend/src/org/jetbrains/kotlin/codegen/MultifileClassCodegen.kt @@ -127,6 +127,7 @@ class MultifileClassCodegen( ProgressIndicatorAndCompilationCanceledStatus.checkCanceled() try { generatePart(file, generateCallableMemberTasks, partFqNames) + state.afterIndependentPart() } catch (e: ProcessCanceledException) { throw e @@ -292,6 +293,9 @@ class MultifileClassCodegen( private fun done() { classBuilder.done() + if (classBuilder.isComputed) { + state.afterIndependentPart() + } } companion object { diff --git a/compiler/backend/src/org/jetbrains/kotlin/codegen/PackageCodegen.java b/compiler/backend/src/org/jetbrains/kotlin/codegen/PackageCodegen.java index 5f78fb78cd6..fc52e2a080d 100644 --- a/compiler/backend/src/org/jetbrains/kotlin/codegen/PackageCodegen.java +++ b/compiler/backend/src/org/jetbrains/kotlin/codegen/PackageCodegen.java @@ -27,7 +27,6 @@ import org.jetbrains.kotlin.codegen.state.GenerationState; import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor; import org.jetbrains.kotlin.diagnostics.DiagnosticUtils; import org.jetbrains.kotlin.fileClasses.JvmFileClassInfo; -import org.jetbrains.kotlin.load.kotlin.PackageParts; import org.jetbrains.kotlin.name.FqName; import org.jetbrains.kotlin.progress.ProgressIndicatorAndCompilationCanceledStatus; import org.jetbrains.kotlin.psi.*; @@ -60,6 +59,7 @@ public class PackageCodegen { ProgressIndicatorAndCompilationCanceledStatus.checkCanceled(); try { generateFile(file); + state.afterIndependentPart(); } catch (ProcessCanceledException e) { throw e; diff --git a/compiler/backend/src/org/jetbrains/kotlin/codegen/state/GenerationState.kt b/compiler/backend/src/org/jetbrains/kotlin/codegen/state/GenerationState.kt index 0f5ce4215b3..bfffcedaa17 100644 --- a/compiler/backend/src/org/jetbrains/kotlin/codegen/state/GenerationState.kt +++ b/compiler/backend/src/org/jetbrains/kotlin/codegen/state/GenerationState.kt @@ -67,7 +67,8 @@ class GenerationState @JvmOverloads constructor( val outDirectory: File? = null, val incrementalCompilationComponents: IncrementalCompilationComponents? = null, val generateOpenMultifileClasses: Boolean = false, - val progress: Progress = Progress.DEAF + val progress: Progress = Progress.DEAF, + private val onIndependentPartCompilationEnd: GenerationStateEventCallback = GenerationStateEventCallback.DO_NOTHING ) { abstract class GenerateClassFilter { abstract fun shouldAnnotateClass(processingClassOrObject: KtClassOrObject): Boolean @@ -170,6 +171,10 @@ class GenerationState @JvmOverloads constructor( CodegenBinding.initTrace(this) } + fun afterIndependentPart() { + onIndependentPartCompilationEnd(this) + } + private fun markUsed() { if (used) throw IllegalStateException("${GenerationState::class.java} cannot be used more than once") @@ -197,3 +202,14 @@ private class LazyJvmDiagnostics(compute: () -> Diagnostics): Diagnostics { override fun iterator() = delegate.iterator() } + +interface GenerationStateEventCallback : (GenerationState) -> Unit { + companion object { + val DO_NOTHING = GenerationStateEventCallback { } + } +} + +fun GenerationStateEventCallback(block: (GenerationState) -> Unit): GenerationStateEventCallback = + object : GenerationStateEventCallback { + override fun invoke(s: GenerationState) = block(s) + } diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinToJVMBytecodeCompiler.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinToJVMBytecodeCompiler.kt index dcdde37d07a..a3b9b8f9852 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinToJVMBytecodeCompiler.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinToJVMBytecodeCompiler.kt @@ -20,6 +20,7 @@ import com.intellij.openapi.util.io.JarUtil import org.jetbrains.kotlin.analyzer.AnalysisResult import org.jetbrains.kotlin.asJava.FilteredJvmDiagnostics import org.jetbrains.kotlin.backend.common.output.OutputFileCollection +import org.jetbrains.kotlin.backend.common.output.SimpleOutputFileCollection import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys import org.jetbrains.kotlin.cli.common.CompilerPluginContext import org.jetbrains.kotlin.cli.common.ExitCode @@ -29,6 +30,7 @@ import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler import org.jetbrains.kotlin.cli.jvm.config.* import org.jetbrains.kotlin.codegen.* import org.jetbrains.kotlin.codegen.state.GenerationState +import org.jetbrains.kotlin.codegen.state.GenerationStateEventCallback import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.addKotlinSourceRoots import org.jetbrains.kotlin.fileClasses.JvmFileClassUtil @@ -82,6 +84,20 @@ object KotlinToJVMBytecodeCompiler { } } + private fun createOutputFilesFlushingCallbackIfPossible( + configuration: CompilerConfiguration, + outputDir: File?, + jarPath: File? + ): GenerationStateEventCallback { + if (jarPath != null) return GenerationStateEventCallback.DO_NOTHING + + return GenerationStateEventCallback { state -> + val currentOutput = SimpleOutputFileCollection(state.factory.currentOutput) + writeOutput(configuration, currentOutput, outputDir, jarPath = null, jarRuntime = false, mainClass = null) + state.factory.releaseGeneratedOutput() + } + } + fun compileModules( environment: KotlinCoreEnvironment, configuration: CompilerConfiguration, @@ -119,8 +135,13 @@ object KotlinToJVMBytecodeCompiler { environment.project, getAbsolutePaths(directory, module), configuration) { s -> throw IllegalStateException("Should have been checked before: " + s) } if (!checkKotlinPackageUsage(environment, ktFiles)) return false val moduleOutputDirectory = File(module.getOutputDirectory()) + + val onIndependentPartCompilationEnd = + createOutputFilesFlushingCallbackIfPossible(configuration, File(module.getOutputDirectory()), jarPath) + val generationState = generate(environment, result, ktFiles, module, moduleOutputDirectory, - module.getModuleName()) + module.getModuleName(), onIndependentPartCompilationEnd) + outputFiles.put(module, generationState.factory) generationStates.add(generationState); } @@ -192,7 +213,9 @@ object KotlinToJVMBytecodeCompiler { } if (!checkKotlinPackageUsage(environment, environment.getSourceFiles())) return false - val generationState = analyzeAndGenerate(environment) ?: return false + + val onIndependentPartCompilationEnd = createOutputFilesFlushingCallbackIfPossible(environment.configuration, outputDir, jar) + val generationState = analyzeAndGenerate(environment, onIndependentPartCompilationEnd) ?: return false val mainClass = findMainClass(generationState, environment.getSourceFiles()) @@ -247,7 +270,7 @@ object KotlinToJVMBytecodeCompiler { configuration: CompilerConfiguration, paths: KotlinPaths, environment: KotlinCoreEnvironment): Class<*>? { - val state = analyzeAndGenerate(environment) ?: return null + val state = analyzeAndGenerate(environment, GenerationStateEventCallback.DO_NOTHING) ?: return null val classLoader: GeneratedClassLoader try { @@ -266,14 +289,17 @@ object KotlinToJVMBytecodeCompiler { } - fun analyzeAndGenerate(environment: KotlinCoreEnvironment): GenerationState? { + fun analyzeAndGenerate( + environment: KotlinCoreEnvironment, + onIndependentPartCompilationEnd: GenerationStateEventCallback + ): GenerationState? { val result = analyze(environment, null) ?: return null if (!result.shouldGenerateCode) return null result.throwIfError() - return generate(environment, result, environment.getSourceFiles(), null, null, null) + return generate(environment, result, environment.getSourceFiles(), null, null, null, onIndependentPartCompilationEnd) } private fun analyze(environment: KotlinCoreEnvironment, targetDescription: String?): AnalysisResult? { @@ -331,7 +357,9 @@ object KotlinToJVMBytecodeCompiler { sourceFiles: List, module: Module?, outputDirectory: File?, - moduleName: String?): GenerationState { + moduleName: String?, + onIndependentPartCompilationEnd: GenerationStateEventCallback + ): GenerationState { val configuration = environment.configuration val incrementalCompilationComponents = configuration.get(JVMConfigurationKeys.INCREMENTAL_COMPILATION_COMPONENTS) @@ -369,7 +397,8 @@ object KotlinToJVMBytecodeCompiler { moduleName, outputDirectory, incrementalCompilationComponents, - configuration.get(JVMConfigurationKeys.MULTIFILE_FACADES_OPEN, false)) + configuration.get(JVMConfigurationKeys.MULTIFILE_FACADES_OPEN, false), + onIndependentPartCompilationEnd = onIndependentPartCompilationEnd) ProgressIndicatorAndCompilationCanceledStatus.checkCanceled() val generationStart = PerformanceCounter.currentTime() diff --git a/compiler/tests/org/jetbrains/kotlin/codegen/StdlibTest.java b/compiler/tests/org/jetbrains/kotlin/codegen/StdlibTest.java index 65bd92b5153..66608f1fb5a 100644 --- a/compiler/tests/org/jetbrains/kotlin/codegen/StdlibTest.java +++ b/compiler/tests/org/jetbrains/kotlin/codegen/StdlibTest.java @@ -29,6 +29,7 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinToJVMBytecodeCompiler; import org.jetbrains.kotlin.cli.jvm.config.JvmContentRootsKt; import org.jetbrains.kotlin.codegen.forTestCompile.ForTestCompileRuntime; import org.jetbrains.kotlin.codegen.state.GenerationState; +import org.jetbrains.kotlin.codegen.state.GenerationStateEventCallback; import org.jetbrains.kotlin.config.CompilerConfiguration; import org.jetbrains.kotlin.config.ContentRootsKt; import org.jetbrains.kotlin.descriptors.ClassDescriptor; @@ -61,7 +62,8 @@ public class StdlibTest extends KotlinTestWithEnvironment { } public void testStdlib() throws ClassNotFoundException { - GenerationState state = KotlinToJVMBytecodeCompiler.INSTANCE.analyzeAndGenerate(getEnvironment()); + GenerationState state = KotlinToJVMBytecodeCompiler.INSTANCE.analyzeAndGenerate( + getEnvironment(), GenerationStateEventCallback.Companion.getDO_NOTHING()); if (state == null) { fail("There were compilation errors"); }