From 9becb2c468fbdf6e37880ff86f10519219f395f3 Mon Sep 17 00:00:00 2001 From: Stanislav Erokhin Date: Fri, 2 Jul 2021 19:35:34 +0300 Subject: [PATCH] Add compiler performance metrics WIP --- .../jetbrains/kotlin/cli/js/K2JSCompiler.java | 4 ++-- .../jetbrains/kotlin/cli/js/K2JsIrCompiler.kt | 2 +- .../kotlin/cli/common/CLICompiler.kt | 4 ++-- .../CommonCompilerPerformanceManager.kt | 24 ++++++++++++++++--- .../jetbrains/kotlin/cli/jvm/K2JVMCompiler.kt | 5 ++-- .../kotlin/cli/metadata/K2MetadataCompiler.kt | 9 +++++-- .../cli/metadata/K2MetadataKlibSerializer.kt | 5 ++++ .../kotlin/cli/metadata/MetadataSerializer.kt | 7 ++++++ .../kotlin/daemon/CompileServiceImpl.kt | 7 +++++- .../IncrementalJsCompilerRunner.kt | 4 +++- .../IncrementalJvmCompilerRunner.kt | 1 + .../org/jetbrains/kotlin/cli/bc/K2Native.kt | 2 +- .../GradleKotlinCompilerWork.kt | 6 ++++- 13 files changed, 63 insertions(+), 17 deletions(-) diff --git a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java index d08ea9469cd..47db4aeca08 100644 --- a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java +++ b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java @@ -99,7 +99,7 @@ public class K2JSCompiler extends CLICompiler { doMain(new K2JSCompiler(), args); } - private final K2JSCompilerPerformanceManager performanceManager = new K2JSCompilerPerformanceManager(); + final K2JSCompilerPerformanceManager performanceManager = new K2JSCompilerPerformanceManager(); @NotNull @Override @@ -564,7 +564,7 @@ public class K2JSCompiler extends CLICompiler { @NotNull @Override - protected CommonCompilerPerformanceManager getPerformanceManager() { + public CommonCompilerPerformanceManager getDefaultPerformanceManager() { return performanceManager; } diff --git a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt index 95f5653cea9..034c65393b7 100644 --- a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt +++ b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt @@ -66,7 +66,7 @@ enum class ProduceKind { class K2JsIrCompiler : CLICompiler() { - override val performanceManager: CommonCompilerPerformanceManager = + override val defaultPerformanceManager: CommonCompilerPerformanceManager = object : CommonCompilerPerformanceManager("Kotlin to JS (IR) Compiler") {} override fun createArguments(): K2JSCompilerArguments { diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/common/CLICompiler.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/common/CLICompiler.kt index 8c27274f5f4..6272347ab90 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/common/CLICompiler.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/common/CLICompiler.kt @@ -43,9 +43,9 @@ import java.io.PrintStream abstract class CLICompiler : CLITool() { - protected abstract val performanceManager: CommonCompilerPerformanceManager + abstract val defaultPerformanceManager: CommonCompilerPerformanceManager - protected open fun createPerformanceManager(arguments: A, services: Services): CommonCompilerPerformanceManager = performanceManager + protected open fun createPerformanceManager(arguments: A, services: Services): CommonCompilerPerformanceManager = defaultPerformanceManager // Used in CompilerRunnerUtil#invokeExecMethod, in Eclipse plugin (KotlinCLICompiler) and in kotlin-gradle-plugin (GradleCompilerRunner) fun execAndOutputXml(errStream: PrintStream, services: Services, vararg args: String): ExitCode { diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/common/CommonCompilerPerformanceManager.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/common/CommonCompilerPerformanceManager.kt index 267bb100434..e768ee7c957 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/common/CommonCompilerPerformanceManager.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/common/CommonCompilerPerformanceManager.kt @@ -5,6 +5,8 @@ package org.jetbrains.kotlin.cli.common +import com.intellij.openapi.util.text.StringUtil +import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.util.PerformanceCounter import java.io.File import java.lang.management.GarbageCollectorMXBean @@ -26,6 +28,7 @@ abstract class CommonCompilerPerformanceManager(private val presentableName: Str private var irGenerationStart: Long = 0 private var targetDescription: String? = null + private var sourceFiles: List? = null protected var files: Int? = null protected var lines: Int? = null @@ -42,12 +45,19 @@ abstract class CommonCompilerPerformanceManager(private val presentableName: Str private fun deltaTime(start: Long): Long = PerformanceCounter.currentTime() - start - open fun notifyCompilerInitialized(files: Int, lines: Int, targetDescription: String) { + private fun countLinesOfCode(sourceFiles: List): Int = + sourceFiles.sumBy { sourceFile -> + val text = sourceFile.text + StringUtil.getLineBreakCount(text) + (if (StringUtil.endsWithLineBreak(text)) 0 else 1) + } + + open fun notifyCompilerInitialized(sourceFiles: List, targetDescription: String) { if (!isEnabled) return recordInitializationTime() - this.files = files - this.lines = lines + this.sourceFiles = sourceFiles + this.files = sourceFiles.size + this.lines = countLinesOfCode(sourceFiles) this.targetDescription = targetDescription } @@ -160,4 +170,12 @@ abstract class CommonCompilerPerformanceManager(private val presentableName: Str private data class GCData(val name: String, val collectionTime: Long, val collectionCount: Long) { constructor(bean: GarbageCollectorMXBean) : this(bean.name, bean.collectionTime, bean.collectionCount) } + + fun renderCompilerPerformance(): String { + val relevantMeasurements = getMeasurementResults().filter { + it is CompilerInitializationMeasurement || it is CodeAnalysisMeasurement || it is CodeGenerationMeasurement || it is PerformanceCounterMeasurement + } + + return "Compiler perf stats:\n" + relevantMeasurements.joinToString(separator = "\n") { " ${it.render()}" } + } } diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/K2JVMCompiler.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/K2JVMCompiler.kt index c8632cadea5..41ae2d3edfb 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/K2JVMCompiler.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/K2JVMCompiler.kt @@ -228,7 +228,7 @@ class K2JVMCompiler : CLICompiler() { val sourceFiles = environment.getSourceFiles() configuration[CLIConfigurationKeys.PERF_MANAGER]?.notifyCompilerInitialized( - sourceFiles.size, environment.countLinesOfCode(sourceFiles), targetDescription + sourceFiles, targetDescription ) return if (messageCollector.hasErrors()) null else environment @@ -274,8 +274,7 @@ class K2JVMCompiler : CLICompiler() { } - override val performanceManager: CommonCompilerPerformanceManager - get() = error("Unsupported") + override val defaultPerformanceManager: CommonCompilerPerformanceManager = K2JVMCompilerPerformanceManager() override fun createPerformanceManager(arguments: K2JVMCompilerArguments, services: Services): CommonCompilerPerformanceManager { val externalManager = services[CommonCompilerPerformanceManager::class.java] diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataCompiler.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataCompiler.kt index e35fa2a6577..b3cd8e1aa07 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataCompiler.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataCompiler.kt @@ -39,7 +39,7 @@ import java.io.File class K2MetadataCompiler : CLICompiler() { - override val performanceManager = K2MetadataCompilerPerformanceManager() + override val defaultPerformanceManager: CommonCompilerPerformanceManager = K2MetadataCompilerPerformanceManager() override fun createArguments() = K2MetadataCompilerArguments() @@ -58,6 +58,7 @@ class K2MetadataCompiler : CLICompiler() { paths: KotlinPaths? ): ExitCode { val collector = configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY) + val performanceManager = configuration.getNotNull(CLIConfigurationKeys.PERF_MANAGER) val pluginLoadResult = loadPlugins(paths, arguments, configuration) if (pluginLoadResult != ExitCode.OK) return pluginLoadResult @@ -70,7 +71,8 @@ class K2MetadataCompiler : CLICompiler() { configuration.addJvmClasspathRoots(arguments.classpath!!.split(File.pathSeparatorChar).map(::File)) } - configuration.put(CommonConfigurationKeys.MODULE_NAME, arguments.moduleName ?: JvmProtoBufUtil.DEFAULT_MODULE_NAME) + val moduleName = arguments.moduleName ?: JvmProtoBufUtil.DEFAULT_MODULE_NAME + configuration.put(CommonConfigurationKeys.MODULE_NAME, moduleName) configuration.put(CLIConfigurationKeys.ALLOW_KOTLIN_PACKAGE, arguments.allowKotlinPackage) @@ -93,6 +95,9 @@ class K2MetadataCompiler : CLICompiler() { val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, EnvironmentConfigFiles.METADATA_CONFIG_FILES) + val mode = if(arguments.expectActualLinker) "KLib" else "metadata" + performanceManager.notifyCompilerInitialized(environment.getSourceFiles(), "$mode mode for $moduleName module") + if (environment.getSourceFiles().isEmpty()) { if (arguments.version) { return ExitCode.OK diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataKlibSerializer.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataKlibSerializer.kt index 160584fbcac..c5742927008 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataKlibSerializer.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/K2MetadataKlibSerializer.kt @@ -48,20 +48,25 @@ import java.io.File internal class K2MetadataKlibSerializer(private val metadataVersion: BuiltInsBinaryVersion) { fun serialize(environment: KotlinCoreEnvironment) { val configuration = environment.configuration + val performanceManager = configuration.getNotNull(CLIConfigurationKeys.PERF_MANAGER) val dependencyContainer = KlibMetadataDependencyContainer( configuration, LockBasedStorageManager("K2MetadataKlibSerializer") ) + performanceManager.notifyAnalysisStarted() val analyzer = runCommonAnalysisForSerialization(environment, true, dependencyContainer) + performanceManager.notifyAnalysisFinished() if (analyzer == null || analyzer.hasErrors()) return val (_, moduleDescriptor) = analyzer.analysisResult + performanceManager.notifyGenerationStarted() val destDir = checkNotNull(environment.destDir) performSerialization(configuration, moduleDescriptor, destDir, environment.project) + performanceManager.notifyGenerationFinished() } private fun performSerialization( diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/MetadataSerializer.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/MetadataSerializer.kt index bc2d56cf7bd..1ce733853e9 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/MetadataSerializer.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/MetadataSerializer.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.cli.metadata import com.intellij.openapi.project.Project +import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.codegen.JvmCodegenUtil import org.jetbrains.kotlin.descriptors.ClassDescriptor @@ -39,14 +40,20 @@ open class MetadataSerializer( protected var totalFiles = 0 fun serialize(environment: KotlinCoreEnvironment) { + val performanceManager = environment.configuration.getNotNull(CLIConfigurationKeys.PERF_MANAGER) + + performanceManager.notifyAnalysisStarted() val analyzer = runCommonAnalysisForSerialization(environment, dependOnOldBuiltIns, dependencyContainer = null) + performanceManager.notifyAnalysisFinished() if (analyzer == null || analyzer.hasErrors()) return val (bindingContext, moduleDescriptor) = analyzer.analysisResult + performanceManager.notifyGenerationStarted() val destDir = checkNotNull(environment.destDir) performSerialization(environment.getSourceFiles(), bindingContext, moduleDescriptor, destDir, environment.project) + performanceManager.notifyGenerationFinished() } protected open fun performSerialization( diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt index e04ebf0b397..71dedf3f782 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt @@ -326,7 +326,12 @@ abstract class CompileServiceImplBase( } CompilerMode.NON_INCREMENTAL_COMPILER -> { doCompile(sessionId, daemonReporter, tracer = null) { _, _ -> - compiler.exec(messageCollector, Services.EMPTY, k2PlatformArgs) + val exitCode = compiler.exec(messageCollector, Services.EMPTY, k2PlatformArgs) + + val perfString = compiler.defaultPerformanceManager.renderCompilerPerformance() + (compilationResults as CompilationResults).add(CompilationResultCategory.BUILD_REPORT_LINES.code, arrayListOf(perfString)) + + exitCode } } CompilerMode.INCREMENTAL_COMPILER -> { diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt index 206a6019976..67b731f99bf 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt @@ -186,11 +186,13 @@ class IncrementalJsCompilerRunner( ): ExitCode { val freeArgsBackup = args.freeArgs + val compiler = K2JSCompiler() return try { args.freeArgs += sourcesToCompile.map { it.absolutePath } - K2JSCompiler().exec(messageCollector, services, args) + compiler.exec(messageCollector, services, args) } finally { args.freeArgs = freeArgsBackup + reporter.report { compiler.defaultPerformanceManager.renderCompilerPerformance() } } } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt index cd4ce90f98f..3932af62ea9 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt @@ -433,6 +433,7 @@ class IncrementalJvmCompilerRunner( args.allowNoSourceFiles = true val exitCode = compiler.exec(messageCollector, services, args) args.freeArgs = freeArgsBackup + reporter.report { compiler.defaultPerformanceManager.renderCompilerPerformance() } return exitCode } } diff --git a/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt b/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt index ba52d078d74..b7a26f35e9b 100644 --- a/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt +++ b/kotlin-native/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt @@ -40,7 +40,7 @@ class K2Native : CLICompiler() { override fun createMetadataVersion(versionArray: IntArray): BinaryVersion = KlibMetadataVersion(*versionArray) - override val performanceManager:CommonCompilerPerformanceManager by lazy { + override val defaultPerformanceManager: CommonCompilerPerformanceManager by lazy { K2NativeCompilerPerformanceManager() } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt index 09ef4689730..5b778429421 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt @@ -270,8 +270,12 @@ internal class GradleKotlinCompilerWork @Inject constructor( kotlinScriptExtensions = kotlinScriptExtensions ) val servicesFacade = GradleCompilerServicesFacadeImpl(log, bufferingMessageCollector) + val compilationResults = GradleCompilationResults(log, projectRootFile) return metrics.measure(BuildTime.NON_INCREMENTAL_COMPILATION_DAEMON) { - daemon.compile(sessionId, compilerArgs, compilationOptions, servicesFacade, compilationResults = null) + daemon.compile(sessionId, compilerArgs, compilationOptions, servicesFacade, compilationResults) + }.also { + metrics.addMetrics(compilationResults.buildMetrics) + icLogLines = compilationResults.icLogLines } }