Profiling and repeat support for JVM CLI Compiler

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2JVMCompilerArguments.kt

@@ -352,6 +352,23 @@ class K2JVMCompilerArguments : CommonCompilerArguments() {
     )
     var noKotlinNothingValueException: Boolean by FreezableVar(false)
 
+    @Argument(
+        value = "-Xprofile",
+        valueDescription = "<profilerPath:command:outputDir>",
+        description = "Debug option: Run compiler with async profiler, save snapshots to outputDir, command is passed to async-profiler on start\n" +
+                "You'll have to provide async-profiler.jar on classpath to use this\n" +
+                "profilerPath is a path to libasyncProfiler.so\n" +
+                "Example: -Xprofile=<PATH_TO_ASYNC_PROFILER>/async-profiler/build/libasyncProfiler.so:event=cpu,interval=1ms,threads,start,framebuf=50000000:<SNAPSHOT_DIR_PATH>"
+    )
+    var profileCompilerCommand: String? by NullableStringFreezableVar(null)
+
+    @Argument(
+        value = "-Xrepeat",
+        valueDescription = "<number>",
+        description = "Debug option: Repeats modules compilation <number> times"
+    )
+    var repeatCompileModules: String? by NullableStringFreezableVar(null)
+
     override fun configureAnalysisFlags(collector: MessageCollector): MutableMap<AnalysisFlag<*>, Any> {
         val result = super.configureAnalysisFlags(collector)
         result[JvmAnalysisFlags.strictMetadataVersionSemantics] = strictMetadataVersionSemantics

compiler/cli/src/org/jetbrains/kotlin/cli/common/CLICompiler.kt

@@ -45,6 +45,8 @@ abstract class CLICompiler<A : CommonCompilerArguments> : CLITool<A>() {
 
     protected abstract val performanceManager: CommonCompilerPerformanceManager
 
+    protected open fun createPerformanceManager(arguments: A): CommonCompilerPerformanceManager = performanceManager
+
     // Used in CompilerRunnerUtil#invokeExecMethod, in Eclipse plugin (KotlinCLICompiler) and in kotlin-gradle-plugin (GradleCompilerRunner)
     fun execAndOutputXml(errStream: PrintStream, services: Services, vararg args: String): ExitCode {
         return exec(errStream, services, MessageRenderer.XML, args)
@@ -56,7 +58,7 @@ abstract class CLICompiler<A : CommonCompilerArguments> : CLITool<A>() {
     }
 
     public override fun execImpl(messageCollector: MessageCollector, services: Services, arguments: A): ExitCode {
-        val performanceManager = performanceManager
+        val performanceManager = createPerformanceManager(arguments)
         if (arguments.reportPerf || arguments.dumpPerf != null) {
             performanceManager.enableCollectingPerformanceStatistics()
         }

compiler/cli/src/org/jetbrains/kotlin/cli/common/CLIConfigurationKeys.java

@@ -48,6 +48,9 @@ public class CLIConfigurationKeys {
     public static final CompilerConfigurationKey<PhaseConfig> PHASE_CONFIG =
             CompilerConfigurationKey.create("phase configuration");
 
+    public static final CompilerConfigurationKey<Integer> REPEAT_COMPILE_MODULES =
+            CompilerConfigurationKey.create("debug key for profiling, repeats compileModules");
+
     private CLIConfigurationKeys() {
     }
 }

compiler/cli/src/org/jetbrains/kotlin/cli/common/CommonCompilerPerformanceManager.kt

@@ -87,4 +87,6 @@ abstract class CommonCompilerPerformanceManager(private val presentableName: Str
         appendln("$presentableName performance report")
         measurements.map { it.render() }.sorted().forEach { appendln(it) }
     }.toByteArray()
+
+    open fun notifyRepeat(total: Int, number: Int) {}
 }

compiler/cli/src/org/jetbrains/kotlin/cli/common/profiling/AsyncProfilerWrapper.kt

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.cli.common.profiling
+
+import java.lang.invoke.MethodHandles
+import java.lang.invoke.MethodType
+
+interface AsyncProfilerReflected {
+    fun execute(command: String): String
+    fun stop()
+    val version: String
+}
+
+
+
+object AsyncProfilerHelper {
+    private val profilerClass = Class.forName("one.profiler.AsyncProfiler")
+    private val getInstanceHandle =
+        MethodHandles.lookup().findStatic(profilerClass, "getInstance", MethodType.methodType(profilerClass, String::class.java))
+    private val executeHandle =
+        MethodHandles.lookup().findVirtual(
+            profilerClass,
+            "execute",
+            MethodType.methodType(String::class.java, String::class.java)
+        )
+    private val stopHandle =
+        MethodHandles.lookup().findVirtual(profilerClass, "stop", MethodType.methodType(Void.TYPE))
+    private val getVersionHandle =
+        MethodHandles.lookup().findVirtual(profilerClass, "getVersion", MethodType.methodType(String::class.java))
+
+    fun getInstance(libPath: String?): AsyncProfilerReflected {
+        val instance = getInstanceHandle.invokeWithArguments(libPath)
+        return object : AsyncProfilerReflected {
+            private val boundExecute = executeHandle.bindTo(instance)
+            private val boundStop = stopHandle.bindTo(instance)
+            private val boundGetVersion = getVersionHandle.bindTo(instance)
+
+            override fun execute(command: String): String {
+                return boundExecute.invokeWithArguments(command) as String
+            }
+
+            override fun stop() {
+                boundStop.invokeWithArguments()
+            }
+
+            override val version: String
+                get() = boundGetVersion.invokeWithArguments() as String
+
+        }
+    }
+}

compiler/cli/src/org/jetbrains/kotlin/cli/common/profiling/ProfilingCompilerPerformanceManager.kt

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.cli.common.profiling
+
+import org.jetbrains.kotlin.cli.common.CommonCompilerPerformanceManager
+import java.io.File
+import java.text.SimpleDateFormat
+import java.util.*
+
+class ProfilingCompilerPerformanceManager(
+    profilerPath: String,
+    val command: String,
+    val outputDir: File
+) : CommonCompilerPerformanceManager("Profiling") {
+    private val profiler = AsyncProfilerHelper.getInstance(profilerPath)
+
+    private val runDate = Date()
+    private val formatter = SimpleDateFormat("yyyy-MM-dd__HH-mm")
+    private var active = false
+
+    init {
+        startProfiling()
+    }
+
+    private fun startProfiling() {
+        profiler.execute(command)
+        active = true
+    }
+
+    private fun stopProfiling() {
+        if (active) profiler.stop()
+        active = false
+    }
+
+    private fun restartProfiling() {
+        stopProfiling()
+        startProfiling()
+    }
+
+    private fun dumpProfile(postfix: String) {
+        outputDir.mkdirs()
+        val outputFile = outputDir.resolve("snapshot-${formatter.format(runDate)}-$postfix.collapsed")
+        outputFile.writeText(profiler.execute("collapsed"))
+        active = false
+    }
+
+    override fun notifyRepeat(total: Int, number: Int) {
+        dumpProfile("repeat$number")
+        restartProfiling()
+    }
+
+    override fun notifyCompilationFinished() {
+        dumpProfile("final")
+        stopProfiling()
+    }
+
+    companion object {
+        fun create(profileCompilerArgument: String): ProfilingCompilerPerformanceManager {
+            val (path, command, outputDir) = profileCompilerArgument.split(":", limit = 3)
+            return ProfilingCompilerPerformanceManager(path, command, File(outputDir))
+        }
+    }
+}

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/K2JVMCompiler.kt

@@ -31,6 +31,7 @@ import org.jetbrains.kotlin.cli.common.messages.MessageUtil
 import org.jetbrains.kotlin.cli.common.messages.OutputMessageUtil
 import org.jetbrains.kotlin.cli.common.modules.ModuleBuilder
 import org.jetbrains.kotlin.cli.common.modules.ModuleChunk
+import org.jetbrains.kotlin.cli.common.profiling.ProfilingCompilerPerformanceManager
 import org.jetbrains.kotlin.cli.jvm.compiler.CompileEnvironmentUtil
 import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
@@ -50,8 +51,6 @@ import java.io.File
 
 class K2JVMCompiler : CLICompiler<K2JVMCompilerArguments>() {
 
-    override val performanceManager = K2JVMCompilerPerformanceManager()
-
     override fun doExecute(
         arguments: K2JVMCompilerArguments,
         configuration: CompilerConfiguration,
@@ -60,6 +59,7 @@ class K2JVMCompiler : CLICompiler<K2JVMCompilerArguments>() {
     ): ExitCode {
         val messageCollector = configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)
 
+        configuration.putIfNotNull(CLIConfigurationKeys.REPEAT_COMPILE_MODULES, arguments.repeatCompileModules?.toIntOrNull())
         configuration.put(CLIConfigurationKeys.PHASE_CONFIG, createPhaseConfig(jvmPhases, arguments, messageCollector))
 
         if (!configuration.configureJdkHome(arguments)) return COMPILATION_ERROR
@@ -219,7 +219,7 @@ class K2JVMCompiler : CLICompiler<K2JVMCompilerArguments>() {
 
         val environment = KotlinCoreEnvironment.createForProduction(rootDisposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES)
 
-        performanceManager.notifyCompilerInitialized()
+        configuration[CLIConfigurationKeys.PERF_MANAGER]?.notifyCompilerInitialized()
 
         return if (messageCollector.hasErrors()) null else environment
     }
@@ -263,6 +263,14 @@ class K2JVMCompiler : CLICompiler<K2JVMCompilerArguments>() {
         }
 
     }
+
+    override val performanceManager: CommonCompilerPerformanceManager
+        get() = error("Unsupported")
+
+    override fun createPerformanceManager(arguments: K2JVMCompilerArguments): CommonCompilerPerformanceManager {
+        val argument = arguments.profileCompilerCommand ?: return K2JVMCompilerPerformanceManager()
+        return ProfilingCompilerPerformanceManager.create(argument)
+    }
 }
 
 

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinToJVMBytecodeCompiler.kt

@@ -162,9 +162,24 @@ object KotlinToJVMBytecodeCompiler {
         }
     }
 
-    internal fun compileModules(environment: KotlinCoreEnvironment, buildFile: File?, chunk: List<Module>): Boolean {
+    internal fun compileModules(
+        environment: KotlinCoreEnvironment,
+        buildFile: File?,
+        chunk: List<Module>,
+        repeat: Boolean = false
+    ): Boolean {
         ProgressIndicatorAndCompilationCanceledStatus.checkCanceled()
 
+        val repeats = environment.configuration[CLIConfigurationKeys.REPEAT_COMPILE_MODULES]
+        if (repeats != null && !repeat) {
+            val performanceManager = environment.configuration[CLIConfigurationKeys.PERF_MANAGER]
+            return (0 until repeats).map {
+                val result = compileModules(environment, buildFile, chunk, repeat = true)
+                performanceManager?.notifyRepeat(repeats, it)
+                result
+            }.last()
+        }
+
         val moduleVisibilityManager = ModuleVisibilityManager.SERVICE.getInstance(environment.project)
         for (module in chunk) {
             moduleVisibilityManager.addModule(module)
@@ -289,6 +304,8 @@ object KotlinToJVMBytecodeCompiler {
 
     private fun compileModulesUsingFrontendIR(environment: KotlinCoreEnvironment, buildFile: File?, chunk: List<Module>): Boolean {
         val project = environment.project
+        val performanceManager = environment.configuration.get(CLIConfigurationKeys.PERF_MANAGER)
+
         Extensions.getArea(project)
             .getExtensionPoint(PsiElementFinder.EP_NAME)
             .unregisterExtension(JavaElementFinder::class.java)
@@ -297,6 +314,7 @@ object KotlinToJVMBytecodeCompiler {
         val localFileSystem = VirtualFileManager.getInstance().getFileSystem(StandardFileSystems.FILE_PROTOCOL)
         val outputs = newLinkedHashMapWithExpectedSize<Module, GenerationState>(chunk.size)
         for (module in chunk) {
+            performanceManager?.notifyAnalysisStarted()
             ProgressIndicatorAndCompilationCanceledStatus.checkCanceled()
 
             val ktFiles = module.getSourceFiles(environment, localFileSystem, chunk.size > 1, buildFile)
@@ -349,6 +367,11 @@ object KotlinToJVMBytecodeCompiler {
                 }
             }
 
+            val debugTargetDescription = "target " + module.getModuleName() + "-" + module.getModuleType() + " "
+            val codeLines = environment.countLinesOfCode(ktFiles)
+            performanceManager?.notifyAnalysisFinished(ktFiles.size, codeLines, debugTargetDescription)
+
+            performanceManager?.notifyGenerationStarted()
             val signaturer = IdSignatureDescriptor(JvmManglerDesc())
 
             val (moduleFragment, symbolTable, sourceManager, components) =
@@ -379,8 +402,7 @@ object KotlinToJVMBytecodeCompiler {
 
             ProgressIndicatorAndCompilationCanceledStatus.checkCanceled()
 
-            val performanceManager = environment.configuration.get(CLIConfigurationKeys.PERF_MANAGER)
-            performanceManager?.notifyGenerationStarted()
+
             generationState.beforeCompile()
             codegenFactory.generateModuleInFrontendIRMode(
                 generationState, moduleFragment, symbolTable, sourceManager
@@ -391,8 +413,8 @@ object KotlinToJVMBytecodeCompiler {
             generationState.factory.done()
             performanceManager?.notifyGenerationFinished(
                 ktFiles.size,
-                environment.countLinesOfCode(ktFiles),
-                additionalDescription = "target " + module.getModuleName() + "-" + module.getModuleType() + " "
+                codeLines,
+                additionalDescription = debugTargetDescription
             )
 
             ProgressIndicatorAndCompilationCanceledStatus.checkCanceled()