Use Workers API for NMPP tasks

This change enables parallel execution of compile tasks in NMPP projects within a subproject. #KT-28155 In Progress
2018-10-25 00:30:15 +03:00
parent ba5795519b
commit 37dfe2b608
22 changed files with 700 additions and 468 deletions
@@ -0,0 +1,12 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.compilerRunner
+
+object KotlinCompilerClass {
+    const val JVM = "org.jetbrains.kotlin.cli.jvm.K2JVMCompiler"
+    const val JS = "org.jetbrains.kotlin.cli.js.K2JSCompiler"
+    const val METADATA = "org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler"
+}
@@ -43,115 +43,28 @@ interface KotlinLogger {
 }

 abstract class KotlinCompilerRunner<in Env : CompilerEnvironment> {
-    protected val K2JVM_COMPILER = "org.jetbrains.kotlin.cli.jvm.K2JVMCompiler"
-    protected val K2JS_COMPILER = "org.jetbrains.kotlin.cli.js.K2JSCompiler"
-    protected val K2METADATA_COMPILER = "org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler"
    protected val INTERNAL_ERROR = ExitCode.INTERNAL_ERROR.toString()

    protected abstract val log: KotlinLogger

-    protected abstract fun getDaemonConnection(environment: Env): CompileServiceSession?
-
-    @Synchronized
-    protected fun newDaemonConnection(
-            compilerId: CompilerId,
-            clientAliveFlagFile: File,
-            sessionAliveFlagFile: File,
-            environment: Env,
-            daemonOptions: DaemonOptions = configureDaemonOptions(),
-            additionalJvmParams: Array<String> = arrayOf()
-    ): CompileServiceSession? {
-        val daemonJVMOptions = configureDaemonJVMOptions(
-            additionalParams = *additionalJvmParams,
-            inheritMemoryLimits = true,
-            inheritOtherJvmOptions = false,
-            inheritAdditionalProperties = true
-        )
-
-        val daemonReportMessages = ArrayList<DaemonReportMessage>()
-        val daemonReportingTargets = DaemonReportingTargets(messages = daemonReportMessages)
-
-        val profiler = if (daemonOptions.reportPerf) WallAndThreadAndMemoryTotalProfiler(withGC = false) else DummyProfiler()
-
-        val connection = profiler.withMeasure(null) {
-            KotlinCompilerClient.connectAndLease(compilerId,
-                                                 clientAliveFlagFile,
-                                                 daemonJVMOptions,
-                                                 daemonOptions,
-                                                 daemonReportingTargets,
-                                                 autostart = true,
-                                                 leaseSession = true,
-                                                 sessionAliveFlagFile = sessionAliveFlagFile)
-        }
-
-        if (connection == null || log.isDebugEnabled) {
-            for (message in daemonReportMessages) {
-                val severity = when(message.category) {
-                    DaemonReportCategory.DEBUG -> CompilerMessageSeverity.INFO
-                    DaemonReportCategory.INFO -> CompilerMessageSeverity.INFO
-                    DaemonReportCategory.EXCEPTION -> CompilerMessageSeverity.EXCEPTION
-                }
-                environment.messageCollector.report(severity, message.message)
-            }
-        }
-
-        fun reportTotalAndThreadPerf(message: String, daemonOptions: DaemonOptions, messageCollector: MessageCollector, profiler: Profiler) {
-            if (daemonOptions.reportPerf) {
-                fun Long.ms() = TimeUnit.NANOSECONDS.toMillis(this)
-                val counters = profiler.getTotalCounters()
-                messageCollector.report(CompilerMessageSeverity.INFO, "PERF: $message ${counters.time.ms()} ms, thread ${counters.threadTime.ms()}")
-            }
-        }
-
-        reportTotalAndThreadPerf("Daemon connect", daemonOptions, environment.messageCollector, profiler)
-        return connection
-    }
-
-    protected fun processCompilerOutput(
-            environment: Env,
-            stream: ByteArrayOutputStream,
-            exitCode: ExitCode?
+    protected open fun runCompiler(
+        compilerClassName: String,
+        compilerArgs: CommonCompilerArguments,
+        environment: Env
    ) {
-        val reader = BufferedReader(StringReader(stream.toString()))
-        CompilerOutputParser.parseCompilerMessagesFromReader(environment.messageCollector, reader, environment.outputItemsCollector)
-
-        if (ExitCode.INTERNAL_ERROR == exitCode) {
-            reportInternalCompilerError(environment.messageCollector)
-        }
-    }
-
-    protected fun reportInternalCompilerError(messageCollector: MessageCollector): ExitCode {
-        messageCollector.report(CompilerMessageSeverity.ERROR, "Compiler terminated with internal error")
-        return ExitCode.INTERNAL_ERROR
-    }
-
-    protected fun runCompiler(
-            compilerClassName: String,
-            compilerArgs: CommonCompilerArguments,
-            environment: Env): ExitCode {
-        return try {
+        try {
            compileWithDaemonOrFallback(compilerClassName, compilerArgs, environment)
-        }
-        catch (e: Throwable) {
+        } catch (e: Throwable) {
            MessageCollectorUtil.reportException(environment.messageCollector, e)
            reportInternalCompilerError(environment.messageCollector)
        }
    }

    protected abstract fun compileWithDaemonOrFallback(
-            compilerClassName: String,
-            compilerArgs: CommonCompilerArguments,
-            environment: Env
-    ): ExitCode
-
-    /**
-     * Returns null if could not connect to daemon
-     */
-    protected abstract fun compileWithDaemon(
-            compilerClassName: String,
-            compilerArgs: CommonCompilerArguments,
-            environment: Env
-    ): ExitCode?
+        compilerClassName: String,
+        compilerArgs: CommonCompilerArguments,
+        environment: Env
+    )

    protected fun exitCodeFromProcessExitCode(code: Int): ExitCode = Companion.exitCodeFromProcessExitCode(log, code)

@@ -163,6 +76,69 @@ abstract class KotlinCompilerRunner<in Env : CompilerEnvironment> {
            log.debug("Could not find exit code by value: $code")
            return if (code == 0) ExitCode.OK else ExitCode.COMPILATION_ERROR
        }
+
+        @Synchronized
+        @JvmStatic
+        protected fun newDaemonConnection(
+            compilerId: CompilerId,
+            clientAliveFlagFile: File,
+            sessionAliveFlagFile: File,
+            messageCollector: MessageCollector,
+            isDebugEnabled: Boolean,
+            daemonOptions: DaemonOptions = configureDaemonOptions(),
+            additionalJvmParams: Array<String> = arrayOf()
+        ): CompileServiceSession? {
+            val daemonJVMOptions = configureDaemonJVMOptions(
+                additionalParams = *additionalJvmParams,
+                inheritMemoryLimits = true,
+                inheritOtherJvmOptions = false,
+                inheritAdditionalProperties = true
+            )
+
+            val daemonReportMessages = ArrayList<DaemonReportMessage>()
+            val daemonReportingTargets = DaemonReportingTargets(messages = daemonReportMessages)
+
+            val profiler = if (daemonOptions.reportPerf) WallAndThreadAndMemoryTotalProfiler(withGC = false) else DummyProfiler()
+
+            val connection = profiler.withMeasure(null) {
+                KotlinCompilerClient.connectAndLease(
+                    compilerId,
+                    clientAliveFlagFile,
+                    daemonJVMOptions,
+                    daemonOptions,
+                    daemonReportingTargets,
+                    autostart = true,
+                    leaseSession = true,
+                    sessionAliveFlagFile = sessionAliveFlagFile
+                )
+            }
+
+            if (connection == null || isDebugEnabled) {
+                for (message in daemonReportMessages) {
+                    val severity = when (message.category) {
+                        DaemonReportCategory.DEBUG -> CompilerMessageSeverity.INFO
+                        DaemonReportCategory.INFO -> CompilerMessageSeverity.INFO
+                        DaemonReportCategory.EXCEPTION -> CompilerMessageSeverity.EXCEPTION
+                    }
+                    messageCollector.report(severity, message.message)
+                }
+            }
+
+            fun reportTotalAndThreadPerf(
+                message: String, daemonOptions: DaemonOptions, messageCollector: MessageCollector, profiler: Profiler
+            ) {
+                if (daemonOptions.reportPerf) {
+                    fun Long.ms() = TimeUnit.NANOSECONDS.toMillis(this)
+                    val counters = profiler.getTotalCounters()
+                    messageCollector.report(
+                        CompilerMessageSeverity.INFO, "PERF: $message ${counters.time.ms()} ms, thread ${counters.threadTime.ms()}"
+                    )
+                }
+            }
+
+            reportTotalAndThreadPerf("Daemon connect", daemonOptions, MessageCollector.NONE, profiler)
+            return connection
+        }
    }
 }

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.compilerRunner
+
+import org.jetbrains.kotlin.cli.common.ExitCode
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import java.io.BufferedReader
+import java.io.ByteArrayOutputStream
+import java.io.StringReader
+
+fun reportInternalCompilerError(messageCollector: MessageCollector) {
+    messageCollector.report(CompilerMessageSeverity.ERROR, "Compiler terminated with internal error")
+}
+
+fun processCompilerOutput(
+    environment: CompilerEnvironment,
+    stream: ByteArrayOutputStream,
+    exitCode: ExitCode?
+) {
+    processCompilerOutput(environment.messageCollector, environment.outputItemsCollector, stream, exitCode)
+}
+
+fun processCompilerOutput(
+    messageCollector: MessageCollector,
+    outputItemsCollector: OutputItemsCollector,
+    stream: ByteArrayOutputStream,
+    exitCode: ExitCode?
+) {
+    val reader = BufferedReader(StringReader(stream.toString()))
+    CompilerOutputParser.parseCompilerMessagesFromReader(messageCollector, reader, outputItemsCollector)
+
+    if (ExitCode.INTERNAL_ERROR == exitCode) {
+        reportInternalCompilerError(messageCollector)
+    }
+}