Report compiler messages from daemon in Gradle thread

#KT-28836 fixed

RMI messages are reported from RMI threads.
When Gradle logger is used from non-Gradle thread,
messages are not grouped in command-line
and they are not shown in build scans.
To fix this, we store all messages from Kotlin daemon
in a buffer, then report them from a Gradle thread.

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleIncrementalCompilerServicesFacadeImpl.kt

@@ -5,17 +5,20 @@
 
 package org.jetbrains.kotlin.compilerRunner
 
-import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.daemon.client.reportFromDaemon
 import org.jetbrains.kotlin.daemon.common.*
 import org.jetbrains.kotlin.gradle.plugin.kotlinDebug
+import org.jetbrains.kotlin.gradle.logging.GradleBufferingMessageCollector
 import java.io.Serializable
 import java.rmi.Remote
 import java.rmi.server.UnicastRemoteObject
 
 internal open class GradleCompilerServicesFacadeImpl(
     private val log: KotlinLogger,
-    private val compilerMessageCollector: MessageCollector,
+    // RMI messages are reported from RMI threads.
+    // Messages reported from non-Gradle threads are not grouped and not shown in build scans.
+    // To fix this, we store all messages in a buffer, then report them from a Gradle thread
+    private val compilerMessageCollector: GradleBufferingMessageCollector,
     port: Int = SOCKET_ANY_FREE_PORT
 ) : UnicastRemoteObject(port, LoopbackNetworkInterface.clientLoopbackSocketFactory, LoopbackNetworkInterface.serverLoopbackSocketFactory),
     CompilerServicesFacadeBase,
@@ -44,7 +47,7 @@ internal open class GradleCompilerServicesFacadeImpl(
 
 internal class GradleIncrementalCompilerServicesFacadeImpl(
     log: KotlinLogger,
-    messageCollector: MessageCollector,
+    messageCollector: GradleBufferingMessageCollector,
     port: Int = SOCKET_ANY_FREE_PORT
 ) : GradleCompilerServicesFacadeImpl(log, messageCollector, port),
     IncrementalCompilerServicesFacade

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt

@@ -6,7 +6,6 @@
 package org.jetbrains.kotlin.compilerRunner
 
 import org.gradle.api.Project
-import org.gradle.api.invocation.Gradle
 import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
@@ -14,7 +13,9 @@ import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.cli.common.messages.MessageRenderer
 import org.jetbrains.kotlin.config.Services
 import org.jetbrains.kotlin.daemon.common.*
+import org.jetbrains.kotlin.gradle.plugin.TaskLoggers
 import org.jetbrains.kotlin.gradle.plugin.kotlinDebug
+import org.jetbrains.kotlin.gradle.logging.GradleBufferingMessageCollector
 import org.jetbrains.kotlin.gradle.tasks.GradleMessageCollector
 import org.jetbrains.kotlin.gradle.tasks.clearLocalStateDirectories
 import org.jetbrains.kotlin.gradle.tasks.throwGradleExceptionIfError
@@ -26,10 +27,8 @@ import java.io.*
 import java.net.URLClassLoader
 import java.rmi.RemoteException
 import java.util.concurrent.Callable
-import java.util.concurrent.ExecutorService
 import java.util.concurrent.Executors
 import javax.inject.Inject
-import kotlin.concurrent.thread
 
 internal class ProjectFilesForCompilation(
     val projectRootFile: File,
@@ -93,7 +92,6 @@ internal class GradleKotlinCompilerWork @Inject constructor(
                 System.err.println("Could not get logger for '$taskPath'. Falling back to sl4j logger")
                 SL4JKotlinLogger(LoggerFactory.getLogger("GradleKotlinCompilerWork"))
             }
-    private val messageCollector = GradleMessageCollector(log)
 
     private val isIncremental: Boolean
         get() = incrementalCompilationEnvironment != null
@@ -103,8 +101,9 @@ internal class GradleKotlinCompilerWork @Inject constructor(
             clearLocalStateDirectories(log, localStateDirectories, "IC is disabled")
         }
 
+        val messageCollector = GradleMessageCollector(log)
         val exitCode = try {
-            compileWithDaemonOrFallbackImpl()
+            compileWithDaemonOrFallbackImpl(messageCollector)
         } catch (e: Throwable) {
             clearLocalStateDirectories(log, localStateDirectories, "exception when running compiler")
             throw e
@@ -127,7 +126,7 @@ internal class GradleKotlinCompilerWork @Inject constructor(
         throwGradleExceptionIfError(exitCode)
     }
 
-    private fun compileWithDaemonOrFallbackImpl(): ExitCode {
+    private fun compileWithDaemonOrFallbackImpl(messageCollector: MessageCollector): ExitCode {
         with(log) {
             kotlinDebug { "Kotlin compiler class: ${compilerClassName}" }
             kotlinDebug { "Kotlin compiler classpath: ${compilerFullClasspath.joinToString { it.canonicalPath }}" }
@@ -136,7 +135,7 @@ internal class GradleKotlinCompilerWork @Inject constructor(
 
         val executionStrategy = kotlinCompilerExecutionStrategy()
         if (executionStrategy == DAEMON_EXECUTION_STRATEGY) {
-            val daemonExitCode = compileWithDaemon()
+            val daemonExitCode = compileWithDaemon(messageCollector)
 
             if (daemonExitCode != null) {
                 return daemonExitCode
@@ -147,21 +146,25 @@ internal class GradleKotlinCompilerWork @Inject constructor(
 
         val isGradleDaemonUsed = System.getProperty("org.gradle.daemon")?.let(String::toBoolean)
         return if (executionStrategy == IN_PROCESS_EXECUTION_STRATEGY || isGradleDaemonUsed == false) {
-            compileInProcess()
+            compileInProcess(messageCollector)
         } else {
             compileOutOfProcess()
         }
     }
 
-    private fun compileWithDaemon(): ExitCode? {
+    private fun compileWithDaemon(messageCollector: MessageCollector): ExitCode? {
+        val isDebugEnabled = log.isDebugEnabled || System.getProperty("kotlin.daemon.debug.log")?.toBoolean() ?: false
+        val daemonMessageCollector =
+            if (isDebugEnabled) messageCollector else MessageCollector.NONE
+
         val connection =
             try {
                 GradleCompilerRunner.getDaemonConnectionImpl(
                     clientIsAliveFlagFile,
                     sessionFlagFile,
                     compilerFullClasspath,
-                    messageCollector,
+                    daemonMessageCollector,
-                    log.isDebugEnabled
+                    isDebugEnabled
                 )
             } catch (e: Throwable) {
                 log.error("Caught an exception trying to connect to Kotlin Daemon:")
@@ -184,14 +187,17 @@ internal class GradleKotlinCompilerWork @Inject constructor(
             KotlinCompilerClass.METADATA -> CompileService.TargetPlatform.METADATA
             else -> throw IllegalArgumentException("Unknown compiler type $compilerClassName")
         }
+        val bufferingMessageCollector = GradleBufferingMessageCollector()
         val exitCode = try {
             val res = if (isIncremental) {
-                incrementalCompilationWithDaemon(daemon, sessionId, targetPlatform)
+                incrementalCompilationWithDaemon(daemon, sessionId, targetPlatform, bufferingMessageCollector)
             } else {
-                nonIncrementalCompilationWithDaemon(daemon, sessionId, targetPlatform)
+                nonIncrementalCompilationWithDaemon(daemon, sessionId, targetPlatform, bufferingMessageCollector)
             }
+            bufferingMessageCollector.flush(messageCollector)
             exitCodeFromProcessExitCode(log, res.get())
         } catch (e: Throwable) {
+            bufferingMessageCollector.flush(messageCollector)
             log.error("Compilation with Kotlin compile daemon was not successful")
             log.error(e.stackTraceAsString())
             null
@@ -211,7 +217,8 @@ internal class GradleKotlinCompilerWork @Inject constructor(
     private fun nonIncrementalCompilationWithDaemon(
         daemon: CompileService,
         sessionId: Int,
-        targetPlatform: CompileService.TargetPlatform
+        targetPlatform: CompileService.TargetPlatform,
+        bufferingMessageCollector: GradleBufferingMessageCollector
     ): CompileService.CallResult<Int> {
         val compilationOptions = CompilationOptions(
             compilerMode = CompilerMode.NON_INCREMENTAL_COMPILER,
@@ -220,14 +227,15 @@ internal class GradleKotlinCompilerWork @Inject constructor(
             reportSeverity = reportSeverity(isVerbose),
             requestedCompilationResults = emptyArray()
         )
-        val servicesFacade = GradleCompilerServicesFacadeImpl(log, messageCollector)
+        val servicesFacade = GradleCompilerServicesFacadeImpl(log, bufferingMessageCollector)
         return daemon.compile(sessionId, compilerArgs, compilationOptions, servicesFacade, compilationResults = null)
     }
 
     private fun incrementalCompilationWithDaemon(
         daemon: CompileService,
         sessionId: Int,
-        targetPlatform: CompileService.TargetPlatform
+        targetPlatform: CompileService.TargetPlatform,
+        bufferingMessageCollector: GradleBufferingMessageCollector
     ): CompileService.CallResult<Int> {
         val icEnv = incrementalCompilationEnvironment ?: error("incrementalCompilationEnvironment is null!")
         val knownChangedFiles = icEnv.changedFiles as? ChangedFiles.Known
@@ -249,7 +257,7 @@ internal class GradleKotlinCompilerWork @Inject constructor(
         )
 
         log.info("Options for KOTLIN DAEMON: $compilationOptions")
-        val servicesFacade = GradleIncrementalCompilerServicesFacadeImpl(log, messageCollector)
+        val servicesFacade = GradleIncrementalCompilerServicesFacadeImpl(log, bufferingMessageCollector)
         val compilationResults = GradleCompilationResults(log, projectRootFile)
         return daemon.compile(sessionId, compilerArgs, compilationOptions, servicesFacade, compilationResults)
     }
@@ -257,21 +265,23 @@ internal class GradleKotlinCompilerWork @Inject constructor(
     private fun compileOutOfProcess(): ExitCode =
         runToolInSeparateProcess(compilerArgs, compilerClassName, compilerFullClasspath, log, loggingMessageCollector)
 
-    private fun compileInProcess(): ExitCode {
+    private fun compileInProcess(messageCollector: MessageCollector): ExitCode {
         // in-process compiler should always be run in a different thread
         // to avoid leaking thread locals from compiler (see KT-28037)
         val threadPool = Executors.newSingleThreadExecutor()
+        val bufferingMessageCollector = GradleBufferingMessageCollector()
         try {
             val future = threadPool.submit(Callable {
-                compileInProcessImpl()
+                compileInProcessImpl(bufferingMessageCollector)
             })
             return future.get()
         } finally {
+            bufferingMessageCollector.flush(messageCollector)
             threadPool.shutdown()
         }
     }
 
-    private fun compileInProcessImpl(): ExitCode {
+    private fun compileInProcessImpl(messageCollector: MessageCollector): ExitCode {
         val stream = ByteArrayOutputStream()
         val out = PrintStream(stream)
         // todo: cache classloader?

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/logging/GradleBufferingMessageCollector.kt

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.logging
+
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import java.util.ArrayList
+
+internal class GradleBufferingMessageCollector : MessageCollector {
+    private class MessageData(
+        val severity: CompilerMessageSeverity,
+        val message: String,
+        val location: CompilerMessageLocation?
+    )
+
+    private val messages = ArrayList<MessageData>()
+
+    override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation?) {
+        synchronized(messages) {
+            messages.add(MessageData(severity, message, location))
+        }
+    }
+
+    override fun hasErrors() =
+        synchronized(messages) {
+            messages.any { it.severity.isError }
+        }
+
+    override fun clear() {
+        synchronized(messages) {
+            messages.clear()
+        }
+    }
+
+    fun flush(delegate: MessageCollector) {
+        messages.forEach {
+            delegate.report(it.severity, it.message, it.location)
+        }
+        clear()
+    }
+}