Refactor JPS daemon client

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/argumentUtils.kt

@@ -17,10 +17,24 @@
 package org.jetbrains.kotlin.cli.common.arguments
 
 import com.intellij.util.xmlb.XmlSerializerUtil
+import com.sampullara.cli.Args
 import java.lang.reflect.Field
 import java.lang.reflect.Modifier
 import java.util.*
 
+fun <A : CommonCompilerArguments> parseArguments(args: Array<String>, arguments: A) {
+    val unparsedArgs = Args.parse(arguments, args, false).partition { it.startsWith("-X") }
+
+    arguments.unknownExtraFlags = unparsedArgs.first
+    arguments.freeArgs = unparsedArgs.second
+
+    for (argument in arguments.freeArgs) {
+        if (argument.startsWith("-")) {
+            throw IllegalArgumentException("Invalid argument: " + argument)
+        }
+    }
+}
+
 fun <T : Any> copyBean(bean: T) = copyFields(bean, bean.javaClass.newInstance(), true)
 
 fun <From : Any, To : From> mergeBeans(from: From, to: To): To {

compiler/cli/src/org/jetbrains/kotlin/cli/common/CLICompiler.java

@@ -30,6 +30,7 @@ import kotlin.jvm.functions.Function1;
 import org.fusesource.jansi.AnsiConsole;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.jetbrains.kotlin.cli.common.arguments.ArgumentUtilsKt;
 import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments;
 import org.jetbrains.kotlin.cli.common.messages.*;
 import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler;
@@ -103,22 +104,7 @@ public abstract class CLICompiler<A extends CommonCompilerArguments> {
 
     @SuppressWarnings("WeakerAccess") // Used in maven (see KotlinCompileMojoBase.java)
     public void parseArguments(@NotNull String[] args, @NotNull A arguments) {
-        Pair<List<String>, List<String>> unparsedArgs =
+        ArgumentUtilsKt.parseArguments(args, arguments);
-                CollectionsKt.partition(Args.parse(arguments, args, false), new Function1<String, Boolean>() {
-                    @Override
-                    public Boolean invoke(String s) {
-                        return s.startsWith("-X");
-                    }
-                });
-
-        arguments.unknownExtraFlags = unparsedArgs.getFirst();
-        arguments.freeArgs = unparsedArgs.getSecond();
-
-        for (String argument : arguments.freeArgs) {
-            if (argument.startsWith("-")) {
-                throw new IllegalArgumentException("Invalid argument: " + argument);
-            }
-        }
     }
 
     @NotNull

compiler/compiler-runner/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.kt

@@ -110,15 +110,10 @@ abstract class KotlinCompilerRunner<in Env : CompilerEnvironment> {
 
     protected fun runCompiler(
             compilerClassName: String,
-            arguments: CommonCompilerArguments,
+            compilerArgs: CommonCompilerArguments,
-            additionalArguments: String,
             environment: Env): ExitCode {
         return try {
-            val argumentsList = ArgumentUtils.convertArgumentsToStringList(arguments)
+            compileWithDaemonOrFallback(compilerClassName, compilerArgs, environment)
-            argumentsList.addAll(additionalArguments.split(" "))
-
-            val argsArray = argumentsList.toTypedArray()
-            doRunCompiler(compilerClassName, argsArray, environment)
         }
         catch (e: Throwable) {
             MessageCollectorUtil.reportException(environment.messageCollector, e)
@@ -126,43 +121,20 @@ abstract class KotlinCompilerRunner<in Env : CompilerEnvironment> {
         }
     }
 
-    protected abstract fun doRunCompiler(
+    protected abstract fun compileWithDaemonOrFallback(
             compilerClassName: String,
-            argsArray: Array<String>,
+            compilerArgs: CommonCompilerArguments,
             environment: Env
     ): ExitCode
 
     /**
      * Returns null if could not connect to daemon
      */
-    protected open fun compileWithDaemon(
+    protected abstract fun compileWithDaemon(
             compilerClassName: String,
-            argsArray: Array<String>,
+            compilerArgs: CommonCompilerArguments,
             environment: Env
-    ): ExitCode? {
+    ): ExitCode?
-        val compilerOut = ByteArrayOutputStream()
-        val daemonOut = ByteArrayOutputStream()
-        val services = CompilationServices(
-                incrementalCompilationComponents = environment.services.get(IncrementalCompilationComponents::class.java),
-                compilationCanceledStatus = environment.services.get(CompilationCanceledStatus::class.java))
-        val targetPlatform = when (compilerClassName) {
-            K2JVM_COMPILER -> CompileService.TargetPlatform.JVM
-            K2JS_COMPILER -> CompileService.TargetPlatform.JS
-            K2METADATA_COMPILER -> CompileService.TargetPlatform.METADATA
-            else -> throw IllegalArgumentException("Unknown compiler type $compilerClassName")
-        }
-
-        val res: Int = withDaemon(environment, retryOnConnectionError = true) { daemon, sessionId ->
-            KotlinCompilerClient.incrementalCompile(daemon, sessionId, targetPlatform, argsArray, services, compilerOut, daemonOut)
-        } ?: return null
-
-        val exitCode = exitCodeFromProcessExitCode(res)
-        processCompilerOutput(environment, compilerOut, exitCode)
-        BufferedReader(StringReader(daemonOut.toString())).forEachLine {
-            environment.messageCollector.report(CompilerMessageSeverity.INFO, it, CompilerMessageLocation.NO_LOCATION)
-        }
-        return exitCode
-    }
 
     protected fun <T> withDaemon(environment: Env, retryOnConnectionError: Boolean, fn: (CompileService, sessionId: Int)->T): T? {
         fun retryOrFalse(e: Exception): T? {

compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/CompilerCallbackServicesFacadeServer.kt

@@ -30,7 +30,7 @@ import org.jetbrains.kotlin.progress.CompilationCanceledStatus
 import java.rmi.server.UnicastRemoteObject
 
 
-class CompilerCallbackServicesFacadeServer(
+open class CompilerCallbackServicesFacadeServer(
         val incrementalCompilationComponents: IncrementalCompilationComponents? = null,
         val compilationCanceledStatus: CompilationCanceledStatus? = null,
         port: Int = SOCKET_ANY_FREE_PORT

compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileService.kt

@@ -38,7 +38,8 @@ interface CompileService : Remote {
 
     enum class CompilerMode : Serializable {
         NON_INCREMENTAL_COMPILER,
-        INCREMENTAL_COMPILER
+        INCREMENTAL_COMPILER,
+        JPS_COMPILER
     }
 
     companion object {

compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerServicesFacadeBase.kt

@@ -49,4 +49,6 @@ interface IncrementalCompilerServicesFacade : CompilerServicesFacadeBase {
 
     @Throws(RemoteException::class)
     fun getChanges(artifact: File, sinceTS: Long): Iterable<SimpleDirtyData>?
-}
+}
+
+interface JpsCompilerServicesFacade : CompilerServicesFacadeBase, CompilerCallbackServicesFacade

compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt

@@ -334,6 +334,14 @@ class CompileServiceImpl(
         val serviceReporter = CompileServiceReporterImpl(servicesFacade, additionalCompilerArguments)
 
         return when (compilerMode) {
+            CompileService.CompilerMode.JPS_COMPILER -> {
+                val jpsServicesFacade = servicesFacade as JpsCompilerServicesFacade
+
+                doCompile(sessionId, serviceReporter, operationsTracer) { eventManger, profiler ->
+                    val services = createCompileServices(jpsServicesFacade, eventManger, profiler)
+                    execCompiler(targetPlatform, services, compilerArguments, messageCollector)
+                }
+            }
             CompileService.CompilerMode.NON_INCREMENTAL_COMPILER -> {
                 doCompile(sessionId, serviceReporter, operationsTracer) { eventManger, profiler ->
                     execCompiler(targetPlatform, Services.EMPTY, compilerArguments, messageCollector)

jps-plugin/src/org/jetbrains/kotlin/compilerRunner/JpsCompilerServicesFacadeImpl.kt

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.compilerRunner
+
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.daemon.client.CompilerCallbackServicesFacadeServer
+import org.jetbrains.kotlin.daemon.common.JpsCompilerServicesFacade
+import org.jetbrains.kotlin.daemon.common.ReportCategory
+import org.jetbrains.kotlin.daemon.common.SOCKET_ANY_FREE_PORT
+import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
+import org.jetbrains.kotlin.progress.CompilationCanceledStatus
+import java.io.Serializable
+
+internal class JpsCompilerServicesFacadeImpl(
+        private val env: JpsCompilerEnvironment,
+        port: Int = SOCKET_ANY_FREE_PORT
+) : CompilerCallbackServicesFacadeServer(env.services.get(IncrementalCompilationComponents::class.java),
+                                         env.services.get(CompilationCanceledStatus::class.java),
+                                         port),
+        JpsCompilerServicesFacade {
+
+    override fun report(category: ReportCategory, severity: Int, message: String?, attachment: Serializable?) {
+        when (category) {
+            ReportCategory.COMPILER_MESSAGE -> {
+                val compilerMessageSeverity = CompilerMessageSeverity.values().firstOrNull { it.value == severity }
+                if (compilerMessageSeverity != null) {
+                    val location = attachment as? CompilerMessageLocation ?: CompilerMessageLocation.NO_LOCATION
+                    env.messageCollector.report(compilerMessageSeverity, message!!, location)
+                }
+                else {
+                    reportUnexpected(category, severity, message, attachment)
+                }
+            }
+            ReportCategory.DAEMON_MESSAGE,
+            ReportCategory.INCREMENTAL_COMPILATION -> {
+                if (message != null) {
+                    env.messageCollector.report(CompilerMessageSeverity.LOGGING, message, CompilerMessageLocation.NO_LOCATION)
+                }
+                else {
+                    reportUnexpected(category, severity, message, attachment)
+                }
+            }
+            else -> {
+                reportUnexpected(category, severity, message, attachment)
+            }
+        }
+    }
+
+    private fun reportUnexpected(category: ReportCategory, severity: Int, message: String?, attachment: Serializable?) {
+        env.messageCollector.report(CompilerMessageSeverity.LOGGING,
+                                    "Unexpected message: category=$category; severity=$severity; message='$message'; attachment=$attachment",
+                                    CompilerMessageLocation.NO_LOCATION)
+    }
+}

jps-plugin/src/org/jetbrains/kotlin/compilerRunner/JpsKotlinCompilerRunner.kt

@@ -19,23 +19,19 @@ package org.jetbrains.kotlin.compilerRunner
 import org.jetbrains.jps.api.GlobalOptions
 import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.cli.common.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY
-import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
+import org.jetbrains.kotlin.cli.common.arguments.*
-import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
-import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
-import org.jetbrains.kotlin.cli.common.arguments.mergeBeans
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
-import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.config.CompilerSettings
-import org.jetbrains.kotlin.daemon.common.CompilerId
+import org.jetbrains.kotlin.daemon.common.*
-import org.jetbrains.kotlin.daemon.common.isDaemonEnabled
 import org.jetbrains.kotlin.jps.build.KotlinBuilder
 import java.io.ByteArrayOutputStream
 import java.io.File
 import java.io.PrintStream
+import java.util.*
 
 class JpsKotlinCompilerRunner : KotlinCompilerRunner<JpsCompilerEnvironment>() {
-    override val log: KotlinLogger = JpsKotlinLogger(KotlinBuilder.LOG)
+        override val log: KotlinLogger = JpsKotlinLogger(KotlinBuilder.LOG)
 
     companion object {
         private @Volatile var jpsDaemonConnection: DaemonConnection? = null
@@ -50,8 +46,10 @@ class JpsKotlinCompilerRunner : KotlinCompilerRunner<JpsCompilerEnvironment>() {
     ) {
         val arguments = mergeBeans(commonArguments, k2jvmArguments)
         setupK2JvmArguments(moduleFile, arguments)
+        val additionalArguments = compilerSettings.additionalArguments.split(" ").toTypedArray()
+        parseArguments(additionalArguments, arguments)
 
-        runCompiler(K2JVM_COMPILER, arguments, compilerSettings.additionalArguments, environment)
+        runCompiler(K2JVM_COMPILER, arguments, environment)
     }
 
     fun runK2JsCompiler(
@@ -65,15 +63,22 @@ class JpsKotlinCompilerRunner : KotlinCompilerRunner<JpsCompilerEnvironment>() {
     ) {
         val arguments = mergeBeans(commonArguments, k2jsArguments)
         setupK2JsArguments(outputFile, sourceFiles, libraryFiles, arguments)
+        val additionalArguments = compilerSettings.additionalArguments.split(" ").toTypedArray()
+        parseArguments(additionalArguments, arguments)
 
-        runCompiler(K2JS_COMPILER, arguments, compilerSettings.additionalArguments, environment)
+        runCompiler(K2JS_COMPILER, arguments, environment)
     }
 
-    override fun doRunCompiler(compilerClassName: String, argsArray: Array<String>, environment: JpsCompilerEnvironment): ExitCode {
+    override fun compileWithDaemonOrFallback(
+            compilerClassName: String,
+            compilerArgs: CommonCompilerArguments,
+            environment: JpsCompilerEnvironment
+    ): ExitCode {
         environment.messageCollector.report(CompilerMessageSeverity.INFO, "Using kotlin-home = " + environment.kotlinPaths.homePath, CompilerMessageLocation.NO_LOCATION)
+        val argsArray = ArgumentUtils.convertArgumentsToStringList(compilerArgs).toTypedArray()
 
         return if (isDaemonEnabled()) {
-            val daemonExitCode = compileWithDaemon(compilerClassName, argsArray, environment)
+            val daemonExitCode = compileWithDaemon(compilerClassName, compilerArgs, environment)
             daemonExitCode ?: fallbackCompileStrategy(argsArray, compilerClassName, environment)
         }
         else {
@@ -81,6 +86,57 @@ class JpsKotlinCompilerRunner : KotlinCompilerRunner<JpsCompilerEnvironment>() {
         }
     }
 
+    override fun compileWithDaemon(
+            compilerClassName: String,
+            compilerArgs: CommonCompilerArguments,
+            environment: JpsCompilerEnvironment
+    ): ExitCode? {
+        val targetPlatform = when (compilerClassName) {
+            K2JVM_COMPILER -> CompileService.TargetPlatform.JVM
+            K2JS_COMPILER -> CompileService.TargetPlatform.JS
+            K2METADATA_COMPILER -> CompileService.TargetPlatform.METADATA
+            else -> throw IllegalArgumentException("Unknown compiler type $compilerClassName")
+        }
+
+        val res = withDaemon(environment, retryOnConnectionError = true) { daemon, sessionId ->
+            daemon.compile(
+                    sessionId,
+                    CompileService.CompilerMode.JPS_COMPILER,
+                    targetPlatform,
+                    compilerArgs,
+                    AdditionalCompilerArguments(reportingFilters = getReportingFilters(compilerArgs.verbose)),
+                    JpsCompilerServicesFacadeImpl(environment),
+                    operationsTracer = null)
+        }
+
+        return res?.get()?.let { exitCodeFromProcessExitCode(it) }
+    }
+
+    private fun getReportingFilters(verbose: Boolean): List<ReportingFilter> {
+        val result = ArrayList<ReportingFilter>()
+
+        val compilerMessagesSeverities = ArrayList<Int>().apply {
+            add(CompilerMessageSeverity.ERROR.value)
+            add(CompilerMessageSeverity.EXCEPTION.value)
+            add(CompilerMessageSeverity.WARNING.value)
+            add(CompilerMessageSeverity.INFO.value)
+            add(CompilerMessageSeverity.OUTPUT.value)
+
+            if (verbose) {
+                add(CompilerMessageSeverity.LOGGING.value)
+            }
+        }
+
+        if (verbose) {
+            result.add(ReportingFilter(ReportCategory.DAEMON_MESSAGE, emptyList()))
+        }
+
+        val compilerMessagesFilter = ReportingFilter(ReportCategory.COMPILER_MESSAGE, compilerMessagesSeverities)
+        result.add(compilerMessagesFilter)
+
+        return result
+    }
+
     private fun fallbackCompileStrategy(
             argsArray: Array<String>,
             compilerClassName: String,

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt

@@ -20,6 +20,7 @@ import org.jetbrains.kotlin.net.rubygrapefruit.platform.Native
 import org.jetbrains.kotlin.net.rubygrapefruit.platform.ProcessLauncher
 import org.gradle.api.Project
 import org.jetbrains.kotlin.cli.common.ExitCode
+import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
 import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
 import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
 import org.jetbrains.kotlin.cli.common.arguments.K2MetadataCompilerArguments
@@ -76,10 +77,8 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
                 friendDirs = args.friendPaths?.map(::File) ?: emptyList())
         args.module = moduleFile.absolutePath
 
-        val additionalArguments = ""
-
         try {
-            return runCompiler(K2JVM_COMPILER, args, additionalArguments, environment)
+            return runCompiler(K2JVM_COMPILER, args, environment)
         }
         finally {
             moduleFile.delete()
@@ -91,8 +90,8 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
             args: K2JSCompilerArguments,
             environment: GradleCompilerEnvironment
     ): ExitCode {
-        val additionalArguments = kotlinSources.joinToString(separator = " ") { it.absolutePath }
+        args.freeArgs.addAll(kotlinSources.map { it.absolutePath })
-        return runCompiler(K2JS_COMPILER, args, additionalArguments, environment)
+        return runCompiler(K2JS_COMPILER, args, environment)
     }
 
     fun runMetadataCompiler(
@@ -100,11 +99,16 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
             args: K2MetadataCompilerArguments,
             environment: GradleCompilerEnvironment
     ): ExitCode {
-        val additionalArguments = kotlinSources.joinToString(separator = " ") { it.absolutePath }
+        args.freeArgs.addAll(kotlinSources.map { it.absolutePath })
-        return runCompiler(K2METADATA_COMPILER, args, additionalArguments, environment)
+        return runCompiler(K2METADATA_COMPILER, args, environment)
     }
 
-    override fun doRunCompiler(compilerClassName: String, argsArray: Array<String>, environment: GradleCompilerEnvironment): ExitCode {
+    override fun compileWithDaemonOrFallback(
+            compilerClassName: String,
+            compilerArgs: CommonCompilerArguments,
+            environment: GradleCompilerEnvironment
+    ): ExitCode {
+        val argsArray = ArgumentUtils.convertArgumentsToStringList(compilerArgs).toTypedArray()
         with (project.logger) {
             kotlinDebug { "Kotlin compiler class: $compilerClassName" }
             kotlinDebug { "Kotlin compiler classpath: ${environment.compilerClasspath.map { it.canonicalPath }.joinToString()}" }
@@ -113,7 +117,7 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
 
         val executionStrategy = System.getProperty(KOTLIN_COMPILER_EXECUTION_STRATEGY_PROPERTY) ?: DAEMON_EXECUTION_STRATEGY
         if (executionStrategy == DAEMON_EXECUTION_STRATEGY) {
-            val daemonExitCode = compileWithDaemon(compilerClassName, argsArray, environment)
+            val daemonExitCode = compileWithDaemon(compilerClassName, compilerArgs, environment)
 
             if (daemonExitCode != null) {
                 return daemonExitCode
@@ -132,7 +136,7 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
         }
     }
 
-    override fun compileWithDaemon(compilerClassName: String, argsArray: Array<String>, environment: GradleCompilerEnvironment): ExitCode? {
+    override fun compileWithDaemon(compilerClassName: String, compilerArgs: CommonCompilerArguments, environment: GradleCompilerEnvironment): ExitCode? {
         val exitCode = if (environment is GradleIncrementalCompilerEnvironment) {
             incrementalCompilationWithDaemon(environment)
         }