From c4719175f9bce0b235645d6c311219f60ed69ef3 Mon Sep 17 00:00:00 2001 From: Ilya Chernikov Date: Sun, 6 Sep 2015 13:53:38 +0200 Subject: [PATCH] Refactoring startup and shutdown, refactoring service implementation, implementing error and info reporting to compiler output, idle autoshutdown mechanisms, fixing TargetId serializability, some other refactoring Fixing stream to log handler (by removing non-working optimization), fixing idle time calculation, reporting refactorings --- .../rmi/kotlinr/KotlinCompilerClient.kt | 113 ++++++++----- .../jetbrains/kotlin/rmi/CompileService.kt | 13 +- .../org/jetbrains/kotlin/rmi/DaemonParams.kt | 72 ++++++-- .../kotlin/rmi/service/CompileDaemon.kt | 143 +++++++++------- .../kotlin/rmi/service/CompileServiceImpl.kt | 156 ++++++++++++------ .../kotlin/daemon/CompilerDaemonTest.kt | 4 +- .../org/jetbrains/kotlin/modules/TargetId.kt | 4 +- .../kotlin/idea/PluginStartupComponent.java | 19 +++ .../KotlinBuildProcessParametersProvider.kt | 20 ++- .../compilerRunner/KotlinCompilerRunner.java | 131 +++++++++------ 10 files changed, 448 insertions(+), 227 deletions(-) diff --git a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt index b72faff5b43..8ab5998f3f9 100644 --- a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt +++ b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt @@ -35,6 +35,15 @@ fun Process.isAlive() = true } + +public enum class DaemonReportCategory { + DEBUG, INFO, EXCEPTION; +} + +public data class DaemonReportMessage(public val category: DaemonReportCategory, public val message: String) + +public class DaemonReportingTargets(public val out: PrintStream? = null, public val messages: MutableCollection? = null) + public class KotlinCompilerClient { companion object { @@ -42,17 +51,25 @@ public class KotlinCompilerClient { val DAEMON_DEFAULT_STARTUP_TIMEOUT_MS = 10000L val DAEMON_CONNECT_CYCLE_ATTEMPTS = 3 - private fun tryFindDaemon(registryDir: File, compilerId: CompilerId, errStream: PrintStream): CompileService? { + val verboseReporting = System.getProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) != null + + fun DaemonReportingTargets.report(category: DaemonReportCategory, message: String, source: String = "daemon client") { + if (category == DaemonReportCategory.DEBUG && !verboseReporting) return + out?.println("[$source] ${category.name()}: $message") + messages?.add(DaemonReportMessage(category, "[$source] $message")) + } + + private fun tryFindDaemon(registryDir: File, compilerId: CompilerId, reportingTargets: DaemonReportingTargets): CompileService? { val classPathDigest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest() - var port = 0 val daemons = registryDir.walk() .map { Pair(it, makeRunFilenameRegex(digest = classPathDigest, port = "(\\d+)").match(it.name)?.groups?.get(1)?.value?.toInt() ?: 0) } .filter { it.second != 0 } .map { - val daemon = tryConnectToDaemon(it.second, errStream) + reportingTargets.report(DaemonReportCategory.DEBUG, "found suitable daemon on port ${it.second}, trying to connect") + val daemon = tryConnectToDaemon(it.second, reportingTargets) // cleaning orphaned file; note: daemon should shut itself down if it detects that the run file is deleted if (daemon == null && !it.first.delete()) { - errStream.println("WARNING: Unable to delete seemingly orphaned file '', cleanup recommended") + reportingTargets.report(DaemonReportCategory.INFO, "WARNING: unable to delete seemingly orphaned file '${it.first.absolutePath}', cleanup recommended") } daemon } @@ -66,24 +83,24 @@ public class KotlinCompilerClient { } } - private fun tryConnectToDaemon(port: Int, errStream: PrintStream): CompileService? { + private fun tryConnectToDaemon(port: Int, reportingTargets: DaemonReportingTargets): CompileService? { try { val daemon = LocateRegistry.getRegistry("localhost", port) ?.lookup(COMPILER_SERVICE_RMI_NAME) if (daemon != null) return daemon as? CompileService ?: throw ClassCastException("Unable to cast compiler service, actual class received: ${daemon.javaClass}") - errStream.println("[daemon client] daemon not found") + reportingTargets.report(DaemonReportCategory.EXCEPTION, "daemon not found") } catch (e: ConnectException) { - errStream.println("[daemon client] cannot connect to registry: " + (e.getCause()?.getMessage() ?: e.getMessage() ?: "unknown exception")) + reportingTargets.report(DaemonReportCategory.EXCEPTION, "cannot connect to registry: " + (e.getCause()?.getMessage() ?: e.getMessage() ?: "unknown exception")) // ignoring it - processing below } return null } - private fun startDaemon(compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, daemonOptions: DaemonOptions, errStream: PrintStream) { + private fun startDaemon(compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, daemonOptions: DaemonOptions, reportingTargets: DaemonReportingTargets) { val javaExecutable = File(System.getProperty("java.home"), "bin").let { val javaw = File(it, "javaw.exe") if (javaw.exists()) javaw @@ -96,7 +113,7 @@ public class KotlinCompilerClient { COMPILER_DAEMON_CLASS_FQN + daemonOptions.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } + compilerId.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } - errStream.println("[daemon client] starting the daemon as: " + args.joinToString(" ")) + reportingTargets.report(DaemonReportCategory.DEBUG, "starting the daemon as: " + args.joinToString(" ")) val processBuilder = ProcessBuilder(args).redirectErrorStream(true) // assuming daemon process is deaf and (mostly) silent, so do not handle streams val daemon = processBuilder.start() @@ -113,22 +130,21 @@ public class KotlinCompilerClient { isEchoRead.release() return@forEachLine } - errStream.println("[daemon] " + it) + reportingTargets.report(DaemonReportCategory.DEBUG, it, "daemon") } } try { // trying to wait for process + val daemonStartupTimeout = System.getProperty(COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY)?.let { + try { + it.toLong() + } + catch (e: Exception) { + reportingTargets.report(DaemonReportCategory.INFO, "unable to interpret $COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY property ('$it'); using default timeout $DAEMON_DEFAULT_STARTUP_TIMEOUT_MS ms") + null + } + } ?: DAEMON_DEFAULT_STARTUP_TIMEOUT_MS if (daemonOptions.runFilesPath.isNotEmpty()) { - val daemonStartupTimeout = System.getProperty(COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY)?.let { - try { - it.toLong() - } - catch (e: Exception) { - errStream.println("[daemon client] unable to interpret $COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY property ('$it'); using default timeout $DAEMON_DEFAULT_STARTUP_TIMEOUT_MS ms") - null - } - } ?: DAEMON_DEFAULT_STARTUP_TIMEOUT_MS - errStream.println("[daemon client] waiting for daemon to respond") val succeeded = isEchoRead.tryAcquire(daemonStartupTimeout, TimeUnit.MILLISECONDS) if (!daemon.isAlive()) throw Exception("Daemon terminated unexpectedly") @@ -137,7 +153,7 @@ public class KotlinCompilerClient { } else // without startEcho defined waiting for max timeout - Thread.sleep(DAEMON_DEFAULT_STARTUP_TIMEOUT_MS) + Thread.sleep(daemonStartupTimeout) } finally { // assuming that all important output is already done, the rest should be routed to the log by the daemon itself @@ -147,10 +163,8 @@ public class KotlinCompilerClient { } } - public fun checkCompilerId(compiler: CompileService, localId: CompilerId, errStream: PrintStream): Boolean { + public fun checkCompilerId(compiler: CompileService, localId: CompilerId): Boolean { val remoteId = compiler.getCompilerId() - errStream.println("[daemon client] remoteId = " + remoteId.toString()) - errStream.println("[daemon client] localId = " + localId.toString()) return (localId.compilerVersion.isEmpty() || localId.compilerVersion == remoteId.compilerVersion) && (localId.compilerClasspath.all { remoteId.compilerClasspath.contains(it) }) && (localId.compilerDigest.isEmpty() || remoteId.compilerDigest.isEmpty() || localId.compilerDigest == remoteId.compilerDigest) @@ -202,28 +216,36 @@ public class KotlinCompilerClient { } - public fun connectToCompileService(compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, daemonOptions: DaemonOptions, errStream: PrintStream, autostart: Boolean = true, checkId: Boolean = true): CompileService? { + + public fun connectToCompileService(compilerId: CompilerId, + daemonJVMOptions: DaemonJVMOptions, + daemonOptions: DaemonOptions, + reportingTargets: DaemonReportingTargets, + autostart: Boolean = true, + checkId: Boolean = true + ): CompileService? { + var attempts = 0 var fileLock: FileBasedLock? = null try { while (attempts++ < DAEMON_CONNECT_CYCLE_ATTEMPTS) { - val service = tryFindDaemon(File(daemonOptions.runFilesPath), compilerId, errStream) + val service = tryFindDaemon(File(daemonOptions.runFilesPath), compilerId, reportingTargets) if (service != null) { - if (!checkId || checkCompilerId(service, compilerId, errStream)) { - errStream.println("[daemon client] found the suitable daemon") + if (!checkId || checkCompilerId(service, compilerId)) { + reportingTargets.report(DaemonReportCategory.DEBUG, "connected to the daemon") return service } - errStream.println("[daemon client] compiler identity don't match: " + compilerId.mappers.flatMap { it.toArgs("") }.joinToString(" ")) + reportingTargets.report(DaemonReportCategory.DEBUG, "compiler identity don't match: " + compilerId.mappers.flatMap { it.toArgs("") }.joinToString(" ")) if (!autostart) return null - errStream.println("[daemon client] shutdown the daemon") + reportingTargets.report(DaemonReportCategory.DEBUG, "shutdown the daemon") service.shutdown() // TODO: find more reliable way Thread.sleep(1000) - errStream.println("[daemon client] daemon shut down correctly, restarting") + reportingTargets.report(DaemonReportCategory.DEBUG, "daemon shut down correctly, restarting search") } else { if (!autostart) return null - errStream.println("[daemon client] cannot connect to Compile Daemon, trying to start") + reportingTargets.report(DaemonReportCategory.DEBUG, "no suitable daemon found, starting a new one") } if (fileLock == null || !fileLock.isLocked()) { @@ -234,11 +256,14 @@ public class KotlinCompilerClient { attempts-- } else { - startDaemon(compilerId, daemonJVMOptions, daemonOptions, errStream) - errStream.println("[daemon client] daemon started, trying to connect") + startDaemon(compilerId, daemonJVMOptions, daemonOptions, reportingTargets) + reportingTargets.report(DaemonReportCategory.DEBUG, "daemon started, trying to resolve") } } } + catch (e: Exception) { + reportingTargets.report(DaemonReportCategory.EXCEPTION, e.toString()) + } finally { fileLock?.release() } @@ -246,7 +271,7 @@ public class KotlinCompilerClient { } public fun shutdownCompileService(daemonOptions: DaemonOptions): Unit { - KotlinCompilerClient.connectToCompileService(CompilerId(), DaemonJVMOptions(), daemonOptions, System.out, autostart = false, checkId = false) + KotlinCompilerClient.connectToCompileService(CompilerId(), DaemonJVMOptions(), daemonOptions, DaemonReportingTargets(out = System.out), autostart = false, checkId = false) ?.shutdown() } @@ -258,24 +283,26 @@ public class KotlinCompilerClient { val outStrm = RemoteOutputStreamServer(out) try { - return compiler.remoteCompile(args, outStrm, CompileService.OutputFormat.PLAIN) + return compiler.remoteCompile(args, outStrm, CompileService.OutputFormat.PLAIN, outStrm) } finally { outStrm.disconnect() } } - public fun incrementalCompile(compiler: CompileService, args: Array, caches: Map, out: OutputStream): Int { + public fun incrementalCompile(compiler: CompileService, args: Array, caches: Map, compilerOut: OutputStream, daemonOut: OutputStream): Int { - val outStrm = RemoteOutputStreamServer(out) + val compilerOutStreamServer = RemoteOutputStreamServer(compilerOut) + val daemonOutStreamServer = RemoteOutputStreamServer(daemonOut) val cacheServers = hashMapOf() try { caches.mapValuesTo(cacheServers, { RemoteIncrementalCacheServer( it.getValue()) }) - return compiler.remoteIncrementalCompile(args, cacheServers, outStrm, CompileService.OutputFormat.XML) + return compiler.remoteIncrementalCompile(args, cacheServers, compilerOutStreamServer, CompileService.OutputFormat.XML, daemonOutStreamServer) } finally { cacheServers.forEach { it.getValue().disconnect() } - outStrm.disconnect() + compilerOutStreamServer.disconnect() + daemonOutStreamServer.disconnect() } } @@ -299,7 +326,7 @@ public class KotlinCompilerClient { System.err.println("compiler wasn't explicitly specified, attempt to find appropriate jar") System.getProperty("java.class.path") ?.split(File.pathSeparator) - ?.map { File(it).parent } + ?.map { File(it).parentFile } ?.distinct() ?.map { it?.walk() @@ -317,7 +344,7 @@ public class KotlinCompilerClient { compilerId.updateDigest() } - val daemon = connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, System.out, autostart = !clientOptions.stop, checkId = !clientOptions.stop) + val daemon = connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, DaemonReportingTargets(out = System.out), autostart = !clientOptions.stop, checkId = !clientOptions.stop) if (daemon == null) { if (clientOptions.stop) System.err.println("No daemon found to shut down") @@ -335,7 +362,7 @@ public class KotlinCompilerClient { try { val memBefore = daemon.getUsedMemory() / 1024 val startTime = System.nanoTime() - val res = daemon.remoteCompile(filteredArgs.toArrayList().toTypedArray(), outStrm, CompileService.OutputFormat.PLAIN) + val res = daemon.remoteCompile(filteredArgs.toArrayList().toTypedArray(), outStrm, CompileService.OutputFormat.PLAIN, outStrm) val endTime = System.nanoTime() println("Compilation result code: $res") val memAfter = daemon.getUsedMemory() / 1024 diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt index 676adc3741c..5160d1caa97 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt @@ -57,12 +57,19 @@ public interface CompileService : Remote { public fun shutdown() throws(RemoteException::class) - public fun remoteCompile(args: Array, errStream: RemoteOutputStream, outputFormat: OutputFormat): Int + public fun remoteCompile( + args: Array, + compilerOutputStream: RemoteOutputStream, + outputFormat: OutputFormat, + serviceOutputStream: RemoteOutputStream + ): Int throws(RemoteException::class) public fun remoteIncrementalCompile( args: Array, caches: Map, - outputStream: RemoteOutputStream, - outputFormat: OutputFormat): Int + compilerOutputStream: RemoteOutputStream, + compilerOutputFormat: OutputFormat, + serviceOutputStream: RemoteOutputStream + ): Int } diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt index 05343ffcdc9..84555bb547e 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt @@ -36,23 +36,34 @@ public val COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_CHECK_MS: Long = 100L public val COMPILE_DAEMON_ENABLED_PROPERTY: String = "kotlin.daemon.enabled" public val COMPILE_DAEMON_JVM_OPTIONS_PROPERTY: String = "kotlin.daemon.jvm.options" public val COMPILE_DAEMON_OPTIONS_PROPERTY: String = "kotlin.daemon.options" +public val COMPILE_DAEMON_CLIENT_ALIVE_PATH_PROPERTY: String = "kotlin.daemon.client.alive.path" +public val COMPILE_DAEMON_REPORT_PERF_PROPERTY: String = "kotlin.daemon.perf" +public val COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY: String = "kotlin.daemon.verbose" public val COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX: String = "--daemon-" public val COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY: String ="kotlin.daemon.startup.timeout" +public val COMPILE_DAEMON_DEFAULT_FILES_PREFIX: String = "kotlin-daemon" +public val COMPILE_DAEMON_DATA_DIRECTORY_NAME: String = "." + COMPILE_DAEMON_DEFAULT_FILES_PREFIX public val COMPILE_DAEMON_TIMEOUT_INFINITE_S: Int = 0 +public val COMPILE_DAEMON_DEFAULT_IDLE_TIMEOUT_S: Int = 7200 // 2 hours public val COMPILE_DAEMON_MEMORY_THRESHOLD_INFINITE: Long = 0L +public val COMPILE_DAEMON_DEFAULT_RUN_DIR_PATH: String get() = + // TODO consider special case for windows - local appdata + File(System.getProperty("user.home"), COMPILE_DAEMON_DATA_DIRECTORY_NAME).absolutePath + + val COMPILER_ID_DIGEST = "MD5" -public fun makeRunFilenameString(ts: String, digest: String, port: String, esc: String = ""): String = "kotlin-daemon$esc.$ts$esc.$digest$esc.$port$esc.run" -public fun makeRunFilenameRegex(ts: String = "[0-9-]+", digest: String = "[0-9a-f]+", port: String = "\\d+"): Regex = makeRunFilenameString(ts, digest, port, esc = "\\").toRegex() +public fun makeRunFilenameString(ts: String, digest: String, port: String, esc: String = ""): String = "$COMPILE_DAEMON_DEFAULT_FILES_PREFIX$esc.$ts$esc.$digest$esc.$port$esc.run" +public fun makeRunFilenameRegex(ts: String = "[0-9TZ:\\.\\+-]+", digest: String = "[0-9a-f]+", port: String = "\\d+"): Regex = makeRunFilenameString(ts, digest, port, esc = "\\").toRegex() open class PropMapper>(val dest: C, val prop: P, val names: List = listOf(prop.name), - val fromString: (s: String) -> V, - val toString: ((v: V) -> String?) = { it.toString() }, - val skipIf: ((v: V) -> Boolean) = { false }, + val fromString: (String) -> V, + val toString: ((V) -> String?) = { it.toString() }, + val skipIf: ((V) -> Boolean) = { false }, val mergeDelimiter: String? = null) { open fun toArgs(prefix: String = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX): List = @@ -64,6 +75,17 @@ open class PropMapper>(val dest: C, open fun apply(s: String) = prop.set(dest, fromString(s)) } + +class NullablePropMapper>(dest: C, + prop: P, + names: List = listOf(), + fromString: ((String) -> V), + toString: ((V) -> String?) = { it.toString() }, + skipIf: ((V) -> Boolean) = { it == null }, + mergeDelimiter: String? = null) +: PropMapper(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name), + fromString = fromString, toString = toString, skipIf = skipIf, mergeDelimiter = mergeDelimiter) + class StringPropMapper>(dest: C, prop: P, names: List = listOf(), @@ -74,10 +96,12 @@ class StringPropMapper>(dest: C, : PropMapper(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name), fromString = fromString, toString = toString, skipIf = skipIf, mergeDelimiter = mergeDelimiter) + class BoolPropMapper>(dest: C, prop: P, names: List = listOf()) : PropMapper(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name), fromString = { true }, toString = { null }, skipIf = { !prop.get(dest) }) + class RestPropMapper>>(dest: C, prop: P) : PropMapper, P>(dest = dest, prop = prop, toString = { null }, fromString = { arrayListOf() }) { @@ -87,7 +111,7 @@ class RestPropMapper>>(dest } -inline fun Iterable.firstMapOrNull(mappingPredicate: (T) -> Pair): R? { +inline fun Iterable.findWithTransform(mappingPredicate: (T) -> Pair): R? { for (element in this) { val (found, mapped) = mappingPredicate(element) if (found) return mapped @@ -103,9 +127,10 @@ fun Iterable.filterExtractProps(propMappers: List>, pr while (iter.hasNext()) { val param = iter.next() - val (propMapper, matchingOption) = propMappers.firstMapOrNull { - val name = if (it !is RestPropMapper<*,*>) it.names.firstOrNull { param.startsWith(prefix + it) } else null - Pair(name != null, Pair(it, name)) + val (propMapper, matchingOption) = propMappers.findWithTransform { mapper -> + mapper.names + .firstOrNull { param.startsWith(prefix + it) } + .let { Pair(it != null, Pair(mapper, it)) } } ?: Pair(null, null) when { @@ -174,16 +199,19 @@ public data class DaemonJVMOptions( get() = RestPropMapper(this, ::jvmParams) } + public data class DaemonOptions( - public var runFilesPath: String = File(System.getProperty("java.io.tmpdir"), "kotlin_daemon").absolutePath, + public var runFilesPath: String = COMPILE_DAEMON_DEFAULT_RUN_DIR_PATH, public var autoshutdownMemoryThreshold: Long = COMPILE_DAEMON_MEMORY_THRESHOLD_INFINITE, - public var autoshutdownIdleSeconds: Int = COMPILE_DAEMON_TIMEOUT_INFINITE_S + public var autoshutdownIdleSeconds: Int = COMPILE_DAEMON_DEFAULT_IDLE_TIMEOUT_S, + public var clientAliveFlagPath: String? = null ) : OptionsGroup { override val mappers: List> get() = listOf( PropMapper(this, ::runFilesPath, fromString = { it.trim('"') }), - PropMapper(this, ::autoshutdownMemoryThreshold, fromString = { it.toLong() }, skipIf = { it == 0L }), - PropMapper(this, ::autoshutdownIdleSeconds, fromString = { it.toInt() }, skipIf = { it == 0 })) + PropMapper(this, ::autoshutdownMemoryThreshold, fromString = { it.toLong() }, skipIf = { it == 0L }, mergeDelimiter = "="), + PropMapper(this, ::autoshutdownIdleSeconds, fromString = { it.toInt() }, skipIf = { it == 0 }, mergeDelimiter = "="), + NullablePropMapper(this, ::clientAliveFlagPath, fromString = { it }, toString = { "${it?.trim('\"','\'')}" }, mergeDelimiter = "=")) } @@ -256,7 +284,7 @@ public data class CompilerId( public fun isDaemonEnabled(): Boolean = System.getProperty(COMPILE_DAEMON_ENABLED_PROPERTY) != null -public fun configureDaemonLaunchingOptions(opts: DaemonJVMOptions, inheritMemoryLimits: Boolean): DaemonJVMOptions { +public fun configureDaemonJVMOptions(opts: DaemonJVMOptions, inheritMemoryLimits: Boolean): DaemonJVMOptions { // note: sequence matters, explicit override in COMPILE_DAEMON_JVM_OPTIONS_PROPERTY should be done after inputArguments processing if (inheritMemoryLimits) ManagementFactory.getRuntimeMXBean().inputArguments.filterExtractProps(opts.mappers, "-") @@ -267,13 +295,16 @@ public fun configureDaemonLaunchingOptions(opts: DaemonJVMOptions, inheritMemory .map { it.replace("[\\\\](.)".toRegex(), "$1") } .filterExtractProps(opts.mappers, "-", opts.restMapper)) } + + System.getProperty(COMPILE_DAEMON_REPORT_PERF_PROPERTY)?.let { opts.jvmParams.add("D" + COMPILE_DAEMON_REPORT_PERF_PROPERTY) } + System.getProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY)?.let { opts.jvmParams.add("D" + COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) } return opts } -public fun configureDaemonLaunchingOptions(inheritMemoryLimits: Boolean): DaemonJVMOptions = - configureDaemonLaunchingOptions(DaemonJVMOptions(), inheritMemoryLimits = inheritMemoryLimits) +public fun configureDaemonJVMOptions(inheritMemoryLimits: Boolean): DaemonJVMOptions = + configureDaemonJVMOptions(DaemonJVMOptions(), inheritMemoryLimits = inheritMemoryLimits) -jvmOverloads public fun configureDaemonOptions(opts: DaemonOptions = DaemonOptions()): DaemonOptions { +public fun configureDaemonOptions(opts: DaemonOptions): DaemonOptions { System.getProperty(COMPILE_DAEMON_OPTIONS_PROPERTY)?.let { val unrecognized = it.trim('"', '\'').split(",").filterExtractProps(opts.mappers, "") if (unrecognized.any()) @@ -281,6 +312,13 @@ jvmOverloads public fun configureDaemonOptions(opts: DaemonOptions = DaemonOptio "Unrecognized daemon options passed via property $COMPILE_DAEMON_OPTIONS_PROPERTY: " + unrecognized.joinToString(" ") + "\nSupported options: " + opts.mappers.joinToString(", ", transform = { it.names.first() })) } + System.getProperty(COMPILE_DAEMON_CLIENT_ALIVE_PATH_PROPERTY)?.let { + val trimmed = it.trim('"','\'') + if (!trimmed.isBlank()) { + opts.clientAliveFlagPath = trimmed + } + } return opts } +public fun configureDaemonOptions(): DaemonOptions = configureDaemonOptions(DaemonOptions()) diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt index db4c6784402..15ccf7a8491 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt @@ -19,6 +19,7 @@ package org.jetbrains.kotlin.rmi.service import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler import org.jetbrains.kotlin.rmi.* import org.jetbrains.kotlin.service.CompileServiceImpl +import org.jetbrains.kotlin.service.nowSeconds import java.io.File import java.io.IOException import java.io.OutputStream @@ -33,6 +34,10 @@ import java.util.* import java.util.jar.Manifest import java.util.logging.LogManager import java.util.logging.Logger +import kotlin.concurrent.timer + +val DAEMON_PERIODIC_CHECK_INTERVAL_MS = 1000L + class LogStream(name: String) : OutputStream() { @@ -45,25 +50,6 @@ class LogStream(name: String) : OutputStream() { else lineBuf.append(byte.toChar()) } - override fun write(data: ByteArray, offset: Int, length: Int) { - var ofs = offset - var lineStart = ofs - while (ofs < length) { - if (data[ofs].toChar() == '\n') { - flush(data, lineStart, ofs - lineStart) - lineStart = ofs + 1 - } - ofs++ - } - if (lineStart < length) - lineBuf.append(data, lineStart, length - lineStart) - } - - fun flush(data: ByteArray, offset: Int, length: Int) { - log.info(lineBuf.toString() + data.toString().substring(offset, length)) - lineBuf.setLength(0) - } - override fun flush() { log.info(lineBuf.toString()) lineBuf.setLength(0) @@ -86,7 +72,7 @@ public class CompileDaemon { "java.util.logging.FileHandler.limit = 1073741824\n" + // 1Mb "java.util.logging.FileHandler.count = 3\n" + "java.util.logging.FileHandler.append = false\n" + - "java.util.logging.FileHandler.pattern = $logPath/kotlin-daemon.$logTime.%g.log\n" + + "java.util.logging.FileHandler.pattern = $logPath/$COMPILE_DAEMON_DEFAULT_FILES_PREFIX.$logTime.%u%g.log\n" + "java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s\\n\n" LogManager.getLogManager().readConfiguration(cfg.byteInputStream()) @@ -114,47 +100,88 @@ public class CompileDaemon { val compilerId = CompilerId() val daemonOptions = DaemonOptions() - val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) - if (filteredArgs.any()) { - val helpLine = "usage: " - log.info(helpLine) - println(helpLine) - throw IllegalArgumentException("Unknown arguments") + try { + val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) + + if (filteredArgs.any()) { + val helpLine = "usage: " + log.info(helpLine) + println(helpLine) + throw IllegalArgumentException("Unknown arguments: " + filteredArgs.joinToString(" ")) + } + + log.info("starting daemon") + + // TODO: find minimal set of permissions and restore security management + // if (System.getSecurityManager() == null) + // System.setSecurityManager (RMISecurityManager()) + // + // setDaemonPpermissions(daemonOptions.port) + + val (registry, port) = createRegistry(COMPILE_DAEMON_FIND_PORT_ATTEMPTS) + val runFileDir = File(if (daemonOptions.runFilesPath.isBlank()) COMPILE_DAEMON_DEFAULT_RUN_DIR_PATH else daemonOptions.runFilesPath) + runFileDir.mkdirs() + val runFile = File(runFileDir, + makeRunFilenameString(ts = "%tFT% Boolean, message: String): Boolean { + val res = check() + if (res) { + log.info(message) + } + return res + } + + // stopping daemon if any shutdown condition is met + timer(initialDelay = DAEMON_PERIODIC_CHECK_INTERVAL_MS, period = DAEMON_PERIODIC_CHECK_INTERVAL_MS) { + val idleSeconds = nowSeconds() - compilerService.lastUsedSeconds + if (shutdownCondition({ !runFile.exists() }, "Run file removed, shutting down") || + shutdownCondition({ daemonOptions.autoshutdownIdleSeconds != COMPILE_DAEMON_TIMEOUT_INFINITE_S && idleSeconds > daemonOptions.autoshutdownIdleSeconds}, + "Idle timeout exceeded ${daemonOptions.autoshutdownIdleSeconds}s, shutting down") || + shutdownCondition({ daemonOptions.clientAliveFlagPath?.let { !File(it).exists() } ?: false }, + "Client alive flag ${daemonOptions.clientAliveFlagPath} removed, shutting down")) + { + cancel() + compilerService.shutdown() + } + } + } + catch (e: Exception) { + System.err.println("Exception: " + e.getMessage()) + log.info("Exception" + e.getMessage()) + log.info(e.toString()) + // TODO consider exiting without throwing + throw e } - log.info("starting daemon") - - // TODO: find minimal set of permissions and restore security management -// if (System.getSecurityManager() == null) -// System.setSecurityManager (RMISecurityManager()) -// -// setDaemonPpermissions(daemonOptions.port) - - val (registry, port) = createRegistry(COMPILE_DAEMON_FIND_PORT_ATTEMPTS) - val runFileDir = File(daemonOptions.runFilesPath) - runFileDir.mkdirs() - val runFile = File(runFileDir, - makeRunFilenameString(ts = "%tFT%>( val registry: Registry, val compiler: Compiler, @@ -43,28 +50,73 @@ class CompileServiceImpl>( val daemonOptions: DaemonOptions ) : CompileService, UnicastRemoteObject() { + // RMI-exposed API + + override fun getCompilerId(): CompilerId = ifAlive { selfCompilerId } + + override fun getUsedMemory(): Long = ifAlive { usedMemory() } + + override fun shutdown() { + ifAliveExclusive { + log.info("Shutdown started") + alive = false + UnicastRemoteObject.unexportObject(this, true) + log.info("Shutdown complete") + } + } + + override fun remoteCompile(args: Array, + compilerOutputStream: RemoteOutputStream, + outputFormat: CompileService.OutputFormat, + serviceOutputStream: RemoteOutputStream + ): Int = + doCompile(args, compilerOutputStream, serviceOutputStream) { printStream -> + when (outputFormat) { + CompileService.OutputFormat.PLAIN -> compiler.exec(printStream, *args) + CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStream, Services.EMPTY, *args) + } + } + + override fun remoteIncrementalCompile(args: Array, + caches: Map, + compilerOutputStream: RemoteOutputStream, + compilerOutputFormat: CompileService.OutputFormat, + serviceOutputStream: RemoteOutputStream + ): Int = + doCompile(args, compilerOutputStream, serviceOutputStream) { printStream -> + when (compilerOutputFormat) { + CompileService.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation") + CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStream, createCompileServices(caches), *args) + } + } + + // internal implementation stuff + + private volatile var _lastUsedSeconds = nowSeconds() + public val lastUsedSeconds: Long get() = if (rwlock.isWriteLocked || rwlock.readLockCount - rwlock.readHoldCount > 0) nowSeconds() else _lastUsedSeconds + val log by lazy { Logger.getLogger("compiler") } private val rwlock = ReentrantReadWriteLock() private var alive = false // TODO: consider matching compilerId coming from outside with actual one -// private val selfCompilerId by lazy { -// CompilerId( -// compilerClasspath = System.getProperty("java.class.path") -// ?.split(File.pathSeparator) -// ?.map { File(it) } -// ?.filter { it.exists() } -// ?.map { it.absolutePath } -// ?: listOf(), -// compilerVersion = loadKotlinVersionFromResource() -// ) -// } + // private val selfCompilerId by lazy { + // CompilerId( + // compilerClasspath = System.getProperty("java.class.path") + // ?.split(File.pathSeparator) + // ?.map { File(it) } + // ?.filter { it.exists() } + // ?.map { it.absolutePath } + // ?: listOf(), + // compilerVersion = loadKotlinVersionFromResource() + // ) + // } init { // assuming logically synchronized try { - // cleanup for the case of incorrect restart + // cleanup for the case of incorrect restart and many other situations UnicastRemoteObject.unexportObject(this, false) } catch (e: java.rmi.NoSuchObjectException) { @@ -72,18 +124,28 @@ class CompileServiceImpl>( } val stub = UnicastRemoteObject.exportObject(this, 0) as CompileService - // TODO: use version-specific name registry.rebind (COMPILER_SERVICE_RMI_NAME, stub); alive = true } - public class IncrementalCompilationComponentsImpl(val targetToCache: Map): IncrementalCompilationComponents { + private class IncrementalCompilationComponentsImpl(val targetToCache: Map): IncrementalCompilationComponents { // perf: cheap object, but still the pattern may be costly if there are too many calls to cache with the same id (which seems not to be the case now) override fun getIncrementalCache(target: TargetId): IncrementalCache = RemoteIncrementalCacheClient(targetToCache[target]!!) // TODO: add appropriate proxy into interaction when lookup tracker is needed override fun getLookupTracker(): LookupTracker = LookupTracker.DO_NOTHING } + private fun doCompile(args: Array, compilerMessagesStreamProxy: RemoteOutputStream, serviceOutputStreamProxy: RemoteOutputStream, body: (PrintStream) -> ExitCode): Int = + ifAlive { + val compilerMessagesStream = PrintStream( RemoteOutputStreamClient(compilerMessagesStreamProxy)) + val serviceOutputStream = PrintStream( RemoteOutputStreamClient(serviceOutputStreamProxy)) + checkedCompile(args, serviceOutputStream) { + val res = body( compilerMessagesStream).code + _lastUsedSeconds = nowSeconds() + res + } + } + private fun createCompileServices(incrementalCaches: Map): Services = Services.Builder() .register(IncrementalCompilationComponents::class.java, IncrementalCompilationComponentsImpl(incrementalCaches)) @@ -100,19 +162,46 @@ class CompileServiceImpl>( return (rt.totalMemory() - rt.freeMemory()) } - fun checkedCompile(args: Array, body: () -> R): R { + // TODO: consider using version as a part of compiler ID or drop this function + private fun loadKotlinVersionFromResource(): String { + (javaClass.classLoader as? URLClassLoader) + ?.findResource("META-INF/MANIFEST.MF") + ?.let { + try { + return Manifest(it.openStream()).mainAttributes.getValue("Implementation-Version") ?: "" + } + catch (e: IOException) {} + } + return "" + } + + fun ThreadMXBean.threadCputime() = if (isCurrentThreadCpuTimeSupported()) currentThreadCpuTime else 0L + fun ThreadMXBean.threadUsertime() = if (isCurrentThreadCpuTimeSupported()) currentThreadUserTime else 0L + + fun checkedCompile(args: Array, serviceOut: PrintStream, body: () -> R): R { try { if (args.none()) throw IllegalArgumentException("Error: empty arguments list.") log.info("Starting compilation with args: " + args.joinToString(" ")) + val threadMXBean: ThreadMXBean = ManagementFactory.getThreadMXBean() val startMem = usedMemory() / 1024 val startTime = System.nanoTime() + val startThreadTime = threadMXBean.threadCputime() + val startThreadUserTime = threadMXBean.threadUsertime() val res = body() val endTime = System.nanoTime() + val endThreadTime = threadMXBean.threadCputime() + val endThreadUserTime = threadMXBean.threadUsertime() val endMem = usedMemory() / 1024 log.info("Done with result " + res.toString()) - log.info("Elapsed time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") + val elapsed = TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + val elapsedThread = TimeUnit.NANOSECONDS.toMillis(endThreadTime - startThreadTime) + val elapsedThreadUser = TimeUnit.NANOSECONDS.toMillis(endThreadUserTime - startThreadUserTime) + log.info("Elapsed time: $elapsed ms (thread user: $elapsedThreadUser ms sys: ${elapsedThread - elapsedThreadUser} ms)") log.info("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)") + System.getProperty(COMPILE_DAEMON_REPORT_PERF_PROPERTY)?.let { + serviceOut.println("PERF: TOTAL: $elapsed ms (thread user: $elapsedThreadUser ms sys: ${elapsedThread - elapsedThreadUser} ms); memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)") + } return res } // TODO: consider possibilities to handle OutOfMemory @@ -139,39 +228,4 @@ class CompileServiceImpl>( return res } - override fun getCompilerId(): CompilerId = ifAlive { selfCompilerId } - - override fun getUsedMemory(): Long = ifAlive { usedMemory() } - - override fun shutdown() { - ifAliveExclusive { - log.info("Shutdown started") - alive = false - UnicastRemoteObject.unexportObject(this, true) - log.info("Shutdown complete") - } - } - - override fun remoteCompile(args: Array, errStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int = - doCompile(args, errStream) { printStream -> - when (outputFormat) { - CompileService.OutputFormat.PLAIN -> compiler.exec(printStream, *args) - CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStream, Services.EMPTY, *args) - } - } - - override fun remoteIncrementalCompile(args: Array, caches: Map, outputStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int = - doCompile(args, outputStream) { printStream -> - when (outputFormat) { - CompileService.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation") - CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStream, createCompileServices(caches), *args) - } - } - - fun doCompile(args: Array, errStream: RemoteOutputStream, body: (PrintStream) -> ExitCode): Int = - ifAlive { - checkedCompile(args) { - body( PrintStream( RemoteOutputStreamClient(errStream))).code - } - } } diff --git a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt index dadc5ef6450..1868999d056 100644 --- a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt +++ b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt @@ -19,9 +19,11 @@ package org.jetbrains.kotlin.daemon import junit.framework.TestCase import org.jetbrains.kotlin.cli.CliBaseTest import org.jetbrains.kotlin.integration.KotlinIntegrationTestBase +import org.jetbrains.kotlin.rmi.CompileService import org.jetbrains.kotlin.rmi.CompilerId import org.jetbrains.kotlin.rmi.DaemonJVMOptions import org.jetbrains.kotlin.rmi.DaemonOptions +import org.jetbrains.kotlin.rmi.kotlinr.DaemonReportingTargets import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient import org.jetbrains.kotlin.test.JetTestUtils import java.io.ByteArrayOutputStream @@ -40,7 +42,7 @@ public class CompilerDaemonTest : KotlinIntegrationTestBase() { File("dependencies/bootstrap-compiler/Kotlin/kotlinc/lib/kotlin-reflect.jar")) } private fun compileOnDaemon(args: Array): CompilerResults { - val daemon = KotlinCompilerClient.connectToCompileService(compilerId, daemonJVMOptions, daemonOptions, System.err, autostart = true, checkId = true) + val daemon = KotlinCompilerClient.connectToCompileService(compilerId, daemonJVMOptions, daemonOptions, DaemonReportingTargets(out = System.err), autostart = true, checkId = true) TestCase.assertNotNull("failed to connect daemon", daemon) val strm = ByteArrayOutputStream() val code = KotlinCompilerClient.compile(daemon!!, args, strm) diff --git a/compiler/util/src/org/jetbrains/kotlin/modules/TargetId.kt b/compiler/util/src/org/jetbrains/kotlin/modules/TargetId.kt index 1fe759659ee..437fc394fac 100644 --- a/compiler/util/src/org/jetbrains/kotlin/modules/TargetId.kt +++ b/compiler/util/src/org/jetbrains/kotlin/modules/TargetId.kt @@ -16,7 +16,9 @@ package org.jetbrains.kotlin.modules -public data class TargetId(public val name: String, public val type: String) +import java.io.Serializable + +public data class TargetId(public val name: String, public val type: String) : Serializable public fun TargetId(module: Module): TargetId = TargetId(module.getModuleName(), module.getModuleType()) \ No newline at end of file diff --git a/idea/src/org/jetbrains/kotlin/idea/PluginStartupComponent.java b/idea/src/org/jetbrains/kotlin/idea/PluginStartupComponent.java index 66af694dfb9..61376d0e16c 100644 --- a/idea/src/org/jetbrains/kotlin/idea/PluginStartupComponent.java +++ b/idea/src/org/jetbrains/kotlin/idea/PluginStartupComponent.java @@ -26,6 +26,9 @@ import org.jetbrains.kotlin.idea.framework.KotlinJavaScriptLibraryDetectionUtil; import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient; import org.jetbrains.kotlin.utils.PathUtil; +import java.io.File; +import java.io.IOException; + public class PluginStartupComponent implements ApplicationComponent { private static final String KOTLIN_BUNDLED = "KOTLIN_BUNDLED"; @@ -50,6 +53,22 @@ public class PluginStartupComponent implements ApplicationComponent { macros.setMacro(KOTLIN_BUNDLED, PathUtil.getKotlinPathsForIdeaPlugin().getHomePath().getPath()); } + private String aliveFlagPath; + + public String getAliveFlagPath() { + if (aliveFlagPath == null) { + try { + File flagFile = File.createTempFile("kotlin-idea-", "-is-running"); + flagFile.deleteOnExit(); + aliveFlagPath = flagFile.getAbsolutePath(); + } + catch (IOException e) { + aliveFlagPath = ""; + } + } + return this.aliveFlagPath; + } + @Override public void disposeComponent() { KotlinCompilerClient.Companion.shutdownCompileService(); diff --git a/idea/src/org/jetbrains/kotlin/idea/compiler/configuration/KotlinBuildProcessParametersProvider.kt b/idea/src/org/jetbrains/kotlin/idea/compiler/configuration/KotlinBuildProcessParametersProvider.kt index 2c5b6f910fa..bbac030f243 100644 --- a/idea/src/org/jetbrains/kotlin/idea/compiler/configuration/KotlinBuildProcessParametersProvider.kt +++ b/idea/src/org/jetbrains/kotlin/idea/compiler/configuration/KotlinBuildProcessParametersProvider.kt @@ -17,12 +17,22 @@ package org.jetbrains.kotlin.idea.compiler.configuration import com.intellij.compiler.server.BuildProcessParametersProvider +import org.jetbrains.kotlin.idea.PluginStartupComponent -public class KotlinBuildProcessParametersProvider(private val compilerWorkspaceSettings: KotlinCompilerWorkspaceSettings): BuildProcessParametersProvider() { +public class KotlinBuildProcessParametersProvider(private val compilerWorkspaceSettings: KotlinCompilerWorkspaceSettings, + private val kotlinPluginStartupComponent: PluginStartupComponent +): BuildProcessParametersProvider() { override fun getVMArguments(): MutableList { - return if (compilerWorkspaceSettings.incrementalCompilationEnabled) - arrayListOf() - else - arrayListOf("-Dkotlin.incremental.compilation=false") + val res = arrayListOf() + if (compilerWorkspaceSettings.incrementalCompilationEnabled) { + res.add("-Dkotlin.incremental.compilation=false") + } + kotlinPluginStartupComponent.aliveFlagPath.let { + if (!it.isBlank()) { + // TODO: cosider taking the property name from compiler/rmi-interface (check whether dependency will be not too heavy) + res.add("-Dkotlin.daemon.client.alive.path=$it") + } + } + return res } } diff --git a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java index 36f347dfb71..e22c84941b6 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java +++ b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java @@ -27,12 +27,17 @@ import org.jetbrains.kotlin.cli.common.ExitCode; import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments; import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments; import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments; +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation; +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity; import org.jetbrains.kotlin.cli.common.messages.MessageCollector; import org.jetbrains.kotlin.cli.common.messages.MessageCollectorUtil; import org.jetbrains.kotlin.config.CompilerSettings; import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache; import org.jetbrains.kotlin.modules.TargetId; import org.jetbrains.kotlin.rmi.*; +import org.jetbrains.kotlin.rmi.kotlinr.DaemonReportCategory; +import org.jetbrains.kotlin.rmi.kotlinr.DaemonReportMessage; +import org.jetbrains.kotlin.rmi.kotlinr.DaemonReportingTargets; import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient; import org.jetbrains.kotlin.utils.UtilsPackage; @@ -89,6 +94,24 @@ public class KotlinCompilerRunner { incrementalCaches); } + private static void ProcessCompilerOutput( + MessageCollector messageCollector, + OutputItemsCollector collector, + ByteArrayOutputStream stream, + String exitCode + ) { + BufferedReader reader = new BufferedReader(new StringReader(stream.toString())); + CompilerOutputParser.parseCompilerMessagesFromReader(messageCollector, reader, collector); + + if (INTERNAL_ERROR.equals(exitCode)) { + reportInternalCompilerError(messageCollector); + } + } + + private static void reportInternalCompilerError(MessageCollector messageCollector) { + messageCollector.report(ERROR, "Compiler terminated with internal error", NO_LOCATION); + } + private static void runCompiler( String compilerClassName, CommonCompilerArguments arguments, @@ -97,29 +120,6 @@ public class KotlinCompilerRunner { OutputItemsCollector collector, CompilerEnvironment environment, Map incrementalCaches - ) { - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(stream); - - String exitCode = execCompiler(compilerClassName, arguments, additionalArguments, environment, incrementalCaches, out, messageCollector); - - BufferedReader reader = new BufferedReader(new StringReader(stream.toString())); - CompilerOutputParser.parseCompilerMessagesFromReader(messageCollector, reader, collector); - - if (INTERNAL_ERROR.equals(exitCode)) { - messageCollector.report(ERROR, "Compiler terminated with internal error", NO_LOCATION); - } - } - - @NotNull - private static String execCompiler( - String compilerClassName, - CommonCompilerArguments arguments, - String additionalArguments, - CompilerEnvironment environment, - Map incrementalCaches, - PrintStream out, - MessageCollector messageCollector ) { try { messageCollector.report(INFO, "Using kotlin-home = " + environment.getKotlinPaths().getHomePath(), NO_LOCATION); @@ -129,38 +129,73 @@ public class KotlinCompilerRunner { String[] argsArray = ArrayUtil.toStringArray(argumentsList); - // trying the daemon first - if (incrementalCaches != null && RmiPackage.isDaemonEnabled()) { - File libPath = CompilerRunnerUtil.getLibPath(environment.getKotlinPaths(), messageCollector); - // TODO: it may be a good idea to cache the compilerId, since making it means calculating digest over jar(s) and if \\ - // the lifetime of JPS process is small anyway, we can neglect the probability of changed compiler - CompilerId compilerId = CompilerId.makeCompilerId(new File(libPath, "kotlin-compiler.jar")); - DaemonOptions daemonOptions = RmiPackage.configureDaemonOptions(); - DaemonJVMOptions daemonJVMOptions = RmiPackage.configureDaemonLaunchingOptions(true); - // TODO: find proper stream to report daemon connection progress - CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonJVMOptions, daemonOptions, System.out, true, true); - if (daemon != null) { - Integer res = KotlinCompilerClient.Companion.incrementalCompile(daemon, argsArray, incrementalCaches, out); - return res.toString(); - } + if (!tryCompileWithDaemon(messageCollector, collector, environment, incrementalCaches, argsArray)) { + // otherwise fallback to in-process + + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(stream); + + Object rc = CompilerRunnerUtil.invokeExecMethod( compilerClassName, argsArray, environment, messageCollector, out); + + // exec() returns an ExitCode object, class of which is loaded with a different class loader, + // so we take it's contents through reflection + ProcessCompilerOutput(messageCollector, collector, stream, getReturnCodeFromObject(rc)); } - - // otherwise fallback to in-process - - Object rc = CompilerRunnerUtil.invokeExecMethod( - compilerClassName, argsArray, environment, messageCollector, out - ); - - // exec() returns an ExitCode object, class of which is loaded with a different class loader, - // so we take it's contents through reflection - return getReturnCodeFromObject(rc); } catch (Throwable e) { MessageCollectorUtil.reportException(messageCollector, e); - return INTERNAL_ERROR; + reportInternalCompilerError(messageCollector); } } + private static boolean tryCompileWithDaemon( + MessageCollector messageCollector, + OutputItemsCollector collector, + CompilerEnvironment environment, + Map incrementalCaches, + String[] argsArray + ) throws IOException { + if (incrementalCaches != null && RmiPackage.isDaemonEnabled()) { + File libPath = CompilerRunnerUtil.getLibPath(environment.getKotlinPaths(), messageCollector); + // TODO: it may be a good idea to cache the compilerId, since making it means calculating digest over jar(s) and if \\ + // the lifetime of JPS process is small anyway, we can neglect the probability of changed compiler + CompilerId compilerId = CompilerId.makeCompilerId(new File(libPath, "kotlin-compiler.jar")); + DaemonOptions daemonOptions = RmiPackage.configureDaemonOptions(); + DaemonJVMOptions daemonJVMOptions = RmiPackage.configureDaemonJVMOptions(true); + + ArrayList daemonReportMessages = new ArrayList(); + + CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonJVMOptions, daemonOptions, new DaemonReportingTargets(null, daemonReportMessages), true, true); + + for (DaemonReportMessage msg: daemonReportMessages) { + if (msg.getCategory() == DaemonReportCategory.EXCEPTION && daemon == null) { + messageCollector.report(CompilerMessageSeverity.INFO, + "Falling back to compilation without daemon due to error: " + msg.getMessage(), + CompilerMessageLocation.NO_LOCATION); + } + else { + messageCollector.report(CompilerMessageSeverity.INFO, msg.getMessage(), CompilerMessageLocation.NO_LOCATION); + } + } + + if (daemon != null) { + ByteArrayOutputStream compilerOut = new ByteArrayOutputStream(); + ByteArrayOutputStream daemonOut = new ByteArrayOutputStream(); + + Integer res = KotlinCompilerClient.Companion.incrementalCompile(daemon, argsArray, incrementalCaches, compilerOut, daemonOut); + + ProcessCompilerOutput(messageCollector, collector, compilerOut, res.toString()); + BufferedReader reader = new BufferedReader(new StringReader(daemonOut.toString())); + String line = null; + while ((line = reader.readLine()) != null) { + messageCollector.report(CompilerMessageSeverity.INFO, line, CompilerMessageLocation.NO_LOCATION); + } + return true; + } + } + return false; + } + @NotNull private static String getReturnCodeFromObject(@Nullable Object rc) throws Exception { if (rc == null) {