Refactoring startup and shutdown, refactoring service implementation, implementing error and info reporting to compiler output, idle autoshutdown mechanisms, fixing TargetId serializability, some other refactoring

Fixing stream to log handler (by removing non-working optimization), fixing idle time calculation, reporting refactorings
2015-09-06 13:53:38 +02:00
parent 406bfba473
commit c4719175f9
10 changed files with 448 additions and 227 deletions
@@ -35,6 +35,15 @@ fun Process.isAlive() =
            true
        }

+
+public enum class DaemonReportCategory {
+    DEBUG, INFO, EXCEPTION;
+}
+
+public data class DaemonReportMessage(public val category: DaemonReportCategory, public val message: String)
+
+public class DaemonReportingTargets(public val out: PrintStream? = null, public val messages: MutableCollection<DaemonReportMessage>? = null)
+
 public class KotlinCompilerClient {

    companion object {
@@ -42,17 +51,25 @@ public class KotlinCompilerClient {
        val DAEMON_DEFAULT_STARTUP_TIMEOUT_MS = 10000L
        val DAEMON_CONNECT_CYCLE_ATTEMPTS = 3

-        private fun tryFindDaemon(registryDir: File, compilerId: CompilerId, errStream: PrintStream): CompileService? {
+        val verboseReporting = System.getProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) != null
+
+        fun DaemonReportingTargets.report(category: DaemonReportCategory, message: String, source: String = "daemon client") {
+            if (category == DaemonReportCategory.DEBUG && !verboseReporting) return
+            out?.println("[$source] ${category.name()}: $message")
+            messages?.add(DaemonReportMessage(category, "[$source] $message"))
+        }
+
+        private fun tryFindDaemon(registryDir: File, compilerId: CompilerId, reportingTargets: DaemonReportingTargets): CompileService? {
            val classPathDigest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest()
-            var port = 0
            val daemons = registryDir.walk()
                    .map { Pair(it, makeRunFilenameRegex(digest = classPathDigest, port = "(\\d+)").match(it.name)?.groups?.get(1)?.value?.toInt() ?: 0) }
                    .filter { it.second != 0 }
                    .map {
-                        val daemon = tryConnectToDaemon(it.second, errStream)
+                        reportingTargets.report(DaemonReportCategory.DEBUG, "found suitable daemon on port ${it.second}, trying to connect")
+                        val daemon = tryConnectToDaemon(it.second, reportingTargets)
                        // cleaning orphaned file; note: daemon should shut itself down if it detects that the run file is deleted
                        if (daemon == null && !it.first.delete()) {
-                            errStream.println("WARNING: Unable to delete seemingly orphaned file '', cleanup recommended")
+                            reportingTargets.report(DaemonReportCategory.INFO, "WARNING: unable to delete seemingly orphaned file '${it.first.absolutePath}', cleanup recommended")
                        }
                        daemon
                    }
@@ -66,24 +83,24 @@ public class KotlinCompilerClient {
            }
        }

-        private fun tryConnectToDaemon(port: Int, errStream: PrintStream): CompileService? {
+        private fun tryConnectToDaemon(port: Int, reportingTargets: DaemonReportingTargets): CompileService? {
            try {
                val daemon = LocateRegistry.getRegistry("localhost", port)
                        ?.lookup(COMPILER_SERVICE_RMI_NAME)
                if (daemon != null)
                    return daemon as? CompileService ?:
                       throw ClassCastException("Unable to cast compiler service, actual class received: ${daemon.javaClass}")
-                errStream.println("[daemon client] daemon not found")
+                reportingTargets.report(DaemonReportCategory.EXCEPTION, "daemon not found")
            }
            catch (e: ConnectException) {
-                errStream.println("[daemon client] cannot connect to registry: " + (e.getCause()?.getMessage() ?: e.getMessage() ?: "unknown exception"))
+                reportingTargets.report(DaemonReportCategory.EXCEPTION, "cannot connect to registry: " + (e.getCause()?.getMessage() ?: e.getMessage() ?: "unknown exception"))
                // ignoring it - processing below
            }
            return null
        }


-        private fun startDaemon(compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, daemonOptions: DaemonOptions, errStream: PrintStream) {
+        private fun startDaemon(compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, daemonOptions: DaemonOptions, reportingTargets: DaemonReportingTargets) {
            val javaExecutable = File(System.getProperty("java.home"), "bin").let {
                val javaw = File(it, "javaw.exe")
                if (javaw.exists()) javaw
@@ -96,7 +113,7 @@ public class KotlinCompilerClient {
                       COMPILER_DAEMON_CLASS_FQN +
                       daemonOptions.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } +
                       compilerId.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) }
-            errStream.println("[daemon client] starting the daemon as: " + args.joinToString(" "))
+            reportingTargets.report(DaemonReportCategory.DEBUG, "starting the daemon as: " + args.joinToString(" "))
            val processBuilder = ProcessBuilder(args).redirectErrorStream(true)
            // assuming daemon process is deaf and (mostly) silent, so do not handle streams
            val daemon = processBuilder.start()
@@ -113,22 +130,21 @@ public class KotlinCompilerClient {
                                      isEchoRead.release()
                                      return@forEachLine
                                  }
-                                  errStream.println("[daemon] " + it)
+                                  reportingTargets.report(DaemonReportCategory.DEBUG, it, "daemon")
                              }
                    }
            try {
                // trying to wait for process
+                val daemonStartupTimeout = System.getProperty(COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY)?.let {
+                    try {
+                        it.toLong()
+                    }
+                    catch (e: Exception) {
+                        reportingTargets.report(DaemonReportCategory.INFO, "unable to interpret $COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY property ('$it'); using default timeout $DAEMON_DEFAULT_STARTUP_TIMEOUT_MS ms")
+                        null
+                    }
+                } ?: DAEMON_DEFAULT_STARTUP_TIMEOUT_MS
                if (daemonOptions.runFilesPath.isNotEmpty()) {
-                    val daemonStartupTimeout = System.getProperty(COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY)?.let {
-                            try {
-                                it.toLong()
-                            }
-                            catch (e: Exception) {
-                                errStream.println("[daemon client] unable to interpret $COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY property ('$it'); using default timeout $DAEMON_DEFAULT_STARTUP_TIMEOUT_MS ms")
-                                null
-                            }
-                        } ?: DAEMON_DEFAULT_STARTUP_TIMEOUT_MS
-                    errStream.println("[daemon client] waiting for daemon to respond")
                    val succeeded = isEchoRead.tryAcquire(daemonStartupTimeout, TimeUnit.MILLISECONDS)
                    if (!daemon.isAlive())
                        throw Exception("Daemon terminated unexpectedly")
@@ -137,7 +153,7 @@ public class KotlinCompilerClient {
                }
                else
                // without startEcho defined waiting for max timeout
-                    Thread.sleep(DAEMON_DEFAULT_STARTUP_TIMEOUT_MS)
+                    Thread.sleep(daemonStartupTimeout)
            }
            finally {
                // assuming that all important output is already done, the rest should be routed to the log by the daemon itself
@@ -147,10 +163,8 @@ public class KotlinCompilerClient {
            }
        }

-        public fun checkCompilerId(compiler: CompileService, localId: CompilerId, errStream: PrintStream): Boolean {
+        public fun checkCompilerId(compiler: CompileService, localId: CompilerId): Boolean {
            val remoteId = compiler.getCompilerId()
-            errStream.println("[daemon client] remoteId = " + remoteId.toString())
-            errStream.println("[daemon client] localId = " + localId.toString())
            return (localId.compilerVersion.isEmpty() || localId.compilerVersion == remoteId.compilerVersion) &&
                   (localId.compilerClasspath.all { remoteId.compilerClasspath.contains(it) }) &&
                   (localId.compilerDigest.isEmpty() || remoteId.compilerDigest.isEmpty() || localId.compilerDigest == remoteId.compilerDigest)
@@ -202,28 +216,36 @@ public class KotlinCompilerClient {

        }

-        public fun connectToCompileService(compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, daemonOptions: DaemonOptions, errStream: PrintStream, autostart: Boolean = true, checkId: Boolean = true): CompileService? {
+
+        public fun connectToCompileService(compilerId: CompilerId,
+                                           daemonJVMOptions: DaemonJVMOptions,
+                                           daemonOptions: DaemonOptions,
+                                           reportingTargets: DaemonReportingTargets,
+                                           autostart: Boolean = true,
+                                           checkId: Boolean = true
+        ): CompileService? {
+
            var attempts = 0
            var fileLock: FileBasedLock? = null
            try {
                while (attempts++ < DAEMON_CONNECT_CYCLE_ATTEMPTS) {
-                    val service = tryFindDaemon(File(daemonOptions.runFilesPath), compilerId, errStream)
+                    val service = tryFindDaemon(File(daemonOptions.runFilesPath), compilerId, reportingTargets)
                    if (service != null) {
-                        if (!checkId || checkCompilerId(service, compilerId, errStream)) {
-                            errStream.println("[daemon client] found the suitable daemon")
+                        if (!checkId || checkCompilerId(service, compilerId)) {
+                            reportingTargets.report(DaemonReportCategory.DEBUG, "connected to the daemon")
                            return service
                        }
-                        errStream.println("[daemon client] compiler identity don't match: " + compilerId.mappers.flatMap { it.toArgs("") }.joinToString(" "))
+                        reportingTargets.report(DaemonReportCategory.DEBUG, "compiler identity don't match: " + compilerId.mappers.flatMap { it.toArgs("") }.joinToString(" "))
                        if (!autostart) return null
-                        errStream.println("[daemon client] shutdown the daemon")
+                        reportingTargets.report(DaemonReportCategory.DEBUG, "shutdown the daemon")
                        service.shutdown()
                        // TODO: find more reliable way
                        Thread.sleep(1000)
-                        errStream.println("[daemon client] daemon shut down correctly, restarting")
+                        reportingTargets.report(DaemonReportCategory.DEBUG, "daemon shut down correctly, restarting search")
                    }
                    else {
                        if (!autostart) return null
-                        errStream.println("[daemon client] cannot connect to Compile Daemon, trying to start")
+                        reportingTargets.report(DaemonReportCategory.DEBUG, "no suitable daemon found, starting a new one")
                    }

                    if (fileLock == null || !fileLock.isLocked()) {
@@ -234,11 +256,14 @@ public class KotlinCompilerClient {
                        attempts--
                    }
                    else {
-                        startDaemon(compilerId, daemonJVMOptions, daemonOptions, errStream)
-                        errStream.println("[daemon client] daemon started, trying to connect")
+                        startDaemon(compilerId, daemonJVMOptions, daemonOptions, reportingTargets)
+                        reportingTargets.report(DaemonReportCategory.DEBUG, "daemon started, trying to resolve")
                    }
                }
            }
+            catch (e: Exception) {
+                reportingTargets.report(DaemonReportCategory.EXCEPTION, e.toString())
+            }
            finally {
                fileLock?.release()
            }
@@ -246,7 +271,7 @@ public class KotlinCompilerClient {
        }

        public fun shutdownCompileService(daemonOptions: DaemonOptions): Unit {
-            KotlinCompilerClient.connectToCompileService(CompilerId(), DaemonJVMOptions(), daemonOptions, System.out, autostart = false, checkId = false)
+            KotlinCompilerClient.connectToCompileService(CompilerId(), DaemonJVMOptions(), daemonOptions, DaemonReportingTargets(out = System.out), autostart = false, checkId = false)
                    ?.shutdown()
        }

@@ -258,24 +283,26 @@ public class KotlinCompilerClient {

            val outStrm = RemoteOutputStreamServer(out)
            try {
-                return compiler.remoteCompile(args, outStrm, CompileService.OutputFormat.PLAIN)
+                return compiler.remoteCompile(args, outStrm, CompileService.OutputFormat.PLAIN, outStrm)
            }
            finally {
                outStrm.disconnect()
            }
        }

-        public fun incrementalCompile(compiler: CompileService, args: Array<out String>, caches: Map<TargetId, IncrementalCache>, out: OutputStream): Int {
+        public fun incrementalCompile(compiler: CompileService, args: Array<out String>, caches: Map<TargetId, IncrementalCache>, compilerOut: OutputStream, daemonOut: OutputStream): Int {

-            val outStrm = RemoteOutputStreamServer(out)
+            val compilerOutStreamServer = RemoteOutputStreamServer(compilerOut)
+            val daemonOutStreamServer = RemoteOutputStreamServer(daemonOut)
            val cacheServers = hashMapOf<TargetId, RemoteIncrementalCacheServer>()
            try {
                caches.mapValuesTo(cacheServers, { RemoteIncrementalCacheServer( it.getValue()) })
-                return compiler.remoteIncrementalCompile(args, cacheServers, outStrm, CompileService.OutputFormat.XML)
+                return compiler.remoteIncrementalCompile(args, cacheServers, compilerOutStreamServer, CompileService.OutputFormat.XML, daemonOutStreamServer)
            }
            finally {
                cacheServers.forEach { it.getValue().disconnect() }
-                outStrm.disconnect()
+                compilerOutStreamServer.disconnect()
+                daemonOutStreamServer.disconnect()
            }
        }

@@ -299,7 +326,7 @@ public class KotlinCompilerClient {
                    System.err.println("compiler wasn't explicitly specified, attempt to find appropriate jar")
                    System.getProperty("java.class.path")
                            ?.split(File.pathSeparator)
-                            ?.map { File(it).parent }
+                            ?.map { File(it).parentFile }
                            ?.distinct()
                            ?.map {
                                it?.walk()
@@ -317,7 +344,7 @@ public class KotlinCompilerClient {
                compilerId.updateDigest()
            }

-            val daemon = connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, System.out, autostart = !clientOptions.stop, checkId = !clientOptions.stop)
+            val daemon = connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, DaemonReportingTargets(out = System.out), autostart = !clientOptions.stop, checkId = !clientOptions.stop)

            if (daemon == null) {
                if (clientOptions.stop) System.err.println("No daemon found to shut down")
@@ -335,7 +362,7 @@ public class KotlinCompilerClient {
                    try {
                        val memBefore = daemon.getUsedMemory() / 1024
                        val startTime = System.nanoTime()
-                        val res = daemon.remoteCompile(filteredArgs.toArrayList().toTypedArray(), outStrm, CompileService.OutputFormat.PLAIN)
+                        val res = daemon.remoteCompile(filteredArgs.toArrayList().toTypedArray(), outStrm, CompileService.OutputFormat.PLAIN, outStrm)
                        val endTime = System.nanoTime()
                        println("Compilation result code: $res")
                        val memAfter = daemon.getUsedMemory() / 1024
@@ -57,12 +57,19 @@ public interface CompileService : Remote {
    public fun shutdown()

    throws(RemoteException::class)
-    public fun remoteCompile(args: Array<out String>, errStream: RemoteOutputStream, outputFormat: OutputFormat): Int
+    public fun remoteCompile(
+            args: Array<out String>,
+            compilerOutputStream: RemoteOutputStream,
+            outputFormat: OutputFormat,
+            serviceOutputStream: RemoteOutputStream
+    ): Int

    throws(RemoteException::class)
    public fun remoteIncrementalCompile(
            args: Array<out String>,
            caches: Map<TargetId, RemoteIncrementalCache>,
-            outputStream: RemoteOutputStream,
-            outputFormat: OutputFormat): Int
+            compilerOutputStream: RemoteOutputStream,
+            compilerOutputFormat: OutputFormat,
+            serviceOutputStream: RemoteOutputStream
+    ): Int
 }
@@ -36,23 +36,34 @@ public val COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_CHECK_MS: Long = 100L
 public val COMPILE_DAEMON_ENABLED_PROPERTY: String = "kotlin.daemon.enabled"
 public val COMPILE_DAEMON_JVM_OPTIONS_PROPERTY: String = "kotlin.daemon.jvm.options"
 public val COMPILE_DAEMON_OPTIONS_PROPERTY: String = "kotlin.daemon.options"
+public val COMPILE_DAEMON_CLIENT_ALIVE_PATH_PROPERTY: String = "kotlin.daemon.client.alive.path"
+public val COMPILE_DAEMON_REPORT_PERF_PROPERTY: String = "kotlin.daemon.perf"
+public val COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY: String = "kotlin.daemon.verbose"
 public val COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX: String = "--daemon-"
 public val COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY: String ="kotlin.daemon.startup.timeout"
+public val COMPILE_DAEMON_DEFAULT_FILES_PREFIX: String = "kotlin-daemon"
+public val COMPILE_DAEMON_DATA_DIRECTORY_NAME: String = "." + COMPILE_DAEMON_DEFAULT_FILES_PREFIX
 public val COMPILE_DAEMON_TIMEOUT_INFINITE_S: Int = 0
+public val COMPILE_DAEMON_DEFAULT_IDLE_TIMEOUT_S: Int = 7200 // 2 hours
 public val COMPILE_DAEMON_MEMORY_THRESHOLD_INFINITE: Long = 0L

+public val COMPILE_DAEMON_DEFAULT_RUN_DIR_PATH: String get() =
+    // TODO consider special case for windows - local appdata
+    File(System.getProperty("user.home"), COMPILE_DAEMON_DATA_DIRECTORY_NAME).absolutePath
+
+
 val COMPILER_ID_DIGEST = "MD5"

-public fun makeRunFilenameString(ts: String, digest: String, port: String, esc: String = ""): String = "kotlin-daemon$esc.$ts$esc.$digest$esc.$port$esc.run"
-public fun makeRunFilenameRegex(ts: String = "[0-9-]+", digest: String = "[0-9a-f]+", port: String = "\\d+"): Regex = makeRunFilenameString(ts, digest, port, esc = "\\").toRegex()
+public fun makeRunFilenameString(ts: String, digest: String, port: String, esc: String = ""): String = "$COMPILE_DAEMON_DEFAULT_FILES_PREFIX$esc.$ts$esc.$digest$esc.$port$esc.run"
+public fun makeRunFilenameRegex(ts: String = "[0-9TZ:\\.\\+-]+", digest: String = "[0-9a-f]+", port: String = "\\d+"): Regex = makeRunFilenameString(ts, digest, port, esc = "\\").toRegex()


 open class PropMapper<C, V, P: KMutableProperty1<C, V>>(val dest: C,
                                                        val prop: P,
                                                        val names: List<String> = listOf(prop.name),
-                                                        val fromString: (s: String) -> V,
-                                                        val toString: ((v: V) -> String?) = { it.toString() },
-                                                        val skipIf: ((v: V) -> Boolean) = { false },
+                                                        val fromString: (String) -> V,
+                                                        val toString: ((V) -> String?) = { it.toString() },
+                                                        val skipIf: ((V) -> Boolean) = { false },
                                                        val mergeDelimiter: String? = null)
 {
    open fun toArgs(prefix: String = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX): List<String> =
@@ -64,6 +75,17 @@ open class PropMapper<C, V, P: KMutableProperty1<C, V>>(val dest: C,
    open fun apply(s: String) = prop.set(dest, fromString(s))
 }

+
+class NullablePropMapper<C, V: Any?, P: KMutableProperty1<C, V>>(dest: C,
+                                                                 prop: P,
+                                                                 names: List<String> = listOf(),
+                                                                 fromString: ((String) -> V),
+                                                                 toString: ((V) -> String?) = { it.toString() },
+                                                                 skipIf: ((V) -> Boolean) = { it == null },
+                                                                 mergeDelimiter: String? = null)
+: PropMapper<C, V, P>(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name),
+                           fromString = fromString,  toString = toString, skipIf = skipIf, mergeDelimiter = mergeDelimiter)
+
 class StringPropMapper<C, P: KMutableProperty1<C, String>>(dest: C,
                                                           prop: P,
                                                           names: List<String> = listOf(),
@@ -74,10 +96,12 @@ class StringPropMapper<C, P: KMutableProperty1<C, String>>(dest: C,
 : PropMapper<C, String, P>(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name),
                            fromString = fromString,  toString = toString, skipIf = skipIf, mergeDelimiter = mergeDelimiter)

+
 class BoolPropMapper<C, P: KMutableProperty1<C, Boolean>>(dest: C, prop: P, names: List<String> = listOf())
    : PropMapper<C, Boolean, P>(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name),
                                fromString = { true },  toString = { null }, skipIf = { !prop.get(dest) })

+
 class RestPropMapper<C, P: KMutableProperty1<C, MutableCollection<String>>>(dest: C, prop: P)
    : PropMapper<C, MutableCollection<String>, P>(dest = dest, prop = prop, toString = { null }, fromString = { arrayListOf() })
 {
@@ -87,7 +111,7 @@ class RestPropMapper<C, P: KMutableProperty1<C, MutableCollection<String>>>(dest
 }


-inline fun <T, R: Any> Iterable<T>.firstMapOrNull(mappingPredicate: (T) -> Pair<Boolean, R?>): R? {
+inline fun <T, R: Any> Iterable<T>.findWithTransform(mappingPredicate: (T) -> Pair<Boolean, R?>): R? {
    for (element in this) {
        val (found, mapped) = mappingPredicate(element)
        if (found) return mapped
@@ -103,9 +127,10 @@ fun Iterable<String>.filterExtractProps(propMappers: List<PropMapper<*,*,*>>, pr

    while (iter.hasNext()) {
        val param = iter.next()
-        val (propMapper, matchingOption) = propMappers.firstMapOrNull {
-            val name = if (it !is RestPropMapper<*,*>) it.names.firstOrNull { param.startsWith(prefix + it) } else null
-            Pair(name != null, Pair(it, name))
+        val (propMapper, matchingOption) = propMappers.findWithTransform { mapper ->
+            mapper.names
+                    .firstOrNull { param.startsWith(prefix + it) }
+                    .let { Pair(it != null, Pair(mapper, it)) }
        } ?: Pair(null, null)

        when {
@@ -174,16 +199,19 @@ public data class DaemonJVMOptions(
        get() = RestPropMapper(this, ::jvmParams)
 }

+
 public data class DaemonOptions(
-        public var runFilesPath: String = File(System.getProperty("java.io.tmpdir"), "kotlin_daemon").absolutePath,
+        public var runFilesPath: String = COMPILE_DAEMON_DEFAULT_RUN_DIR_PATH,
        public var autoshutdownMemoryThreshold: Long = COMPILE_DAEMON_MEMORY_THRESHOLD_INFINITE,
-        public var autoshutdownIdleSeconds: Int = COMPILE_DAEMON_TIMEOUT_INFINITE_S
+        public var autoshutdownIdleSeconds: Int = COMPILE_DAEMON_DEFAULT_IDLE_TIMEOUT_S,
+        public var clientAliveFlagPath: String? = null
 ) : OptionsGroup {

    override val mappers: List<PropMapper<*, *, *>>
        get() = listOf( PropMapper(this, ::runFilesPath, fromString = { it.trim('"') }),
-                        PropMapper(this, ::autoshutdownMemoryThreshold, fromString = { it.toLong() }, skipIf = { it == 0L }),
-                        PropMapper(this, ::autoshutdownIdleSeconds, fromString = { it.toInt() }, skipIf = { it == 0 }))
+                        PropMapper(this, ::autoshutdownMemoryThreshold, fromString = { it.toLong() }, skipIf = { it == 0L }, mergeDelimiter = "="),
+                        PropMapper(this, ::autoshutdownIdleSeconds, fromString = { it.toInt() }, skipIf = { it == 0 }, mergeDelimiter = "="),
+                        NullablePropMapper(this, ::clientAliveFlagPath, fromString = { it }, toString = { "${it?.trim('\"','\'')}" }, mergeDelimiter = "="))
 }


@@ -256,7 +284,7 @@ public data class CompilerId(
 public fun isDaemonEnabled(): Boolean = System.getProperty(COMPILE_DAEMON_ENABLED_PROPERTY) != null


-public fun configureDaemonLaunchingOptions(opts: DaemonJVMOptions, inheritMemoryLimits: Boolean): DaemonJVMOptions {
+public fun configureDaemonJVMOptions(opts: DaemonJVMOptions, inheritMemoryLimits: Boolean): DaemonJVMOptions {
    // note: sequence matters, explicit override in COMPILE_DAEMON_JVM_OPTIONS_PROPERTY should be done after inputArguments processing
    if (inheritMemoryLimits)
        ManagementFactory.getRuntimeMXBean().inputArguments.filterExtractProps(opts.mappers, "-")
@@ -267,13 +295,16 @@ public fun configureDaemonLaunchingOptions(opts: DaemonJVMOptions, inheritMemory
                                 .map { it.replace("[\\\\](.)".toRegex(), "$1") }
                                 .filterExtractProps(opts.mappers, "-", opts.restMapper))
    }
+
+    System.getProperty(COMPILE_DAEMON_REPORT_PERF_PROPERTY)?.let { opts.jvmParams.add("D" + COMPILE_DAEMON_REPORT_PERF_PROPERTY) }
+    System.getProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY)?.let { opts.jvmParams.add("D" + COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) }
    return opts
 }

-public fun configureDaemonLaunchingOptions(inheritMemoryLimits: Boolean): DaemonJVMOptions =
-    configureDaemonLaunchingOptions(DaemonJVMOptions(), inheritMemoryLimits = inheritMemoryLimits)
+public fun configureDaemonJVMOptions(inheritMemoryLimits: Boolean): DaemonJVMOptions =
+    configureDaemonJVMOptions(DaemonJVMOptions(), inheritMemoryLimits = inheritMemoryLimits)

-jvmOverloads public fun configureDaemonOptions(opts: DaemonOptions = DaemonOptions()): DaemonOptions {
+public fun configureDaemonOptions(opts: DaemonOptions): DaemonOptions {
    System.getProperty(COMPILE_DAEMON_OPTIONS_PROPERTY)?.let {
        val unrecognized = it.trim('"', '\'').split(",").filterExtractProps(opts.mappers, "")
        if (unrecognized.any())
@@ -281,6 +312,13 @@ jvmOverloads public fun configureDaemonOptions(opts: DaemonOptions = DaemonOptio
                    "Unrecognized daemon options passed via property $COMPILE_DAEMON_OPTIONS_PROPERTY: " + unrecognized.joinToString(" ") +
                    "\nSupported options: " + opts.mappers.joinToString(", ", transform = { it.names.first() }))
    }
+    System.getProperty(COMPILE_DAEMON_CLIENT_ALIVE_PATH_PROPERTY)?.let {
+        val trimmed = it.trim('"','\'')
+        if (!trimmed.isBlank()) {
+            opts.clientAliveFlagPath = trimmed
+        }
+    }
    return opts
 }

+public fun configureDaemonOptions(): DaemonOptions = configureDaemonOptions(DaemonOptions())
@@ -19,6 +19,7 @@ package org.jetbrains.kotlin.rmi.service
 import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
 import org.jetbrains.kotlin.rmi.*
 import org.jetbrains.kotlin.service.CompileServiceImpl
+import org.jetbrains.kotlin.service.nowSeconds
 import java.io.File
 import java.io.IOException
 import java.io.OutputStream
@@ -33,6 +34,10 @@ import java.util.*
 import java.util.jar.Manifest
 import java.util.logging.LogManager
 import java.util.logging.Logger
+import kotlin.concurrent.timer
+
+val DAEMON_PERIODIC_CHECK_INTERVAL_MS = 1000L
+

 class LogStream(name: String) : OutputStream() {

@@ -45,25 +50,6 @@ class LogStream(name: String) : OutputStream() {
        else lineBuf.append(byte.toChar())
    }

-    override fun write(data: ByteArray, offset: Int, length: Int) {
-        var ofs = offset
-        var lineStart = ofs
-        while (ofs < length) {
-            if (data[ofs].toChar() == '\n') {
-                flush(data, lineStart, ofs - lineStart)
-                lineStart = ofs + 1
-            }
-            ofs++
-        }
-        if (lineStart < length)
-            lineBuf.append(data, lineStart, length - lineStart)
-    }
-
-    fun flush(data: ByteArray, offset: Int, length: Int) {
-        log.info(lineBuf.toString() + data.toString().substring(offset, length))
-        lineBuf.setLength(0)
-    }
-
    override fun flush() {
        log.info(lineBuf.toString())
        lineBuf.setLength(0)
@@ -86,7 +72,7 @@ public class CompileDaemon {
                "java.util.logging.FileHandler.limit     = 1073741824\n" + // 1Mb
                "java.util.logging.FileHandler.count     = 3\n" +
                "java.util.logging.FileHandler.append    = false\n" +
-                "java.util.logging.FileHandler.pattern   = $logPath/kotlin-daemon.$logTime.%g.log\n" +
+                "java.util.logging.FileHandler.pattern   = $logPath/$COMPILE_DAEMON_DEFAULT_FILES_PREFIX.$logTime.%u%g.log\n" +
                "java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s\\n\n"

            LogManager.getLogManager().readConfiguration(cfg.byteInputStream())
@@ -114,47 +100,88 @@ public class CompileDaemon {

            val compilerId = CompilerId()
            val daemonOptions = DaemonOptions()
-            val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX)

-            if (filteredArgs.any()) {
-                val helpLine = "usage: <daemon> <compilerId options> <daemon options>"
-                log.info(helpLine)
-                println(helpLine)
-                throw IllegalArgumentException("Unknown arguments")
+            try {
+                val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX)
+
+                if (filteredArgs.any()) {
+                    val helpLine = "usage: <daemon> <compilerId options> <daemon options>"
+                    log.info(helpLine)
+                    println(helpLine)
+                    throw IllegalArgumentException("Unknown arguments: " + filteredArgs.joinToString(" "))
+                }
+
+                log.info("starting daemon")
+
+                // TODO: find minimal set of permissions and restore security management
+                //            if (System.getSecurityManager() == null)
+                //                System.setSecurityManager (RMISecurityManager())
+                //
+                //            setDaemonPpermissions(daemonOptions.port)
+
+                val (registry, port) = createRegistry(COMPILE_DAEMON_FIND_PORT_ATTEMPTS)
+                val runFileDir = File(if (daemonOptions.runFilesPath.isBlank()) COMPILE_DAEMON_DEFAULT_RUN_DIR_PATH else daemonOptions.runFilesPath)
+                runFileDir.mkdirs()
+                val runFile = File(runFileDir,
+                                   makeRunFilenameString(ts = "%tFT%<tT.%<tLZ".format(Calendar.getInstance(TimeZone.getTimeZone("Z"))),
+                                                         digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest(),
+                                                         port = port.toString()))
+                if (!runFile.createNewFile()) {
+                    throw IllegalStateException("Unable to create run file '${runFile.absolutePath}'")
+                }
+                runFile.deleteOnExit()
+
+                val compiler = K2JVMCompiler()
+                val compilerService = CompileServiceImpl(registry, compiler, compilerId, daemonOptions)
+
+                if (daemonOptions.runFilesPath.isNotEmpty())
+                    println(daemonOptions.runFilesPath)
+
+                daemonOptions.clientAliveFlagPath?.let {
+                    if (!File(it).exists()) {
+                        log.info("Client alive flag $it do not exist, disable watching it")
+                        daemonOptions.clientAliveFlagPath = null
+                    }
+                }
+
+                // this supposed to stop redirected streams reader(s) on the client side and prevent some situations with hanging threads, but doesn't work reliably
+                // TODO: implement more reliable scheme
+                System.out.close()
+                System.err.close()
+
+                System.setErr(PrintStream(LogStream("stderr")))
+                System.setOut(PrintStream(LogStream("stdout")))
+
+                fun shutdownCondition(check: () -> Boolean, message: String): Boolean {
+                    val res = check()
+                    if (res) {
+                        log.info(message)
+                    }
+                    return res
+                }
+
+                // stopping daemon if any shutdown condition is met
+                timer(initialDelay = DAEMON_PERIODIC_CHECK_INTERVAL_MS, period = DAEMON_PERIODIC_CHECK_INTERVAL_MS) {
+                    val idleSeconds = nowSeconds() - compilerService.lastUsedSeconds
+                    if (shutdownCondition({ !runFile.exists() }, "Run file removed, shutting down") ||
+                        shutdownCondition({ daemonOptions.autoshutdownIdleSeconds != COMPILE_DAEMON_TIMEOUT_INFINITE_S && idleSeconds > daemonOptions.autoshutdownIdleSeconds},
+                                          "Idle timeout exceeded ${daemonOptions.autoshutdownIdleSeconds}s, shutting down") ||
+                        shutdownCondition({ daemonOptions.clientAliveFlagPath?.let { !File(it).exists() } ?: false },
+                                          "Client alive flag ${daemonOptions.clientAliveFlagPath} removed, shutting down"))
+                    {
+                        cancel()
+                        compilerService.shutdown()
+                    }
+                }
+            }
+            catch (e: Exception) {
+                System.err.println("Exception: " + e.getMessage())
+                log.info("Exception" + e.getMessage())
+                log.info(e.toString())
+                // TODO consider exiting without throwing
+                throw e
            }

-            log.info("starting daemon")
-
-            // TODO: find minimal set of permissions and restore security management
-//            if (System.getSecurityManager() == null)
-//                System.setSecurityManager (RMISecurityManager())
-//
-//            setDaemonPpermissions(daemonOptions.port)
-
-            val (registry, port) = createRegistry(COMPILE_DAEMON_FIND_PORT_ATTEMPTS)
-            val runFileDir = File(daemonOptions.runFilesPath)
-            runFileDir.mkdirs()
-            val runFile = File(runFileDir,
-                 makeRunFilenameString(ts = "%tFT%<tRZ".format(Calendar.getInstance(TimeZone.getTimeZone("Z"))),
-                                       digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest(),
-                                       port = port.toString()))
-            if (!runFile.createNewFile()) {
-                throw IllegalStateException("Unable to create run file '${runFile.absolutePath}'")
-            }
-            runFile.deleteOnExit()
-
-            val compiler = K2JVMCompiler()
-            CompileServiceImpl(registry, compiler, compilerId, daemonOptions)
-
-            if (daemonOptions.runFilesPath.isNotEmpty())
-                println(runFile.name)
-
-            // this stops redirected streams reader(s) on the client side and prevent some situations with hanging threads
-            System.out.close()
-            System.err.close()
-
-            System.setErr(PrintStream(LogStream("stderr")))
-            System.setOut(PrintStream(LogStream("stdout")))
        }

        val random = Random()
@@ -26,16 +26,23 @@ import org.jetbrains.kotlin.modules.TargetId
 import org.jetbrains.kotlin.rmi.*
 import org.jetbrains.kotlin.rmi.service.RemoteIncrementalCacheClient
 import org.jetbrains.kotlin.rmi.service.RemoteOutputStreamClient
+import java.io.IOException
 import java.io.PrintStream
+import java.lang.management.ManagementFactory
+import java.lang.management.ThreadMXBean
+import java.net.URLClassLoader
 import java.rmi.registry.Registry
 import java.rmi.server.UnicastRemoteObject
 import java.util.concurrent.TimeUnit
 import java.util.concurrent.locks.ReentrantReadWriteLock
+import java.util.jar.Manifest
 import java.util.logging.Logger
 import kotlin.concurrent.read
 import kotlin.concurrent.write


+fun nowSeconds() = System.nanoTime() / 1000000000L
+
 class CompileServiceImpl<Compiler: CLICompiler<*>>(
        val registry: Registry,
        val compiler: Compiler,
@@ -43,28 +50,73 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
        val daemonOptions: DaemonOptions
 ) : CompileService, UnicastRemoteObject() {

+    // RMI-exposed API
+
+    override fun getCompilerId(): CompilerId = ifAlive { selfCompilerId }
+
+    override fun getUsedMemory(): Long = ifAlive { usedMemory() }
+
+    override fun shutdown() {
+        ifAliveExclusive {
+            log.info("Shutdown started")
+            alive = false
+            UnicastRemoteObject.unexportObject(this, true)
+            log.info("Shutdown complete")
+        }
+    }
+
+    override fun remoteCompile(args: Array<out String>,
+                               compilerOutputStream: RemoteOutputStream,
+                               outputFormat: CompileService.OutputFormat,
+                               serviceOutputStream: RemoteOutputStream
+    ): Int =
+            doCompile(args, compilerOutputStream, serviceOutputStream) { printStream ->
+                when (outputFormat) {
+                    CompileService.OutputFormat.PLAIN -> compiler.exec(printStream, *args)
+                    CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStream, Services.EMPTY, *args)
+                }
+            }
+
+    override fun remoteIncrementalCompile(args: Array<out String>,
+                                          caches: Map<TargetId, CompileService.RemoteIncrementalCache>,
+                                          compilerOutputStream: RemoteOutputStream,
+                                          compilerOutputFormat: CompileService.OutputFormat,
+                                          serviceOutputStream: RemoteOutputStream
+    ): Int =
+            doCompile(args, compilerOutputStream, serviceOutputStream) { printStream ->
+                when (compilerOutputFormat) {
+                    CompileService.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation")
+                    CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStream, createCompileServices(caches), *args)
+                }
+            }
+
+    // internal implementation stuff
+
+    private volatile var _lastUsedSeconds = nowSeconds()
+    public val lastUsedSeconds: Long get() = if (rwlock.isWriteLocked || rwlock.readLockCount - rwlock.readHoldCount > 0) nowSeconds() else _lastUsedSeconds
+
    val log by lazy { Logger.getLogger("compiler") }

    private val rwlock = ReentrantReadWriteLock()
    private var alive = false

    // TODO: consider matching compilerId coming from outside with actual one
-//    private val selfCompilerId by lazy {
-//        CompilerId(
-//                compilerClasspath = System.getProperty("java.class.path")
-//                                            ?.split(File.pathSeparator)
-//                                            ?.map { File(it) }
-//                                            ?.filter { it.exists() }
-//                                            ?.map { it.absolutePath }
-//                                    ?: listOf(),
-//                compilerVersion = loadKotlinVersionFromResource()
-//        )
-//    }
+    //    private val selfCompilerId by lazy {
+    //        CompilerId(
+    //                compilerClasspath = System.getProperty("java.class.path")
+    //                                            ?.split(File.pathSeparator)
+    //                                            ?.map { File(it) }
+    //                                            ?.filter { it.exists() }
+    //                                            ?.map { it.absolutePath }
+    //                                    ?: listOf(),
+    //                compilerVersion = loadKotlinVersionFromResource()
+    //        )
+    //    }

    init {
        // assuming logically synchronized
        try {
-            // cleanup for the case of incorrect restart
+            // cleanup for the case of incorrect restart and many other situations
            UnicastRemoteObject.unexportObject(this, false)
        }
        catch (e: java.rmi.NoSuchObjectException) {
@@ -72,18 +124,28 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
        }

        val stub = UnicastRemoteObject.exportObject(this, 0) as CompileService
-        // TODO: use version-specific name
        registry.rebind (COMPILER_SERVICE_RMI_NAME, stub);
        alive = true
    }

-    public class IncrementalCompilationComponentsImpl(val targetToCache: Map<TargetId, CompileService.RemoteIncrementalCache>): IncrementalCompilationComponents {
+    private class IncrementalCompilationComponentsImpl(val targetToCache: Map<TargetId, CompileService.RemoteIncrementalCache>): IncrementalCompilationComponents {
        // perf: cheap object, but still the pattern may be costly if there are too many calls to cache with the same id (which seems not to be the case now)
        override fun getIncrementalCache(target: TargetId): IncrementalCache = RemoteIncrementalCacheClient(targetToCache[target]!!)
        // TODO: add appropriate proxy into interaction when lookup tracker is needed
        override fun getLookupTracker(): LookupTracker = LookupTracker.DO_NOTHING
    }

+    private fun doCompile(args: Array<out String>, compilerMessagesStreamProxy: RemoteOutputStream, serviceOutputStreamProxy: RemoteOutputStream, body: (PrintStream) -> ExitCode): Int =
+            ifAlive {
+                val compilerMessagesStream = PrintStream( RemoteOutputStreamClient(compilerMessagesStreamProxy))
+                val serviceOutputStream = PrintStream( RemoteOutputStreamClient(serviceOutputStreamProxy))
+                checkedCompile(args, serviceOutputStream) {
+                    val res = body( compilerMessagesStream).code
+                    _lastUsedSeconds = nowSeconds()
+                    res
+                }
+            }
+
    private fun createCompileServices(incrementalCaches: Map<TargetId, CompileService.RemoteIncrementalCache>): Services =
        Services.Builder()
                .register(IncrementalCompilationComponents::class.java, IncrementalCompilationComponentsImpl(incrementalCaches))
@@ -100,19 +162,46 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
        return (rt.totalMemory() - rt.freeMemory())
    }

-    fun<R> checkedCompile(args: Array<out String>, body: () -> R): R {
+    // TODO: consider using version as a part of compiler ID or drop this function
+    private fun loadKotlinVersionFromResource(): String {
+        (javaClass.classLoader as? URLClassLoader)
+        ?.findResource("META-INF/MANIFEST.MF")
+        ?.let {
+            try {
+                return Manifest(it.openStream()).mainAttributes.getValue("Implementation-Version") ?: ""
+            }
+            catch (e: IOException) {}
+        }
+        return ""
+    }
+
+    fun ThreadMXBean.threadCputime() = if (isCurrentThreadCpuTimeSupported()) currentThreadCpuTime else 0L
+    fun ThreadMXBean.threadUsertime() = if (isCurrentThreadCpuTimeSupported()) currentThreadUserTime else 0L
+
+    fun<R> checkedCompile(args: Array<out String>, serviceOut: PrintStream, body: () -> R): R {
        try {
            if (args.none())
                throw IllegalArgumentException("Error: empty arguments list.")
            log.info("Starting compilation with args: " + args.joinToString(" "))
+            val threadMXBean: ThreadMXBean = ManagementFactory.getThreadMXBean()
            val startMem = usedMemory() / 1024
            val startTime = System.nanoTime()
+            val startThreadTime = threadMXBean.threadCputime()
+            val startThreadUserTime = threadMXBean.threadUsertime()
            val res = body()
            val endTime = System.nanoTime()
+            val endThreadTime = threadMXBean.threadCputime()
+            val endThreadUserTime = threadMXBean.threadUsertime()
            val endMem = usedMemory() / 1024
            log.info("Done with result " + res.toString())
-            log.info("Elapsed time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms")
+            val elapsed = TimeUnit.NANOSECONDS.toMillis(endTime - startTime)
+            val elapsedThread = TimeUnit.NANOSECONDS.toMillis(endThreadTime - startThreadTime)
+            val elapsedThreadUser = TimeUnit.NANOSECONDS.toMillis(endThreadUserTime - startThreadUserTime)
+            log.info("Elapsed time: $elapsed ms (thread user: $elapsedThreadUser ms sys: ${elapsedThread - elapsedThreadUser} ms)")
            log.info("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)")
+            System.getProperty(COMPILE_DAEMON_REPORT_PERF_PROPERTY)?.let {
+                serviceOut.println("PERF: TOTAL: $elapsed ms (thread user: $elapsedThreadUser ms sys: ${elapsedThread - elapsedThreadUser} ms); memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)")
+            }
            return res
        }
        // TODO: consider possibilities to handle OutOfMemory
@@ -139,39 +228,4 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
        return res
    }

-    override fun getCompilerId(): CompilerId = ifAlive { selfCompilerId }
-
-    override fun getUsedMemory(): Long = ifAlive { usedMemory() }
-
-    override fun shutdown() {
-        ifAliveExclusive {
-            log.info("Shutdown started")
-            alive = false
-            UnicastRemoteObject.unexportObject(this, true)
-            log.info("Shutdown complete")
-        }
-    }
-
-    override fun remoteCompile(args: Array<out String>, errStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int =
-        doCompile(args, errStream) { printStream ->
-            when (outputFormat) {
-                CompileService.OutputFormat.PLAIN -> compiler.exec(printStream, *args)
-                CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStream, Services.EMPTY, *args)
-            }
-        }
-
-    override fun remoteIncrementalCompile(args: Array<out String>, caches: Map<TargetId, CompileService.RemoteIncrementalCache>, outputStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int =
-        doCompile(args, outputStream) { printStream ->
-            when (outputFormat) {
-                CompileService.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation")
-                CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStream, createCompileServices(caches), *args)
-            }
-        }
-
-    fun doCompile(args: Array<out String>, errStream: RemoteOutputStream, body: (PrintStream) -> ExitCode): Int =
-        ifAlive {
-            checkedCompile(args) {
-                body( PrintStream( RemoteOutputStreamClient(errStream))).code
-            }
-        }
 }