Daemon log, controlling it from launching, proper stdout/err redirection technique, params to control jvm options of the daemon

compiler/rmi/kotlinr/src/KotlinCompilerClient.kt

@@ -70,12 +70,13 @@ public class KotlinCompilerClient {
         }
 
 
-        private fun startDaemon(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream) {
+        private fun startDaemon(compilerId: CompilerId, daemonLaunchingOptions: DaemonLaunchingOptions, daemonOptions: DaemonOptions, errStream: PrintStream) {
             val javaExecutable = listOf(System.getProperty("java.home"), "bin", "java").joinToString(File.separator)
             // TODO: add some specific environment variables to the cp and may be command line, to allow some specific startup configs
             val args = listOf(javaExecutable,
-                              "-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator),
+                              "-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator)) +
-                              COMPILER_DAEMON_CLASS_FQN) +
+                       daemonLaunchingOptions.jvmParams +
+                       COMPILER_DAEMON_CLASS_FQN +
                        daemonOptions.asParams +
                        compilerId.asParams
             errStream.println("[daemon client] starting the daemon as: " + args.joinToString(" "))
@@ -131,7 +132,7 @@ public class KotlinCompilerClient {
                    (localId.compilerDigest.isEmpty() || remoteId.compilerDigest.isEmpty() || localId.compilerDigest == remoteId.compilerDigest)
         }
 
-        public fun connectToCompileService(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream, autostart: Boolean = true, checkId: Boolean = true): CompileService? {
+        public fun connectToCompileService(compilerId: CompilerId, daemonLaunchingOptions: DaemonLaunchingOptions, daemonOptions: DaemonOptions, errStream: PrintStream, autostart: Boolean = true, checkId: Boolean = true): CompileService? {
             val service = connectToService(compilerId, daemonOptions, errStream)
             if (service != null) {
                 if (!checkId || checkCompilerId(service, compilerId, errStream)) {
@@ -151,7 +152,7 @@ public class KotlinCompilerClient {
                 else errStream.println("[daemon client] cannot connect to Compile Daemon, trying to start")
             }
 
-            startDaemon(compilerId, daemonOptions, errStream)
+            startDaemon(compilerId, daemonLaunchingOptions, daemonOptions, errStream)
             errStream.println("[daemon client] daemon started, trying to connect")
             return connectToService(compilerId, daemonOptions, errStream)
         }
@@ -187,8 +188,9 @@ public class KotlinCompilerClient {
         platformStatic public fun main(vararg args: String) {
             val compilerId = CompilerId()
             val daemonOptions = DaemonOptions()
+            val daemonLaunchingOptions = DaemonLaunchingOptions()
             val clientOptions = ClientOptions()
-            val filteredArgs = args.asIterable().propParseFilter(compilerId, daemonOptions, clientOptions)
+            val filteredArgs = args.asIterable().propParseFilter(compilerId, daemonOptions, daemonLaunchingOptions, clientOptions)
 
             if (!clientOptions.stop) {
                 if (compilerId.compilerClasspath.none()) {
@@ -214,7 +216,7 @@ public class KotlinCompilerClient {
                 compilerId.updateDigest()
             }
 
-            connectToCompileService(compilerId, daemonOptions, System.out, autostart = !clientOptions.stop, checkId = !clientOptions.stop)?.let {
+            connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, System.out, autostart = !clientOptions.stop, checkId = !clientOptions.stop)?.let {
                 when {
                     clientOptions.stop -> {
                         println("Shutdown the daemon")

compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt

@@ -78,6 +78,18 @@ public fun Iterable<String>.propParseFilter(vararg cs: CmdlineParams) : Iterable
     propParseFilter(cs.flatMap { it.parsers })
 
 
+public data class DaemonLaunchingOptions(
+        public var jvmParams: List<String> = listOf()
+) : CmdlineParams {
+
+    override val asParams: Iterable<String>
+        get() =
+            propToParams(::jvmParams, { it.joinToString("##") }) // TODO: consider some other options rather than using potentially dangerous delimiter
+
+    override val parsers: List<PropParser<*,*,*>>
+        get() = listOf( PropParser(this, ::jvmParams, { it.split("##")})) // TODO: see appropriate comment in asParams
+}
+
 public data class DaemonOptions(
         public var port: Int = COMPILE_DAEMON_DEFAULT_PORT,
         public var autoshutdownMemoryThreshold: Long = 0 /* 0 means unchecked */,

compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt

@@ -17,19 +17,83 @@
 package org.jetbrains.kotlin.rmi.service
 
 import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
-import org.jetbrains.kotlin.rmi.COMPILE_DAEMON_DEFAULT_PORT
 import org.jetbrains.kotlin.rmi.CompilerId
 import org.jetbrains.kotlin.rmi.DaemonOptions
 import org.jetbrains.kotlin.rmi.propParseFilter
 import org.jetbrains.kotlin.service.CompileServiceImpl
+import java.io.OutputStream
+import java.io.PrintStream
 import java.rmi.RMISecurityManager
 import java.rmi.registry.LocateRegistry
+import java.text.SimpleDateFormat
+import java.util.*
+import java.util.logging.LogManager
+import java.util.logging.Logger
 import kotlin.platform.platformStatic
 
+class LogStream(name: String) : OutputStream() {
+
+    val log by lazy { Logger.getLogger(name) }
+
+    val lineBuf = StringBuilder()
+
+    override fun write(byte: Int) {
+        if (byte.toChar() == '\n') flush()
+        else lineBuf.append(byte.toChar())
+    }
+
+    override fun write(data: ByteArray, offset: Int, length: Int) {
+        var ofs = offset
+        var lineStart = ofs
+        while (ofs < length) {
+            if (data[ofs].toChar() == '\n') {
+                flush(data, lineStart, ofs - lineStart)
+                lineStart = ofs + 1
+            }
+            ofs++
+        }
+        if (lineStart < length)
+            lineBuf.append(data, lineStart, length - lineStart)
+    }
+
+    fun flush(data: ByteArray, offset: Int, length: Int) {
+        if (offset == 0 && length == data.size())
+            log.info(lineBuf.toString() + data.toString())
+        else
+            log.info(lineBuf.toString() + data.toString().substring(offset, length))
+        lineBuf.setLength(0)
+    }
+
+    override fun flush() {
+        log.info(lineBuf.toString())
+        lineBuf.setLength(0)
+    }
+}
+
+
 public class CompileDaemon {
 
     companion object {
 
+        init {
+            val logPath: String = System.getProperty("kotlin.daemon.log.path")?.trimEnd('/','\\') ?: "%t"
+            val logTime: String = SimpleDateFormat("yyyy-MM-dd.HH-mm-ss-SSS").format(Date())
+            val cfg: String =
+                "handlers = java.util.logging.FileHandler\n" +
+                "java.util.logging.FileHandler.level     = ALL\n" +
+                "java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter\n" +
+                "java.util.logging.FileHandler.encoding  = UTF-8\n" +
+                "java.util.logging.FileHandler.limit     = 1073741824\n" + // 1Mb
+                "java.util.logging.FileHandler.count     = 3\n" +
+                "java.util.logging.FileHandler.append    = false\n" +
+                "java.util.logging.FileHandler.pattern   = $logPath/kotlin-daemon.$logTime.%g.log\n" +
+                "java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s\\n\n"
+
+            LogManager.getLogManager().readConfiguration(cfg.byteInputStream())
+        }
+
+        val log by lazy { Logger.getLogger("daemon") }
+
         platformStatic public fun main(args: Array<String>) {
 
             val compilerId = CompilerId()
@@ -37,10 +101,14 @@ public class CompileDaemon {
             val filteredArgs = args.asIterable().propParseFilter(compilerId, daemonOptions)
 
             if (filteredArgs.any()) {
-                println("usage: <daemon> <compilerId options> <daemon options>")
+                val helpLine = "usage: <daemon> <compilerId options> <daemon options>"
+                log.info(helpLine)
+                println(helpLine)
                 throw IllegalArgumentException("Unknown arguments")
             }
 
+            log.info("starting daemon")
+
             // TODO: find minimal set of permissions and restore security management
 //            if (System.getSecurityManager() == null)
 //                System.setSecurityManager (RMISecurityManager())
@@ -54,6 +122,13 @@ public class CompileDaemon {
 
             if (daemonOptions.startEcho.isNotEmpty())
                 println(daemonOptions.startEcho)
+
+            // this stops redirected streams reader(s) on the client side and prevent some situations with hanging threads
+            System.out.close()
+            System.err.close()
+
+            System.setErr(PrintStream(LogStream("stderr")))
+            System.setOut(PrintStream(LogStream("stdout")))
         }
     }
 }

compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt

@@ -48,6 +48,8 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
         val daemonOptions: DaemonOptions
 ) : CompileService, UnicastRemoteObject() {
 
+    val log by lazy { Logger.getLogger("compiler") }
+
     private val rwlock = ReentrantReadWriteLock()
     private var alive = false
 
@@ -118,19 +120,19 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
         try {
             if (args.none())
                 throw IllegalArgumentException("Error: empty arguments list.")
-            println("Starting compilation with args: " + args.joinToString(" "))
+            log.info("Starting compilation with args: " + args.joinToString(" "))
             val startMem = usedMemory() / 1024
             val startTime = System.nanoTime()
             val res = body()
             val endTime = System.nanoTime()
             val endMem = usedMemory() / 1024
-            println("Done")
+            log.info("Done")
-            println("Elapsed time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms")
+            log.info("Elapsed time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms")
-            println("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)")
+            log.info("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)")
             return res
         }
         catch (e: Exception) {
-            println("Error: $e")
+            log.info("Error: $e")
             throw e
         }
     }
@@ -147,7 +149,7 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
 
     fun<R> spy(msg: String, body: () -> R): R {
         val res = body()
-        println(msg + " = " + res.toString())
+        log.info(msg + " = " + res.toString())
         return res
     }
 
@@ -157,10 +159,10 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
 
     override fun shutdown() {
         ifAliveExclusive {
-            println("Shutdown started")
+            log.info("Shutdown started")
             alive = false
             UnicastRemoteObject.unexportObject(this, true)
-            println("Shutdown complete")
+            log.info("Shutdown complete")
         }
     }