Implement more reliable daemon election, fixes KT-15562

also making shutdown more reliable

compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/KotlinCompilerClient.kt

@@ -291,17 +291,23 @@ object KotlinCompilerClient {
     }
 
     private fun tryFindSuitableDaemonOrNewOpts(registryDir: File, compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, report: (DaemonReportCategory, String) -> Unit): Pair<CompileService?, DaemonJVMOptions> {
-        val aliveWithOpts = walkDaemons(registryDir, compilerId, report = report)
+        registryDir.mkdirs()
-                .map { Pair(it, it.getDaemonJVMOptions()) }
+        val timestampMarker = createTempFile("kotlin-daemon-client-tsmarker", directory = registryDir)
-                .filter { it.second.isGood }
+        val aliveWithMetadata = try {
-                .sortedWith(compareByDescending(DaemonJVMOptionsMemoryComparator(), { it.second.get() }))
+            walkDaemons(registryDir, compilerId, timestampMarker, report = report).toList()
+        }
+        finally {
+            timestampMarker.delete()
+        }
+        val comparator = compareByDescending<DaemonWithMetadata, DaemonJVMOptions>(DaemonJVMOptionsMemoryComparator(), { it.jvmOptions })
+                .thenBy(FileAgeComparator()) { it.runFile }
         val optsCopy = daemonJVMOptions.copy()
         // if required options fit into fattest running daemon - return the daemon and required options with memory params set to actual ones in the daemon
-        return aliveWithOpts.firstOrNull()?.check { daemonJVMOptions memorywiseFitsInto it.second.get() }?.let {
+        return aliveWithMetadata.maxWith(comparator)?.takeIf { daemonJVMOptions memorywiseFitsInto it.jvmOptions }?.let {
-                Pair(it.first, optsCopy.updateMemoryUpperBounds(it.second.get()))
+                Pair(it.daemon, optsCopy.updateMemoryUpperBounds(it.jvmOptions))
             }
             // else combine all options from running daemon to get fattest option for a new daemon to run
-            ?: Pair(null, aliveWithOpts.fold(optsCopy, { opts, d -> opts.updateMemoryUpperBounds(d.second.get()) }))
+            ?: Pair(null, aliveWithMetadata.fold(optsCopy, { opts, d -> opts.updateMemoryUpperBounds(d.jvmOptions) }))
     }
 
 

compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ClientUtils.kt

@@ -40,30 +40,48 @@ fun makePortFromRunFilenameExtractor(digest: String): (String) -> Int? {
     }
 }
 
+private const val ORPHANED_RUN_FILE_AGE_THRESHOLD_MS = 1000000L
+
+data class DaemonWithMetadata(val daemon: CompileService, val runFile: File, val jvmOptions: DaemonJVMOptions)
 
 fun walkDaemons(registryDir: File,
                 compilerId: CompilerId,
-                filter: (File, Int) -> Boolean = { f, p -> true },
+                fileToCompareTimestamp: File,
-                report: (DaemonReportCategory, String) -> Unit = { cat, msg -> }
+                filter: (File, Int) -> Boolean = { _, _ -> true },
-): Sequence<CompileService> {
+                report: (DaemonReportCategory, String) -> Unit = { _, _ -> }
+): Sequence<DaemonWithMetadata> {
     val classPathDigest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest().toHexString()
     val portExtractor = makePortFromRunFilenameExtractor(classPathDigest)
     return registryDir.walk()
             .map { Pair(it, portExtractor(it.name)) }
             .filter { it.second != null && filter(it.first, it.second!!) }
-            .mapNotNull {
+            .mapNotNull { fileWithPort ->
-                assert(it.second!! > 0 && it.second!! < MAX_PORT_NUMBER)
+                assert(fileWithPort.second!! in 1..(MAX_PORT_NUMBER - 1))
-                report(DaemonReportCategory.DEBUG, "found daemon on port ${it.second}, trying to connect")
+                val relativeAge = fileToCompareTimestamp.lastModified() - fileWithPort.first.lastModified()
-                val daemon = tryConnectToDaemon(it.second!!, report)
+                report(DaemonReportCategory.DEBUG, "found daemon on port ${fileWithPort.second} ($relativeAge ms old), trying to connect")
+                val daemon = tryConnectToDaemon(fileWithPort.second!!, report)
                 // cleaning orphaned file; note: daemon should shut itself down if it detects that the run file is deleted
-                if (daemon == null && !it.first.delete()) {
+                if (daemon == null) {
-                    report(DaemonReportCategory.INFO, "WARNING: unable to delete seemingly orphaned file '${it.first.absolutePath}', cleanup recommended")
+                    if (relativeAge - ORPHANED_RUN_FILE_AGE_THRESHOLD_MS <= 0) {
+                        report(DaemonReportCategory.DEBUG, "found fresh run file '${fileWithPort.first.absolutePath}' ($relativeAge ms old), but no daemon, ignoring it")
+                    }
+                    else {
+                        report(DaemonReportCategory.DEBUG, "found seemingly orphaned run file '${fileWithPort.first.absolutePath}' ($relativeAge ms old), deleting it")
+                        if (!fileWithPort.first.delete()) {
+                            report(DaemonReportCategory.INFO, "WARNING: unable to delete seemingly orphaned file '${fileWithPort.first.absolutePath}', cleanup recommended")
+                        }
+                    }
+                }
+                try {
+                    daemon?.let { DaemonWithMetadata(it, fileWithPort.first, it.getDaemonJVMOptions().get()) }
+                }
+                catch (e: Exception) {
+                    report(DaemonReportCategory.INFO, "ERROR: unable to retrieve daemon JVM options, assuming daemon is dead: ${e.message}")
+                    null
                 }
-                daemon
             }
 }
 
-
 private inline fun tryConnectToDaemon(port: Int, report: (DaemonReportCategory, String) -> Unit): CompileService? {
 
     try {
@@ -88,3 +106,17 @@ fun makeAutodeletingFlagFile(keyword: String = "compiler-client", baseDir: File?
     flagFile.deleteOnExit()
     return flagFile
 }
+
+// Comparator for reliable choice between daemons
+class FileAgeComparator : Comparator<File> {
+    override fun compare(left: File, right: File): Int {
+        val leftTS = left.lastModified()
+        val rightTS = right.lastModified()
+        return when {
+            leftTS == 0L || rightTS == 0L -> 0 // cannot read any file timestamp, => undecidable
+            leftTS > rightTS -> -1
+            leftTS < rightTS -> 1
+            else -> compareValues(left.canonicalPath, right.canonicalPath)
+        }
+    }
+}

compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt

@@ -267,20 +267,19 @@ class CompileServiceImpl(
             ifAlive { CompileService.CallResult.Good(usedMemory(withGC = true)) }
 
     override fun shutdown(): CompileService.CallResult<Nothing> = ifAliveExclusive(minAliveness = Aliveness.LastSession, ignoreCompilerChanged = true) {
-        shutdownImpl()
+        shutdownWithDelay()
         CompileService.CallResult.Ok()
     }
 
     override fun scheduleShutdown(graceful: Boolean): CompileService.CallResult<Boolean> = ifAlive(minAliveness = Aliveness.Alive) {
         CompileService.CallResult.Good(
                 if (!graceful || state.alive.compareAndSet(Aliveness.Alive.ordinal, Aliveness.LastSession.ordinal)) {
-                    timer.schedule(0) {
+                    timer.schedule(1) {
                         ifAliveExclusive(minAliveness = Aliveness.LastSession, ignoreCompilerChanged = true) {
-                            if (!graceful || state.sessions.isEmpty()) {
+                            when {
-                                shutdownImpl()
+                                !graceful -> shutdownImpl()
-                            }
+                                state.sessions.isEmpty() -> shutdownWithDelay()
-                            else {
+                                else -> log.info("Some sessions are active, waiting for them to finish")
-                                log.info("Some sessions are active, waiting for them to finish")
                             }
                             CompileService.CallResult.Ok()
                         }
@@ -529,7 +528,7 @@ class CompileServiceImpl(
         val stub = UnicastRemoteObject.exportObject(this, port, LoopbackNetworkInterface.clientLoopbackSocketFactory, LoopbackNetworkInterface.serverLoopbackSocketFactory) as CompileService
         registry.rebind (COMPILER_SERVICE_RMI_NAME, stub)
 
-        timer.schedule(0) {
+        timer.schedule(100) {
             initiateElections()
         }
         timer.schedule(delay = DAEMON_PERIODIC_CHECK_INTERVAL_MS, period = DAEMON_PERIODIC_CHECK_INTERVAL_MS) {
@@ -552,7 +551,7 @@ class CompileServiceImpl(
             // 1. check if unused for a timeout - shutdown
             if (shutdownCondition({ daemonOptions.autoshutdownUnusedSeconds != COMPILE_DAEMON_TIMEOUT_INFINITE_S && compilationsCounter.get() == 0 && nowSeconds() - lastUsedSeconds > daemonOptions.autoshutdownUnusedSeconds },
                                   "Unused timeout exceeded ${daemonOptions.autoshutdownUnusedSeconds}s, shutting down")) {
-                shutdown()
+                scheduleShutdown(true)
             }
             else {
                 var anyDead = state.sessions.cleanDead()
@@ -561,7 +560,7 @@ class CompileServiceImpl(
 
                 // 3. check if in graceful shutdown state and all sessions are closed
                 if (shutdownCondition({ state.alive.get() == Aliveness.LastSession.ordinal && state.sessions.isEmpty() }, "All sessions finished, shutting down")) {
-                    shutdown()
+                    shutdownWithDelay()
                     shuttingDown = true
                 }
 
@@ -587,7 +586,7 @@ class CompileServiceImpl(
                     // TODO: could be too expensive anyway, consider removing this check
                     shutdownCondition({ classpathWatcher.isChanged }, "Compiler changed"))
                 {
-                    shutdown()
+                    scheduleShutdown(true)
                     shuttingDown = true
                 }
 
@@ -604,35 +603,44 @@ class CompileServiceImpl(
 
         ifAlive {
 
-            val aliveWithOpts = walkDaemons(File(daemonOptions.runFilesPathOrDefault), compilerId, filter = { f, p -> p != port }, report = { lvl, msg -> log.info(msg) })
+            val aliveWithOpts = walkDaemons(File(daemonOptions.runFilesPathOrDefault), compilerId, runFile, filter = { f, p -> p != port }, report = { _, msg -> log.info(msg) }).toList()
-                    .map { Pair(it, it.getDaemonJVMOptions()) }
+            val comparator = compareByDescending<DaemonWithMetadata, DaemonJVMOptions>(DaemonJVMOptionsMemoryComparator(), { it.jvmOptions })
-                    .filter { it.second.isGood }
+                    .thenBy(FileAgeComparator()) { it.runFile }
-                    .sortedWith(compareByDescending(DaemonJVMOptionsMemoryComparator(), { it.second.get() }))
+            aliveWithOpts.maxWith(comparator)?.let { bestDaemonWithMetadata ->
-            if (aliveWithOpts.any()) {
+                val fattestOpts = bestDaemonWithMetadata.jvmOptions
-                val fattestOpts = aliveWithOpts.first().second.get()
                 // second part of the condition means that we prefer other daemon if is "equal" to the current one
-                if (fattestOpts memorywiseFitsInto daemonJVMOptions && !(daemonJVMOptions memorywiseFitsInto fattestOpts)) {
+                if (fattestOpts memorywiseFitsInto daemonJVMOptions && FileAgeComparator().compare(bestDaemonWithMetadata.runFile, runFile) < 0 ) {
                     // all others are smaller that me, take overs' clients and shut them down
+                    log.info("Assuming other daemons have lower prio, taking clients from them and schedule them to shutdown: my runfile: ${runFile.name} (${runFile.lastModified()}) vs best other runfile: ${bestDaemonWithMetadata.runFile.name} (${bestDaemonWithMetadata.runFile.lastModified()})")
                     aliveWithOpts.forEach {
-                        it.first.getClients().check { it.isGood }?.let {
+                        try {
-                            it.get().forEach { registerClient(it) }
+                            it.daemon.getClients().check { it.isGood }?.let {
+                                it.get().forEach { registerClient(it) }
+                            }
+                            it.daemon.scheduleShutdown(true)
+                        }
+                        catch (e: Exception) {
+                            log.info("Cannot connect to a daemon, assuming dying ('${it.runFile.canonicalPath}'): ${e.message}")
                         }
-                        it.first.scheduleShutdown(true)
                     }
                 }
-                else if (daemonJVMOptions memorywiseFitsInto fattestOpts) {
+                else if (daemonJVMOptions memorywiseFitsInto fattestOpts && FileAgeComparator().compare(bestDaemonWithMetadata.runFile, runFile) > 0) {
                     // there is at least one bigger, handover my clients to it and shutdown
+                    log.info("Assuming other daemons have higher prio, handover clients to it and schedule shutdown: my runfile: ${runFile.name} (${runFile.lastModified()}) vs best other runfile: ${bestDaemonWithMetadata.runFile.name} (${bestDaemonWithMetadata.runFile.lastModified()})")
                     scheduleShutdown(true)
-                    aliveWithOpts.first().first.let { fattest ->
+                    aliveWithOpts.first().daemon.let { fattest ->
                         getClients().check { it.isGood }?.let {
                             it.get().forEach { fattest.registerClient(it) }
                         }
                     }
                 }
-                // else - do nothing, all daemons are staying
+                else {
-                // TODO: implement some behaviour here, e.g.:
+                    // undecided, do nothing
-                //   - shutdown/takeover smaller daemon
+                    log.info("Assuming other daemon(s) have equal prio, continue: ${runFile.name} (${runFile.lastModified()}) vs best other runfile: ${bestDaemonWithMetadata.runFile.name} (${bestDaemonWithMetadata.runFile.lastModified()})")
-                //   - run (or better persuade client to run) a bigger daemon (in fact may be even simple shutdown will do, because of client's daemon choosing logic)
+                    // TODO: implement some behaviour here, e.g.:
+                    //   - shutdown/takeover smaller daemon
+                    //   - run (or better persuade client to run) a bigger daemon (in fact may be even simple shutdown will do, because of client's daemon choosing logic)
+                }
             }
             CompileService.CallResult.Ok()
         }
@@ -653,8 +661,11 @@ class CompileServiceImpl(
         timer.schedule(daemonOptions.shutdownDelayMilliseconds) {
             state.delayedShutdownQueued.set(false)
             if (currentCompilationsCount == compilationsCounter.get()) {
-                log.fine("Execute delayed shutdown")
+                ifAliveExclusive(minAliveness = Aliveness.LastSession, ignoreCompilerChanged = true) {
-                shutdown()
+                    log.fine("Execute delayed shutdown")
+                    shutdownImpl()
+                    CompileService.CallResult.Ok()
+                }
             }
             else {
                 log.info("Cancel delayed shutdown due to new client")
@@ -801,7 +812,7 @@ class CompileServiceImpl(
             state.alive.get() < minAliveness.ordinal -> CompileService.CallResult.Dying()
             !ignoreCompilerChanged && classpathWatcher.isChanged -> {
                 log.info("Compiler changed, scheduling shutdown")
-                timer.schedule(0) { shutdown() }
+                shutdownWithDelay()
                 CompileService.CallResult.Dying()
             }
             else -> {

compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinCompileDaemon.kt

@@ -30,9 +30,7 @@ import java.net.URLClassLoader
 import java.text.SimpleDateFormat
 import java.util.*
 import java.util.jar.Manifest
-import java.util.logging.Level
+import java.util.logging.*
-import java.util.logging.LogManager
-import java.util.logging.Logger
 import kotlin.concurrent.schedule
 
 val DAEMON_PERIODIC_CHECK_INTERVAL_MS = 1000L
@@ -55,7 +53,6 @@ class LogStream(name: String) : OutputStream() {
     }
 }
 
-
 object KotlinCompileDaemon {
 
     init {
@@ -71,7 +68,7 @@ object KotlinCompileDaemon {
                 "java.util.logging.FileHandler.count     = ${if (fileIsGiven) 1 else 3}\n" +
                 "java.util.logging.FileHandler.append    = $fileIsGiven\n" +
                 "java.util.logging.FileHandler.pattern   = ${if (fileIsGiven) logPath else (logPath + File.separator + "${COMPILE_DAEMON_DEFAULT_FILES_PREFIX}.$logTime.%u%g.log")}\n" +
-                "java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s\\n\n"
+                "java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s%n\n"
 
         LogManager.getLogManager().readConfiguration(cfg.byteInputStream())
     }
@@ -96,6 +93,7 @@ object KotlinCompileDaemon {
         log.info("Kotlin compiler daemon version " + (loadVersionFromResource() ?: "<unknown>"))
         log.info("daemon JVM args: " + ManagementFactory.getRuntimeMXBean().inputArguments.joinToString(" "))
         log.info("daemon args: " + args.joinToString(" "))
+        log.info("daemon process name: " + ManagementFactory.getRuntimeMXBean().name)
 
         val compilerId = CompilerId()
         val daemonOptions = DaemonOptions()
@@ -159,6 +157,8 @@ object KotlinCompileDaemon {
             if (daemonOptions.runFilesPath.isNotEmpty())
                 println(daemonOptions.runFilesPath)
 
+            log.info("daemon is listening on port: $port")
+
             // this supposed to stop redirected streams reader(s) on the client side and prevent some situations with hanging threads, but doesn't work reliably
             // TODO: implement more reliable scheme
             System.out.close()

compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt

@@ -73,10 +73,20 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
 
     private fun run(logName: String, vararg args: String): Int = runJava(getTestBaseDir(), logName, *args)
 
+    override fun setUp() {
+        super.setUp()
+        System.setProperty("COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY", "true")
+    }
+
+    override fun tearDown() {
+        System.clearProperty("COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY")
+        super.tearDown()
+    }
 
     fun testHelloApp() {
         withFlagFile(getTestName(true), ".alive") { flagFile ->
             val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath,
+                                              shutdownDelayMilliseconds = 1,
                                               verbose = true,
                                               reportPerf = true)
 
@@ -141,7 +151,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
         val backupOptions = System.getProperty(COMPILE_DAEMON_OPTIONS_PROPERTY)
         try {
             System.setProperty(COMPILE_DAEMON_OPTIONS_PROPERTY, "runFilesPath=abcd,autoshutdownIdleSeconds=1111")
-            val opts = configureDaemonOptions()
+            val opts = configureDaemonOptions(DaemonOptions(shutdownDelayMilliseconds = 1))
             assertEquals("abcd", opts.runFilesPath)
             assertEquals(1111, opts.autoshutdownIdleSeconds)
         }
@@ -152,7 +162,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
 
     fun testDaemonInstancesSimple() {
         withFlagFile(getTestName(true), ".alive") { flagFile ->
-            val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
+            val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
             val compilerId2 = CompilerId.makeCompilerId(compilerClassPath +
                                                         File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-compiler-sources.jar"))
 
@@ -196,7 +206,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
 
     fun testDaemonAutoshutdownOnUnused() {
         withFlagFile(getTestName(true), ".alive") { flagFile ->
-            val daemonOptions = DaemonOptions(autoshutdownUnusedSeconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
+            val daemonOptions = DaemonOptions(autoshutdownUnusedSeconds = 1, shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
             KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions)
 
             val logFile = createTempFile("kotlin-daemon-test", ".log")
@@ -223,7 +233,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
 
     fun testDaemonAutoshutdownOnIdle() {
         withFlagFile(getTestName(true), ".alive") { flagFile ->
-            val daemonOptions = DaemonOptions(autoshutdownIdleSeconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
+            val daemonOptions = DaemonOptions(autoshutdownIdleSeconds = 1, shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
             KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions)
 
             val logFile = createTempFile("kotlin-daemon-test", ".log")
@@ -255,7 +265,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
 
     fun testDaemonGracefulShutdown() {
         withFlagFile(getTestName(true), ".alive") { flagFile ->
-            val daemonOptions = DaemonOptions(autoshutdownIdleSeconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
+            val daemonOptions = DaemonOptions(autoshutdownIdleSeconds = 1, shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
             KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions)
 
             val logFile = createTempFile("kotlin-daemon-test", ".log")
@@ -303,7 +313,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
     fun testDaemonExecutionViaIntermediateProcess() {
         val clientAliveFile = createTempFile("kotlin-daemon-transitive-run-test", ".run")
         val runFilesPath = File(tmpdir, getTestName(true)).absolutePath
-        val daemonOptions = DaemonOptions(runFilesPath = runFilesPath)
+        val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, runFilesPath = runFilesPath)
         val jar = tmpdir.absolutePath + File.separator + "hello.jar"
         val args = listOf(
                 File(File(System.getProperty("java.home"), "bin"), "java").absolutePath,
@@ -357,7 +367,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
         assertTrue(PARALLEL_THREADS_TO_COMPILE <= LoopbackNetworkInterface.SERVER_SOCKET_BACKLOG_SIZE)
 
         withFlagFile(getTestName(true), ".alive") { flagFile ->
-            val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
+            val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
             val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritAdditionalProperties = false)
             val daemon = KotlinCompilerClient.connectToCompileService(compilerId, flagFile, daemonJVMOptions, daemonOptions, DaemonReportingTargets(out = System.err), autostart = true)
             assertNotNull("failed to connect daemon", daemon)
@@ -406,16 +416,18 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
         val startLatch = CountDownLatch(1)
         val doneLatch = CountDownLatch(PARALLEL_THREADS_TO_START)
 
-        val clients = Array(PARALLEL_THREADS_TO_START, { hashSetOf<String>() } )
         val resultCodes = arrayOfNulls<Int>(PARALLEL_THREADS_TO_START)
         val outStreams = Array(PARALLEL_THREADS_TO_START, { ByteArrayOutputStream() })
+        val logFiles = kotlin.arrayOfNulls<File>(PARALLEL_THREADS_TO_START)
 
-        fun connectThread(threadNo: Int) = thread {
+        fun connectThread(threadNo: Int) = thread(name = "daemonConnect$threadNo") {
-            withFlagFile(getTestName(true), ".alive") { flagFile ->
+            try {
-                val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
+                withFlagFile(getTestName(true), ".alive") { flagFile ->
-                val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritAdditionalProperties = false)
+                    val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1000, runFilesPath = File(tmpdir, getTestName(true)).absolutePath, verbose = true)
-                startLatch.await()
+                    val logFile = createTempFile("kotlin-daemon-test", ".log")
-                try {
+                    val daemonJVMOptions =
+                            configureDaemonJVMOptions("D$COMPILE_DAEMON_LOG_PATH_PROPERTY=\"${logFile.loggerCompatiblePath}\"",
+                                                      inheritMemoryLimits = false, inheritAdditionalProperties = false)
                     val daemon = KotlinCompilerClient.connectToCompileService(compilerId, flagFile, daemonJVMOptions, daemonOptions, DaemonReportingTargets(out = System.err), autostart = true)
                     assertNotNull("failed to connect daemon", daemon)
                     val jar = tmpdir.absolutePath + File.separator + "hello.$threadNo.jar"
@@ -423,34 +435,38 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
                             daemon!!,
                             CompileService.NO_SESSION,
                             CompileService.TargetPlatform.JVM,
-                            arrayOf("-include-runtime", File(getHelloAppBaseDir(), "hello.kt").absolutePath, "-d", jar),
+                            arrayOf(File(getHelloAppBaseDir(), "hello.kt").absolutePath, "-d", jar),
                             outStreams[threadNo])
-                    daemon.getClients().get().let { clients[threadNo].addAll(it) }
                     resultCodes[threadNo] = res
+                    logFiles[threadNo] = logFile
                 }
-                finally {
+            }
-                    doneLatch.countDown()
+            finally {
-                }
+                doneLatch.countDown()
             }
         }
 
         (1..PARALLEL_THREADS_TO_START).forEach { connectThread(it - 1) }
 
-        startLatch.countDown()
+        val succeeded = doneLatch.await(PARALLEL_WAIT_TIMEOUT_S * 2, TimeUnit.SECONDS)
-
-        val succeeded = doneLatch.await(PARALLEL_WAIT_TIMEOUT_S, TimeUnit.SECONDS)
         assertTrue("parallel daemons start failed to complete in $PARALLEL_WAIT_TIMEOUT_S s, ${doneLatch.count} unfinished threads", succeeded)
 
+        Thread.sleep(100) // Wait for processes to finish and close log files
+
+        val electionLogs = logFiles.map { it to it?.readLines()?.find { it.contains("Assuming other daemons have") } }
+
+        assertTrue("No daemon elected: \n${electionLogs.joinToString("\n")}", electionLogs.any { it.second?.let { it.contains("lower prio") || it.contains("equal prio") } ?: false })
+
+        electionLogs.forEach { it.first?.delete() }
+
         (2..PARALLEL_THREADS_TO_START).forEach {
-            assertEquals("daemon ${it - 1} has ${clients[it - 1].size} clients, while first daemon has ${clients[0].size}", clients[0].size, clients[it - 1].size)
-            KtUsefulTestCase.assertSameElements(clients[0], clients[it - 1])
             assertEquals("Compilation on thread $it failed:\n${outStreams[it - 1]}", 0, resultCodes[it - 1])
         }
     }
 
     fun testDaemonConnectionProblems() {
         withFlagFile(getTestName(true), ".alive") { flagFile ->
-            val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
+            val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
             KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions)
 
             val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false)
@@ -481,7 +497,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
 
     fun testDaemonCallbackConnectionProblems() {
         withFlagFile(getTestName(true), ".alive") { flagFile ->
-            val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
+            val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
             KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions)
 
             val logFile = createTempFile("kotlin-daemon-test", ".log")
@@ -617,7 +633,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
 
     fun testDaemonReplAutoshutdownOnIdle() {
         withFlagFile(getTestName(true), ".alive") { flagFile ->
-            val daemonOptions = DaemonOptions(autoshutdownIdleSeconds = 1, autoshutdownUnusedSeconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
+            val daemonOptions = DaemonOptions(autoshutdownIdleSeconds = 1, autoshutdownUnusedSeconds = 1, shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
             KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions)
 
             val logFile = createTempFile("kotlin-daemon-test", ".log")
@@ -658,7 +674,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
 
     internal fun withDaemon(body: (CompileService) -> Unit) {
         withFlagFile(getTestName(true), ".alive") { flagFile ->
-            val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
+            val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
             val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false)
             val daemon: CompileService? = KotlinCompilerClient.connectToCompileService(compilerId, flagFile, daemonJVMOptions, daemonOptions, DaemonReportingTargets(out = System.err), autostart = true)
             assertNotNull("failed to connect daemon", daemon)