From cb7de0e26208a5f39e521a8ca3a2ae571d8acb75 Mon Sep 17 00:00:00 2001 From: Ilya Chernikov Date: Wed, 8 Feb 2017 20:49:23 +0100 Subject: [PATCH] Implement more reliable daemon election, fixes KT-15562 also making shutdown more reliable --- .../daemon/client/KotlinCompilerClient.kt | 20 ++++-- .../kotlin/daemon/common/ClientUtils.kt | 54 +++++++++++--- .../kotlin/daemon/CompileServiceImpl.kt | 71 ++++++++++-------- .../kotlin/daemon/KotlinCompileDaemon.kt | 10 +-- .../kotlin/daemon/CompilerDaemonTest.kt | 72 +++++++++++-------- 5 files changed, 146 insertions(+), 81 deletions(-) diff --git a/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/KotlinCompilerClient.kt b/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/KotlinCompilerClient.kt index 0574a707599..d11dca0dc55 100644 --- a/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/KotlinCompilerClient.kt +++ b/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/KotlinCompilerClient.kt @@ -291,17 +291,23 @@ object KotlinCompilerClient { } private fun tryFindSuitableDaemonOrNewOpts(registryDir: File, compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, report: (DaemonReportCategory, String) -> Unit): Pair { - val aliveWithOpts = walkDaemons(registryDir, compilerId, report = report) - .map { Pair(it, it.getDaemonJVMOptions()) } - .filter { it.second.isGood } - .sortedWith(compareByDescending(DaemonJVMOptionsMemoryComparator(), { it.second.get() })) + registryDir.mkdirs() + val timestampMarker = createTempFile("kotlin-daemon-client-tsmarker", directory = registryDir) + val aliveWithMetadata = try { + walkDaemons(registryDir, compilerId, timestampMarker, report = report).toList() + } + finally { + timestampMarker.delete() + } + val comparator = compareByDescending(DaemonJVMOptionsMemoryComparator(), { it.jvmOptions }) + .thenBy(FileAgeComparator()) { it.runFile } val optsCopy = daemonJVMOptions.copy() // if required options fit into fattest running daemon - return the daemon and required options with memory params set to actual ones in the daemon - return aliveWithOpts.firstOrNull()?.check { daemonJVMOptions memorywiseFitsInto it.second.get() }?.let { - Pair(it.first, optsCopy.updateMemoryUpperBounds(it.second.get())) + return aliveWithMetadata.maxWith(comparator)?.takeIf { daemonJVMOptions memorywiseFitsInto it.jvmOptions }?.let { + Pair(it.daemon, optsCopy.updateMemoryUpperBounds(it.jvmOptions)) } // else combine all options from running daemon to get fattest option for a new daemon to run - ?: Pair(null, aliveWithOpts.fold(optsCopy, { opts, d -> opts.updateMemoryUpperBounds(d.second.get()) })) + ?: Pair(null, aliveWithMetadata.fold(optsCopy, { opts, d -> opts.updateMemoryUpperBounds(d.jvmOptions) })) } diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ClientUtils.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ClientUtils.kt index a02c1bab172..4c32737e199 100644 --- a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ClientUtils.kt +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ClientUtils.kt @@ -40,30 +40,48 @@ fun makePortFromRunFilenameExtractor(digest: String): (String) -> Int? { } } +private const val ORPHANED_RUN_FILE_AGE_THRESHOLD_MS = 1000000L + +data class DaemonWithMetadata(val daemon: CompileService, val runFile: File, val jvmOptions: DaemonJVMOptions) fun walkDaemons(registryDir: File, compilerId: CompilerId, - filter: (File, Int) -> Boolean = { f, p -> true }, - report: (DaemonReportCategory, String) -> Unit = { cat, msg -> } -): Sequence { + fileToCompareTimestamp: File, + filter: (File, Int) -> Boolean = { _, _ -> true }, + report: (DaemonReportCategory, String) -> Unit = { _, _ -> } +): Sequence { val classPathDigest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest().toHexString() val portExtractor = makePortFromRunFilenameExtractor(classPathDigest) return registryDir.walk() .map { Pair(it, portExtractor(it.name)) } .filter { it.second != null && filter(it.first, it.second!!) } - .mapNotNull { - assert(it.second!! > 0 && it.second!! < MAX_PORT_NUMBER) - report(DaemonReportCategory.DEBUG, "found daemon on port ${it.second}, trying to connect") - val daemon = tryConnectToDaemon(it.second!!, report) + .mapNotNull { fileWithPort -> + assert(fileWithPort.second!! in 1..(MAX_PORT_NUMBER - 1)) + val relativeAge = fileToCompareTimestamp.lastModified() - fileWithPort.first.lastModified() + report(DaemonReportCategory.DEBUG, "found daemon on port ${fileWithPort.second} ($relativeAge ms old), trying to connect") + val daemon = tryConnectToDaemon(fileWithPort.second!!, report) // cleaning orphaned file; note: daemon should shut itself down if it detects that the run file is deleted - if (daemon == null && !it.first.delete()) { - report(DaemonReportCategory.INFO, "WARNING: unable to delete seemingly orphaned file '${it.first.absolutePath}', cleanup recommended") + if (daemon == null) { + if (relativeAge - ORPHANED_RUN_FILE_AGE_THRESHOLD_MS <= 0) { + report(DaemonReportCategory.DEBUG, "found fresh run file '${fileWithPort.first.absolutePath}' ($relativeAge ms old), but no daemon, ignoring it") + } + else { + report(DaemonReportCategory.DEBUG, "found seemingly orphaned run file '${fileWithPort.first.absolutePath}' ($relativeAge ms old), deleting it") + if (!fileWithPort.first.delete()) { + report(DaemonReportCategory.INFO, "WARNING: unable to delete seemingly orphaned file '${fileWithPort.first.absolutePath}', cleanup recommended") + } + } + } + try { + daemon?.let { DaemonWithMetadata(it, fileWithPort.first, it.getDaemonJVMOptions().get()) } + } + catch (e: Exception) { + report(DaemonReportCategory.INFO, "ERROR: unable to retrieve daemon JVM options, assuming daemon is dead: ${e.message}") + null } - daemon } } - private inline fun tryConnectToDaemon(port: Int, report: (DaemonReportCategory, String) -> Unit): CompileService? { try { @@ -88,3 +106,17 @@ fun makeAutodeletingFlagFile(keyword: String = "compiler-client", baseDir: File? flagFile.deleteOnExit() return flagFile } + +// Comparator for reliable choice between daemons +class FileAgeComparator : Comparator { + override fun compare(left: File, right: File): Int { + val leftTS = left.lastModified() + val rightTS = right.lastModified() + return when { + leftTS == 0L || rightTS == 0L -> 0 // cannot read any file timestamp, => undecidable + leftTS > rightTS -> -1 + leftTS < rightTS -> 1 + else -> compareValues(left.canonicalPath, right.canonicalPath) + } + } +} diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt index b195caeb7bd..d0e3933a789 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt @@ -267,20 +267,19 @@ class CompileServiceImpl( ifAlive { CompileService.CallResult.Good(usedMemory(withGC = true)) } override fun shutdown(): CompileService.CallResult = ifAliveExclusive(minAliveness = Aliveness.LastSession, ignoreCompilerChanged = true) { - shutdownImpl() + shutdownWithDelay() CompileService.CallResult.Ok() } override fun scheduleShutdown(graceful: Boolean): CompileService.CallResult = ifAlive(minAliveness = Aliveness.Alive) { CompileService.CallResult.Good( if (!graceful || state.alive.compareAndSet(Aliveness.Alive.ordinal, Aliveness.LastSession.ordinal)) { - timer.schedule(0) { + timer.schedule(1) { ifAliveExclusive(minAliveness = Aliveness.LastSession, ignoreCompilerChanged = true) { - if (!graceful || state.sessions.isEmpty()) { - shutdownImpl() - } - else { - log.info("Some sessions are active, waiting for them to finish") + when { + !graceful -> shutdownImpl() + state.sessions.isEmpty() -> shutdownWithDelay() + else -> log.info("Some sessions are active, waiting for them to finish") } CompileService.CallResult.Ok() } @@ -529,7 +528,7 @@ class CompileServiceImpl( val stub = UnicastRemoteObject.exportObject(this, port, LoopbackNetworkInterface.clientLoopbackSocketFactory, LoopbackNetworkInterface.serverLoopbackSocketFactory) as CompileService registry.rebind (COMPILER_SERVICE_RMI_NAME, stub) - timer.schedule(0) { + timer.schedule(100) { initiateElections() } timer.schedule(delay = DAEMON_PERIODIC_CHECK_INTERVAL_MS, period = DAEMON_PERIODIC_CHECK_INTERVAL_MS) { @@ -552,7 +551,7 @@ class CompileServiceImpl( // 1. check if unused for a timeout - shutdown if (shutdownCondition({ daemonOptions.autoshutdownUnusedSeconds != COMPILE_DAEMON_TIMEOUT_INFINITE_S && compilationsCounter.get() == 0 && nowSeconds() - lastUsedSeconds > daemonOptions.autoshutdownUnusedSeconds }, "Unused timeout exceeded ${daemonOptions.autoshutdownUnusedSeconds}s, shutting down")) { - shutdown() + scheduleShutdown(true) } else { var anyDead = state.sessions.cleanDead() @@ -561,7 +560,7 @@ class CompileServiceImpl( // 3. check if in graceful shutdown state and all sessions are closed if (shutdownCondition({ state.alive.get() == Aliveness.LastSession.ordinal && state.sessions.isEmpty() }, "All sessions finished, shutting down")) { - shutdown() + shutdownWithDelay() shuttingDown = true } @@ -587,7 +586,7 @@ class CompileServiceImpl( // TODO: could be too expensive anyway, consider removing this check shutdownCondition({ classpathWatcher.isChanged }, "Compiler changed")) { - shutdown() + scheduleShutdown(true) shuttingDown = true } @@ -604,35 +603,44 @@ class CompileServiceImpl( ifAlive { - val aliveWithOpts = walkDaemons(File(daemonOptions.runFilesPathOrDefault), compilerId, filter = { f, p -> p != port }, report = { lvl, msg -> log.info(msg) }) - .map { Pair(it, it.getDaemonJVMOptions()) } - .filter { it.second.isGood } - .sortedWith(compareByDescending(DaemonJVMOptionsMemoryComparator(), { it.second.get() })) - if (aliveWithOpts.any()) { - val fattestOpts = aliveWithOpts.first().second.get() + val aliveWithOpts = walkDaemons(File(daemonOptions.runFilesPathOrDefault), compilerId, runFile, filter = { f, p -> p != port }, report = { _, msg -> log.info(msg) }).toList() + val comparator = compareByDescending(DaemonJVMOptionsMemoryComparator(), { it.jvmOptions }) + .thenBy(FileAgeComparator()) { it.runFile } + aliveWithOpts.maxWith(comparator)?.let { bestDaemonWithMetadata -> + val fattestOpts = bestDaemonWithMetadata.jvmOptions // second part of the condition means that we prefer other daemon if is "equal" to the current one - if (fattestOpts memorywiseFitsInto daemonJVMOptions && !(daemonJVMOptions memorywiseFitsInto fattestOpts)) { + if (fattestOpts memorywiseFitsInto daemonJVMOptions && FileAgeComparator().compare(bestDaemonWithMetadata.runFile, runFile) < 0 ) { // all others are smaller that me, take overs' clients and shut them down + log.info("Assuming other daemons have lower prio, taking clients from them and schedule them to shutdown: my runfile: ${runFile.name} (${runFile.lastModified()}) vs best other runfile: ${bestDaemonWithMetadata.runFile.name} (${bestDaemonWithMetadata.runFile.lastModified()})") aliveWithOpts.forEach { - it.first.getClients().check { it.isGood }?.let { - it.get().forEach { registerClient(it) } + try { + it.daemon.getClients().check { it.isGood }?.let { + it.get().forEach { registerClient(it) } + } + it.daemon.scheduleShutdown(true) + } + catch (e: Exception) { + log.info("Cannot connect to a daemon, assuming dying ('${it.runFile.canonicalPath}'): ${e.message}") } - it.first.scheduleShutdown(true) } } - else if (daemonJVMOptions memorywiseFitsInto fattestOpts) { + else if (daemonJVMOptions memorywiseFitsInto fattestOpts && FileAgeComparator().compare(bestDaemonWithMetadata.runFile, runFile) > 0) { // there is at least one bigger, handover my clients to it and shutdown + log.info("Assuming other daemons have higher prio, handover clients to it and schedule shutdown: my runfile: ${runFile.name} (${runFile.lastModified()}) vs best other runfile: ${bestDaemonWithMetadata.runFile.name} (${bestDaemonWithMetadata.runFile.lastModified()})") scheduleShutdown(true) - aliveWithOpts.first().first.let { fattest -> + aliveWithOpts.first().daemon.let { fattest -> getClients().check { it.isGood }?.let { it.get().forEach { fattest.registerClient(it) } } } } - // else - do nothing, all daemons are staying - // TODO: implement some behaviour here, e.g.: - // - shutdown/takeover smaller daemon - // - run (or better persuade client to run) a bigger daemon (in fact may be even simple shutdown will do, because of client's daemon choosing logic) + else { + // undecided, do nothing + log.info("Assuming other daemon(s) have equal prio, continue: ${runFile.name} (${runFile.lastModified()}) vs best other runfile: ${bestDaemonWithMetadata.runFile.name} (${bestDaemonWithMetadata.runFile.lastModified()})") + // TODO: implement some behaviour here, e.g.: + // - shutdown/takeover smaller daemon + // - run (or better persuade client to run) a bigger daemon (in fact may be even simple shutdown will do, because of client's daemon choosing logic) + } } CompileService.CallResult.Ok() } @@ -653,8 +661,11 @@ class CompileServiceImpl( timer.schedule(daemonOptions.shutdownDelayMilliseconds) { state.delayedShutdownQueued.set(false) if (currentCompilationsCount == compilationsCounter.get()) { - log.fine("Execute delayed shutdown") - shutdown() + ifAliveExclusive(minAliveness = Aliveness.LastSession, ignoreCompilerChanged = true) { + log.fine("Execute delayed shutdown") + shutdownImpl() + CompileService.CallResult.Ok() + } } else { log.info("Cancel delayed shutdown due to new client") @@ -801,7 +812,7 @@ class CompileServiceImpl( state.alive.get() < minAliveness.ordinal -> CompileService.CallResult.Dying() !ignoreCompilerChanged && classpathWatcher.isChanged -> { log.info("Compiler changed, scheduling shutdown") - timer.schedule(0) { shutdown() } + shutdownWithDelay() CompileService.CallResult.Dying() } else -> { diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinCompileDaemon.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinCompileDaemon.kt index eb20b6b788a..9d7413d6416 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinCompileDaemon.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinCompileDaemon.kt @@ -30,9 +30,7 @@ import java.net.URLClassLoader import java.text.SimpleDateFormat import java.util.* import java.util.jar.Manifest -import java.util.logging.Level -import java.util.logging.LogManager -import java.util.logging.Logger +import java.util.logging.* import kotlin.concurrent.schedule val DAEMON_PERIODIC_CHECK_INTERVAL_MS = 1000L @@ -55,7 +53,6 @@ class LogStream(name: String) : OutputStream() { } } - object KotlinCompileDaemon { init { @@ -71,7 +68,7 @@ object KotlinCompileDaemon { "java.util.logging.FileHandler.count = ${if (fileIsGiven) 1 else 3}\n" + "java.util.logging.FileHandler.append = $fileIsGiven\n" + "java.util.logging.FileHandler.pattern = ${if (fileIsGiven) logPath else (logPath + File.separator + "${COMPILE_DAEMON_DEFAULT_FILES_PREFIX}.$logTime.%u%g.log")}\n" + - "java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s\\n\n" + "java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s%n\n" LogManager.getLogManager().readConfiguration(cfg.byteInputStream()) } @@ -96,6 +93,7 @@ object KotlinCompileDaemon { log.info("Kotlin compiler daemon version " + (loadVersionFromResource() ?: "")) log.info("daemon JVM args: " + ManagementFactory.getRuntimeMXBean().inputArguments.joinToString(" ")) log.info("daemon args: " + args.joinToString(" ")) + log.info("daemon process name: " + ManagementFactory.getRuntimeMXBean().name) val compilerId = CompilerId() val daemonOptions = DaemonOptions() @@ -159,6 +157,8 @@ object KotlinCompileDaemon { if (daemonOptions.runFilesPath.isNotEmpty()) println(daemonOptions.runFilesPath) + log.info("daemon is listening on port: $port") + // this supposed to stop redirected streams reader(s) on the client side and prevent some situations with hanging threads, but doesn't work reliably // TODO: implement more reliable scheme System.out.close() diff --git a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt index fa2ef49d2d2..5e6a3236ea1 100644 --- a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt +++ b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt @@ -73,10 +73,20 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { private fun run(logName: String, vararg args: String): Int = runJava(getTestBaseDir(), logName, *args) + override fun setUp() { + super.setUp() + System.setProperty("COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY", "true") + } + + override fun tearDown() { + System.clearProperty("COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY") + super.tearDown() + } fun testHelloApp() { withFlagFile(getTestName(true), ".alive") { flagFile -> val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath, + shutdownDelayMilliseconds = 1, verbose = true, reportPerf = true) @@ -141,7 +151,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { val backupOptions = System.getProperty(COMPILE_DAEMON_OPTIONS_PROPERTY) try { System.setProperty(COMPILE_DAEMON_OPTIONS_PROPERTY, "runFilesPath=abcd,autoshutdownIdleSeconds=1111") - val opts = configureDaemonOptions() + val opts = configureDaemonOptions(DaemonOptions(shutdownDelayMilliseconds = 1)) assertEquals("abcd", opts.runFilesPath) assertEquals(1111, opts.autoshutdownIdleSeconds) } @@ -152,7 +162,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { fun testDaemonInstancesSimple() { withFlagFile(getTestName(true), ".alive") { flagFile -> - val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath) + val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) val compilerId2 = CompilerId.makeCompilerId(compilerClassPath + File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-compiler-sources.jar")) @@ -196,7 +206,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { fun testDaemonAutoshutdownOnUnused() { withFlagFile(getTestName(true), ".alive") { flagFile -> - val daemonOptions = DaemonOptions(autoshutdownUnusedSeconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) + val daemonOptions = DaemonOptions(autoshutdownUnusedSeconds = 1, shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) val logFile = createTempFile("kotlin-daemon-test", ".log") @@ -223,7 +233,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { fun testDaemonAutoshutdownOnIdle() { withFlagFile(getTestName(true), ".alive") { flagFile -> - val daemonOptions = DaemonOptions(autoshutdownIdleSeconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) + val daemonOptions = DaemonOptions(autoshutdownIdleSeconds = 1, shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) val logFile = createTempFile("kotlin-daemon-test", ".log") @@ -255,7 +265,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { fun testDaemonGracefulShutdown() { withFlagFile(getTestName(true), ".alive") { flagFile -> - val daemonOptions = DaemonOptions(autoshutdownIdleSeconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) + val daemonOptions = DaemonOptions(autoshutdownIdleSeconds = 1, shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) val logFile = createTempFile("kotlin-daemon-test", ".log") @@ -303,7 +313,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { fun testDaemonExecutionViaIntermediateProcess() { val clientAliveFile = createTempFile("kotlin-daemon-transitive-run-test", ".run") val runFilesPath = File(tmpdir, getTestName(true)).absolutePath - val daemonOptions = DaemonOptions(runFilesPath = runFilesPath) + val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, runFilesPath = runFilesPath) val jar = tmpdir.absolutePath + File.separator + "hello.jar" val args = listOf( File(File(System.getProperty("java.home"), "bin"), "java").absolutePath, @@ -357,7 +367,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { assertTrue(PARALLEL_THREADS_TO_COMPILE <= LoopbackNetworkInterface.SERVER_SOCKET_BACKLOG_SIZE) withFlagFile(getTestName(true), ".alive") { flagFile -> - val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath) + val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritAdditionalProperties = false) val daemon = KotlinCompilerClient.connectToCompileService(compilerId, flagFile, daemonJVMOptions, daemonOptions, DaemonReportingTargets(out = System.err), autostart = true) assertNotNull("failed to connect daemon", daemon) @@ -406,16 +416,18 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { val startLatch = CountDownLatch(1) val doneLatch = CountDownLatch(PARALLEL_THREADS_TO_START) - val clients = Array(PARALLEL_THREADS_TO_START, { hashSetOf() } ) val resultCodes = arrayOfNulls(PARALLEL_THREADS_TO_START) val outStreams = Array(PARALLEL_THREADS_TO_START, { ByteArrayOutputStream() }) + val logFiles = kotlin.arrayOfNulls(PARALLEL_THREADS_TO_START) - fun connectThread(threadNo: Int) = thread { - withFlagFile(getTestName(true), ".alive") { flagFile -> - val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath) - val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritAdditionalProperties = false) - startLatch.await() - try { + fun connectThread(threadNo: Int) = thread(name = "daemonConnect$threadNo") { + try { + withFlagFile(getTestName(true), ".alive") { flagFile -> + val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1000, runFilesPath = File(tmpdir, getTestName(true)).absolutePath, verbose = true) + val logFile = createTempFile("kotlin-daemon-test", ".log") + val daemonJVMOptions = + configureDaemonJVMOptions("D$COMPILE_DAEMON_LOG_PATH_PROPERTY=\"${logFile.loggerCompatiblePath}\"", + inheritMemoryLimits = false, inheritAdditionalProperties = false) val daemon = KotlinCompilerClient.connectToCompileService(compilerId, flagFile, daemonJVMOptions, daemonOptions, DaemonReportingTargets(out = System.err), autostart = true) assertNotNull("failed to connect daemon", daemon) val jar = tmpdir.absolutePath + File.separator + "hello.$threadNo.jar" @@ -423,34 +435,38 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { daemon!!, CompileService.NO_SESSION, CompileService.TargetPlatform.JVM, - arrayOf("-include-runtime", File(getHelloAppBaseDir(), "hello.kt").absolutePath, "-d", jar), + arrayOf(File(getHelloAppBaseDir(), "hello.kt").absolutePath, "-d", jar), outStreams[threadNo]) - daemon.getClients().get().let { clients[threadNo].addAll(it) } resultCodes[threadNo] = res + logFiles[threadNo] = logFile } - finally { - doneLatch.countDown() - } + } + finally { + doneLatch.countDown() } } (1..PARALLEL_THREADS_TO_START).forEach { connectThread(it - 1) } - startLatch.countDown() - - val succeeded = doneLatch.await(PARALLEL_WAIT_TIMEOUT_S, TimeUnit.SECONDS) + val succeeded = doneLatch.await(PARALLEL_WAIT_TIMEOUT_S * 2, TimeUnit.SECONDS) assertTrue("parallel daemons start failed to complete in $PARALLEL_WAIT_TIMEOUT_S s, ${doneLatch.count} unfinished threads", succeeded) + Thread.sleep(100) // Wait for processes to finish and close log files + + val electionLogs = logFiles.map { it to it?.readLines()?.find { it.contains("Assuming other daemons have") } } + + assertTrue("No daemon elected: \n${electionLogs.joinToString("\n")}", electionLogs.any { it.second?.let { it.contains("lower prio") || it.contains("equal prio") } ?: false }) + + electionLogs.forEach { it.first?.delete() } + (2..PARALLEL_THREADS_TO_START).forEach { - assertEquals("daemon ${it - 1} has ${clients[it - 1].size} clients, while first daemon has ${clients[0].size}", clients[0].size, clients[it - 1].size) - KtUsefulTestCase.assertSameElements(clients[0], clients[it - 1]) assertEquals("Compilation on thread $it failed:\n${outStreams[it - 1]}", 0, resultCodes[it - 1]) } } fun testDaemonConnectionProblems() { withFlagFile(getTestName(true), ".alive") { flagFile -> - val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath) + val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false) @@ -481,7 +497,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { fun testDaemonCallbackConnectionProblems() { withFlagFile(getTestName(true), ".alive") { flagFile -> - val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath) + val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) val logFile = createTempFile("kotlin-daemon-test", ".log") @@ -617,7 +633,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { fun testDaemonReplAutoshutdownOnIdle() { withFlagFile(getTestName(true), ".alive") { flagFile -> - val daemonOptions = DaemonOptions(autoshutdownIdleSeconds = 1, autoshutdownUnusedSeconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) + val daemonOptions = DaemonOptions(autoshutdownIdleSeconds = 1, autoshutdownUnusedSeconds = 1, shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) val logFile = createTempFile("kotlin-daemon-test", ".log") @@ -658,7 +674,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { internal fun withDaemon(body: (CompileService) -> Unit) { withFlagFile(getTestName(true), ".alive") { flagFile -> - val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath) + val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false) val daemon: CompileService? = KotlinCompilerClient.connectToCompileService(compilerId, flagFile, daemonJVMOptions, daemonOptions, DaemonReportingTargets(out = System.err), autostart = true) assertNotNull("failed to connect daemon", daemon)