From 2c35675fd653c3907b25709754711e0ba5803b9d Mon Sep 17 00:00:00 2001 From: Ilya Chernikov Date: Fri, 23 Jun 2017 18:01:40 +0200 Subject: [PATCH] Refactor daemon parallel start test to improve diagnostics also add daemon info to the RPC and remove redundant daemon stopping calls, that should speed up tests execution --- .../kotlin/daemon/common/CompileService.kt | 3 + .../kotlin/daemon/CompileServiceImpl.kt | 4 + .../kotlin/daemon/CompilerDaemonTest.kt | 85 ++++++++++--------- 3 files changed, 54 insertions(+), 38 deletions(-) diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileService.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileService.kt index 148d16b0ac0..a3a4482840e 100644 --- a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileService.kt +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileService.kt @@ -77,6 +77,9 @@ interface CompileService : Remote { @Throws(RemoteException::class) fun getDaemonOptions(): CallResult + @Throws(RemoteException::class) + fun getDaemonInfo(): CallResult + @Throws(RemoteException::class) fun getDaemonJVMOptions(): CallResult diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt index 103c6ba9a69..e523bf054be 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt @@ -206,6 +206,10 @@ class CompileServiceImpl( // RMI-exposed API + override fun getDaemonInfo(): CompileService.CallResult = ifAlive(minAliveness = Aliveness.Dying) { + CompileService.CallResult.Good("Kotlin daemon on port $port") + } + override fun getDaemonOptions(): CompileService.CallResult = ifAlive { CompileService.CallResult.Good(daemonOptions) } diff --git a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt index 0938023d1ea..f887acbabdb 100644 --- a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt +++ b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt @@ -32,6 +32,7 @@ import java.io.PrintStream import java.lang.management.ManagementFactory import java.net.URL import java.net.URLClassLoader +import java.nio.charset.Charset import java.rmi.server.UnicastRemoteObject import java.util.concurrent.CountDownLatch import java.util.concurrent.Future @@ -105,8 +106,6 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { withFlagFile(getTestName(true), ".alive") { flagFile -> val daemonOptions = makeTestDaemonOptions(getTestName(true)) - KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) - withLogFile("kotlin-daemon-test") { logFile -> val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) var daemonShotDown = false @@ -190,9 +189,6 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { val compilerId2 = CompilerId.makeCompilerId(compilerClassPath + File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-compiler-sources.jar")) - KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) - KotlinCompilerClient.shutdownCompileService(compilerId2, daemonOptions) - withLogFile("kotlin-daemon1-test") { logFile1 -> withLogFile("kotlin-daemon2-test") { logFile2 -> val daemonJVMOptions1 = makeTestDaemonJvmOptions(logFile1) @@ -229,8 +225,6 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { withFlagFile(getTestName(true), ".alive") { flagFile -> val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, verbose = true, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) - KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) - val daemonJVMOptions = configureDaemonJVMOptions("-abracadabra", inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false) val messageCollector = TestMessageCollector() @@ -249,8 +243,6 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { withFlagFile(getTestName(true), ".alive") { flagFile -> val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, verbose = true, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) - KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) - val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false) val messageCollector = TestMessageCollector() @@ -270,7 +262,6 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { fun testDaemonAutoshutdownOnUnused() { withFlagFile(getTestName(true), ".alive") { flagFile -> val daemonOptions = DaemonOptions(autoshutdownUnusedSeconds = 1, shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) - KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) withLogFile("kotlin-daemon-test") { logFile -> val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) @@ -295,7 +286,6 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { fun testDaemonAutoshutdownOnIdle() { withFlagFile(getTestName(true), ".alive") { flagFile -> val daemonOptions = DaemonOptions(autoshutdownIdleSeconds = 1, shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) - KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) withLogFile("kotlin-daemon-test") { logFile -> val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) @@ -328,7 +318,6 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { fun testDaemonGracefulShutdown() { withFlagFile(getTestName(true), ".alive") { flagFile -> val daemonOptions = DaemonOptions(autoshutdownIdleSeconds = 1, shutdownDelayMilliseconds = 1, forceShutdownTimeoutMilliseconds = 60000, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) - KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) withLogFile("kotlin-daemon-test") { logFile -> val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) @@ -386,7 +375,6 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { File(getHelloAppBaseDir(), "hello.kt").absolutePath + "-d" + jar try { - KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) var resOutput: String? = null var resCode: Int? = null // running intermediate process (daemon command line controller) that executes the daemon @@ -408,17 +396,6 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { } } - private class SynchronizationTracer(val startSignal: CountDownLatch, val doneSignal: CountDownLatch, port: Int) : RemoteOperationsTracer, - java.rmi.server.UnicastRemoteObject(port, LoopbackNetworkInterface.clientLoopbackSocketFactory, LoopbackNetworkInterface.serverLoopbackSocketFactory) - { - override fun before(id: String) { - startSignal.await() - } - override fun after(id: String) { - doneSignal.countDown() - } - } - private val PARALLEL_THREADS_TO_COMPILE = 10 private val PARALLEL_WAIT_TIMEOUT_S = 60L @@ -477,20 +454,22 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { val resultCodes = arrayOfNulls(PARALLEL_THREADS_TO_START) val outStreams = Array(PARALLEL_THREADS_TO_START, { ByteArrayOutputStream() }) val logFiles = arrayOfNulls(PARALLEL_THREADS_TO_START) + val daemonInfos = arrayOfNulls?, Int?>>(PARALLEL_THREADS_TO_START) + val daemonOptions = makeTestDaemonOptions(getTestName(true), 100) - KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) fun connectThread(threadNo: Int) = thread(name = "daemonConnect$threadNo") { try { withFlagFile(getTestName(true), ".alive") { flagFile -> - withLogFile("kotlin-daemon-test") { logFile -> + withLogFile("kotlin-daemon-test", printLogOnException = false) { logFile -> logFiles[threadNo] = logFile val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) val compileServiceSession = KotlinCompilerClient.connectAndLease(compilerId, flagFile, daemonJVMOptions, daemonOptions, - DaemonReportingTargets(out = System.err), autostart = true, + DaemonReportingTargets(out = PrintStream(outStreams[threadNo])), autostart = true, leaseSession = true) assertNotNull("failed to connect daemon", compileServiceSession?.compileService) + daemonInfos[threadNo] = compileServiceSession?.compileService?.getDaemonInfo() to compileServiceSession?.sessionId val jar = tmpdir.absolutePath + File.separator + "hello.$threadNo.jar" val res = KotlinCompilerClient.compile( compileServiceSession!!.compileService, @@ -524,20 +503,49 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { Thread.sleep(100) // Wait for processes to finish and close log files - val electionLogs = (0..(PARALLEL_THREADS_TO_START - 1)).map { - val logContents = logFiles[it]?.readLines() - logContents?.find { it.contains(LOG_PREFIX_ASSUMING_OTHER_DAEMONS_HAVE) } + val electionLogs = arrayOfNulls(PARALLEL_THREADS_TO_START) + val port2logs = arrayOfNulls>(PARALLEL_THREADS_TO_START) + + for (i in 0..(PARALLEL_THREADS_TO_START - 1)) { + val logContents = logFiles[i]?.readLines() + port2logs[i] = logContents?.find { it.contains("daemon is listening on port") }?.split(" ")?.last()?.toIntOrNull() to logFiles[i] + electionLogs[i] = logContents?.find { it.contains(LOG_PREFIX_ASSUMING_OTHER_DAEMONS_HAVE) } } - assertTrue("No daemon elected: \n${electionLogs.joinToString("\n")}", - electionLogs.any { it != null && (it.contains("lower prio") || it.contains("equal prio")) }) + val electionsSuccess = electionLogs.any { it != null && (it.contains("lower prio") || it.contains("equal prio")) } + val resultsSuccess = resultCodes.all { it == 0 } + + if (!electionsSuccess || !resultsSuccess) { + val msg = buildString { + for (i in 0..PARALLEL_THREADS_TO_START - 1) { + val daemonInfoRes = daemonInfos[i]?.first + var compiledPort: Int? = null + val daemonInfo = when (daemonInfoRes) { + is CompileService.CallResult.Good -> daemonInfoRes.get().also { + compiledPort = split(" ").last().toIntOrNull() + } + is CompileService.CallResult.Dying -> "" + is CompileService.CallResult.Error -> "" + else -> "?" + } + appendln("#$i\tcompiled on $daemonInfo, session ${daemonInfos[i]?.second}, result ${resultCodes[i]}; started daemon on port ${port2logs[i]?.first}, log: ${logFiles[i]?.canonicalPath}") + if (resultCodes[i] != 0 || electionLogs[i] == null) { + appendln("--- out $i, result ${resultCodes[i]}:\n${outStreams[i].toByteArray().toString(Charset.defaultCharset())}\n---") + compiledPort?.let { port -> port2logs.find { it?.first == port } }?.second?.let { logFile -> + appendln("--- log file ${logFile.name}:\n${logFile.readText()}\n---") + } + ?: appendln("--- log not found") + } + } + } + assertTrue("No daemon elected:\n\n$msg\n--- elections:\n${electionLogs.joinToString("\n")}\n---", electionsSuccess) + assertTrue("Some compilations failed:\n\nmsg", resultsSuccess) + } } fun testDaemonConnectionProblems() { withFlagFile(getTestName(true), ".alive") { flagFile -> val daemonOptions = makeTestDaemonOptions(getTestName(true)) - KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) - val daemonJVMOptions = makeTestDaemonJvmOptions() val daemon = KotlinCompilerClient.connectToCompileService(compilerId, flagFile, daemonJVMOptions, daemonOptions, DaemonReportingTargets(out = System.err), autostart = true) assertNotNull("failed to connect daemon", daemon) @@ -567,7 +575,6 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { fun testDaemonCallbackConnectionProblems() { withFlagFile(getTestName(true), ".alive") { flagFile -> val daemonOptions = makeTestDaemonOptions(getTestName(true)) - KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) withLogFile("kotlin-daemon-test") { logFile -> val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) @@ -670,7 +677,6 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { fun testDaemonReplAutoshutdownOnIdle() { withFlagFile(getTestName(true), ".alive") { flagFile -> val daemonOptions = DaemonOptions(autoshutdownIdleSeconds = 1, autoshutdownUnusedSeconds = 1, shutdownDelayMilliseconds = 1, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) - KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) withLogFile("kotlin-daemon-test") { logFile -> val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) @@ -791,13 +797,16 @@ internal inline fun withFlagFile(prefix: String, suffix: String? = null, body: ( } } -internal inline fun withLogFile(prefix: String, suffix: String = ".log", body: (File) -> Unit) { +internal inline fun withLogFile(prefix: String, suffix: String = ".log", printLogOnException: Boolean = true, body: (File) -> Unit) { val logFile = createTempFile(prefix, suffix) try { body(logFile) } catch (e: Exception) { - System.err.println("--- log file ${logFile.name}:\n${logFile.readText()}\n---") + if (printLogOnException) { + Thread.sleep(100) // trying to wait log flushing + System.err.println("--- log file ${logFile.name}:\n${logFile.readText()}\n---") + } throw e } }