From 9d2ae54d2c08e5ff30ec98ef43c04e8c2dfe7aba Mon Sep 17 00:00:00 2001 From: Ilya Chernikov Date: Thu, 4 May 2017 23:15:44 +0200 Subject: [PATCH] Introduce other JVM options inheritance for daemon --- .../compilerRunner/KotlinCompilerRunner.kt | 2 +- .../daemon/client/KotlinCompilerClient.kt | 28 +++++------ .../kotlin/daemon/common/ClientUtils.kt | 4 ++ .../kotlin/daemon/common/DaemonParams.kt | 46 ++++++++++++++++--- .../kotlin/daemon/CompileServiceImpl.kt | 6 ++- .../kotlin/daemon/KotlinCompileDaemon.kt | 8 +++- .../kotlin/daemon/CompilerApiTest.kt | 11 ++++- .../kotlin/daemon/CompilerDaemonTest.kt | 36 ++++++++++++--- .../sourceSections/SourceSectionsTest.kt | 2 +- 9 files changed, 110 insertions(+), 33 deletions(-) diff --git a/compiler/compiler-runner/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.kt b/compiler/compiler-runner/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.kt index 408bb8d3b9b..b2ae2267fc7 100644 --- a/compiler/compiler-runner/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.kt +++ b/compiler/compiler-runner/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.kt @@ -60,7 +60,7 @@ abstract class KotlinCompilerRunner { environment: Env, daemonOptions: DaemonOptions = configureDaemonOptions() ): CompileServiceSession? { - val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritAdditionalProperties = true) + val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritOtherJvmOptions = false, inheritAdditionalProperties = true) val daemonReportMessages = ArrayList() val daemonReportingTargets = DaemonReportingTargets(messages = daemonReportMessages) diff --git a/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/KotlinCompilerClient.kt b/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/KotlinCompilerClient.kt index 99f48cdbaea..364a1638442 100644 --- a/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/KotlinCompilerClient.kt +++ b/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/KotlinCompilerClient.kt @@ -102,7 +102,7 @@ object KotlinCompilerClient { autostart: Boolean, leaseSession: Boolean, sessionAliveFlagFile: File? = null - ): CompileServiceSession? = connectLoop(reportingTargets) { + ): CompileServiceSession? = connectLoop(reportingTargets, autostart) { isLastAttempt -> ensureServerHostnameIsSetUp() val (service, newJVMOptions) = tryFindSuitableDaemonOrNewOpts(File(daemonOptions.runFilesPath), compilerId, daemonJVMOptions, { cat, msg -> reportingTargets.report(cat, msg) }) if (service != null) { @@ -119,7 +119,7 @@ object KotlinCompilerClient { } } else { reportingTargets.report(DaemonReportCategory.DEBUG, "no suitable daemon found") - if (autostart) { + if (!isLastAttempt && autostart) { startDaemon(compilerId, newJVMOptions, daemonOptions, reportingTargets) reportingTargets.report(DaemonReportCategory.DEBUG, "new daemon started, trying to find it") } @@ -236,7 +236,7 @@ object KotlinCompilerClient { fun main(vararg args: String) { val compilerId = CompilerId() val daemonOptions = configureDaemonOptions() - val daemonLaunchingOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritAdditionalProperties = true) + val daemonLaunchingOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritOtherJvmOptions = false, inheritAdditionalProperties = true) val clientOptions = configureClientOptions() val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, daemonLaunchingOptions, clientOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) @@ -311,24 +311,26 @@ object KotlinCompilerClient { // --- Implementation --------------------------------------- @Synchronized - private inline fun connectLoop(reportingTargets: DaemonReportingTargets, body: () -> R?): R? { + private inline fun connectLoop(reportingTargets: DaemonReportingTargets, autostart: Boolean, body: (Boolean) -> R?): R? { try { - var attempts = 0 - while (attempts < DAEMON_CONNECT_CYCLE_ATTEMPTS) { - attempts += 1 + var attempts = 1 + while (true) { val (res, err) = try { - body() to null + body(attempts >= DAEMON_CONNECT_CYCLE_ATTEMPTS) to null } catch (e: SocketException) { null to e } catch (e: ConnectException) { null to e } catch (e: ConnectIOException) { null to e } catch (e: UnmarshalException) { null to e } - when { - res != null -> return res - err != null && attempts >= DAEMON_CONNECT_CYCLE_ATTEMPTS -> throw err - err != null -> - reportingTargets.report(DaemonReportCategory.INFO, "retrying($attempts) on: " + err.toString()) + if (res != null) return res + + reportingTargets.report(DaemonReportCategory.INFO, + (if (attempts >= DAEMON_CONNECT_CYCLE_ATTEMPTS || !autostart) "no more retries on: " else "retrying($attempts) on: ") + + err?.toString()) + + if (attempts++ > DAEMON_CONNECT_CYCLE_ATTEMPTS || !autostart) { + return null } } } diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ClientUtils.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ClientUtils.kt index c47d3e53859..7913e392b46 100644 --- a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ClientUtils.kt +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ClientUtils.kt @@ -44,6 +44,10 @@ private const val ORPHANED_RUN_FILE_AGE_THRESHOLD_MS = 1000000L data class DaemonWithMetadata(val daemon: CompileService, val runFile: File, val jvmOptions: DaemonJVMOptions) +// TODO: write metadata into discovery file to speed up selection +// TODO: consider using compiler jar signature (checksum) as a CompilerID (plus java version, plus ???) instead of classpath checksum +// would allow to use same compiler from taken different locations +// reqs: check that plugins (or anything els) should not be part of the CP fun walkDaemons(registryDir: File, compilerId: CompilerId, fileToCompareTimestamp: File, diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonParams.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonParams.kt index 4e57c4b24cf..3c5ab1e9457 100644 --- a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonParams.kt +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonParams.kt @@ -257,15 +257,44 @@ data class CompilerId( fun isDaemonEnabled(): Boolean = System.getProperty(COMPILE_DAEMON_ENABLED_PROPERTY) != null - fun configureDaemonJVMOptions(opts: DaemonJVMOptions, vararg additionalParams: String, inheritMemoryLimits: Boolean, + inheritOtherJvmOptions: Boolean, + inheritAdditionalProperties: Boolean +): DaemonJVMOptions = + configureDaemonJVMOptions(opts, additionalParams.asIterable(), inheritMemoryLimits, inheritOtherJvmOptions, inheritAdditionalProperties) + +// TODO: expose sources for testability and test properly +fun configureDaemonJVMOptions(opts: DaemonJVMOptions, + additionalParams: Iterable, + inheritMemoryLimits: Boolean, + inheritOtherJvmOptions: Boolean, inheritAdditionalProperties: Boolean ): DaemonJVMOptions { - // note: sequence matters, explicit override in COMPILE_DAEMON_JVM_OPTIONS_PROPERTY should be done after inputArguments processing - if (inheritMemoryLimits) { - ManagementFactory.getRuntimeMXBean().inputArguments.filterExtractProps(opts.mappers, "-") + // note: sequence matters, e.g. explicit override in COMPILE_DAEMON_JVM_OPTIONS_PROPERTY should be done after inputArguments processing + if (inheritMemoryLimits || inheritOtherJvmOptions) { + val jvmArguments = ManagementFactory.getRuntimeMXBean().inputArguments + val targetOptions = if (inheritMemoryLimits) opts else DaemonJVMOptions() + val otherArgs = jvmArguments.filterExtractProps(targetOptions.mappers, prefix = "-") + + if (inheritMemoryLimits && opts.maxMemory.isBlank()) { + val maxMemBytes = Runtime.getRuntime().maxMemory() + // rounding up + val maxMemMegabytes = maxMemBytes / (1024 * 1024) + if (maxMemBytes % (1024 * 1024) == 0L) 0 else 1 + opts.maxMemory = "${maxMemMegabytes}m" + } + + if (inheritOtherJvmOptions) { + opts.jvmParams.addAll( + otherArgs.filterNot { + it.startsWith("agentlib") || + it.startsWith("D" + COMPILE_DAEMON_LOG_PATH_PROPERTY) || + it.startsWith("D" + KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY) || + it.startsWith("D" + COMPILE_DAEMON_JVM_OPTIONS_PROPERTY) || + it.startsWith("D" + COMPILE_DAEMON_OPTIONS_PROPERTY) + }) + } } System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)?.let { opts.jvmParams.addAll( @@ -275,7 +304,10 @@ fun configureDaemonJVMOptions(opts: DaemonJVMOptions, .filterExtractProps(opts.mappers, "-", opts.restMapper)) } + // assuming that from the conflicting options the last one is taken + // TODO: compare and override opts.jvmParams.addAll(additionalParams) + if (inheritAdditionalProperties) { System.getProperty(COMPILE_DAEMON_LOG_PATH_PROPERTY)?.let { opts.jvmParams.add("D$COMPILE_DAEMON_LOG_PATH_PROPERTY=\"$it\"") } System.getProperty(KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY)?.let { opts.jvmParams.add("D$KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY") } @@ -285,12 +317,14 @@ fun configureDaemonJVMOptions(opts: DaemonJVMOptions, fun configureDaemonJVMOptions(vararg additionalParams: String, - inheritMemoryLimits: Boolean, - inheritAdditionalProperties: Boolean + inheritMemoryLimits: Boolean, + inheritOtherJvmOptions: Boolean, + inheritAdditionalProperties: Boolean ): DaemonJVMOptions = configureDaemonJVMOptions(DaemonJVMOptions(), additionalParams = *additionalParams, inheritMemoryLimits = inheritMemoryLimits, + inheritOtherJvmOptions = inheritOtherJvmOptions, inheritAdditionalProperties = inheritAdditionalProperties) diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt index b491c88c2df..43ef01aa15f 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt @@ -107,6 +107,8 @@ class CompileServiceImpl( val onShutdown: () -> Unit ) : CompileService { + private val log by lazy { Logger.getLogger("compiler") } + init { System.setProperty(KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY, "true") } @@ -184,8 +186,6 @@ class CompileServiceImpl( @Volatile private var _lastUsedSeconds = nowSeconds() val lastUsedSeconds: Long get() = if (rwlock.isWriteLocked || rwlock.readLockCount - rwlock.readHoldCount > 0) nowSeconds() else _lastUsedSeconds - private val log by lazy { Logger.getLogger("compiler") } - private val rwlock = ReentrantReadWriteLock() private var runFile: File @@ -212,6 +212,8 @@ class CompileServiceImpl( } override fun getDaemonJVMOptions(): CompileService.CallResult = ifAlive { + log.info("getDaemonJVMOptions: $daemonJVMOptions")// + daemonJVMOptions.mappers.flatMap { it.toArgs("-") }) + CompileService.CallResult.Good(daemonJVMOptions) } diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinCompileDaemon.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinCompileDaemon.kt index 885f7d93f8b..b33e23f5052 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinCompileDaemon.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinCompileDaemon.kt @@ -91,8 +91,10 @@ object KotlinCompileDaemon { fun main(args: Array) { ensureServerHostnameIsSetUp() + val jvmArguments = ManagementFactory.getRuntimeMXBean().inputArguments + log.info("Kotlin compiler daemon version " + (loadVersionFromResource() ?: "")) - log.info("daemon JVM args: " + ManagementFactory.getRuntimeMXBean().inputArguments.joinToString(" ")) + log.info("daemon JVM args: " + jvmArguments.joinToString(" ")) log.info("daemon args: " + args.joinToString(" ")) setIdeaIoUseFallback() @@ -101,7 +103,9 @@ object KotlinCompileDaemon { val daemonOptions = DaemonOptions() try { - val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritAdditionalProperties = true) + val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, + inheritOtherJvmOptions = true, + inheritAdditionalProperties = true) val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) diff --git a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerApiTest.kt b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerApiTest.kt index b6743ae56d7..13869568460 100644 --- a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerApiTest.kt +++ b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerApiTest.kt @@ -128,7 +128,7 @@ class CompilerApiTest : KotlinIntegrationTestBase() { val logFile = createTempFile("kotlin-daemon-test.", ".log") val daemonJVMOptions = configureDaemonJVMOptions("D$COMPILE_DAEMON_LOG_PATH_PROPERTY=\"${logFile.loggerCompatiblePath}\"", - inheritMemoryLimits = false, inheritAdditionalProperties = false) + inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false) val jar = tmpdir.absolutePath + File.separator + "hello.jar" try { @@ -169,7 +169,7 @@ class CompilerApiTest : KotlinIntegrationTestBase() { val logFile = createTempFile("kotlin-daemon-test.", ".log") val daemonJVMOptions = configureDaemonJVMOptions("D$COMPILE_DAEMON_LOG_PATH_PROPERTY=\"${logFile.loggerCompatiblePath}\"", - inheritMemoryLimits = false, inheritAdditionalProperties = false) + inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false) try { val (code, outputs) = compileOnDaemon( flagFile, compilerId, daemonJVMOptions, daemonOptions, TestMessageCollector(), @@ -203,3 +203,10 @@ class TestMessageCollector : MessageCollector { override fun hasErrors(): Boolean = messages.any { it.severity == CompilerMessageSeverity.EXCEPTION || it.severity == CompilerMessageSeverity.ERROR } } + +fun TestMessageCollector.assertHasMessage(msg: String) { + assert(messages.any { it.message.contains(msg) }) { + "Expecting message \"$msg\", actual:\n${messages.joinToString("\n") { it.message }}" + } +} + diff --git a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt index f5108aaeb79..0938023d1ea 100644 --- a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt +++ b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt @@ -97,7 +97,8 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { baseOpts, *additionalArgs.toTypedArray(), inheritMemoryLimits = xmx > 0, - inheritAdditionalProperties = false) + inheritAdditionalProperties = false, + inheritOtherJvmOptions = false) } fun testHelloApp() { @@ -144,14 +145,14 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { val backupJvmOptions = System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY) try { System.setProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY, "-aaa,-bbb\\,ccc,-ddd,-Xmx200m,-XX:MaxPermSize=10k,-XX:ReservedCodeCacheSize=100,-xxx\\,yyy") - val opts = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false) + val opts = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false, inheritOtherJvmOptions = false) assertEquals("200m", opts.maxMemory) assertEquals("10k", opts.maxPermSize) assertEquals("100", opts.reservedCodeCacheSize) assertEquals(arrayListOf("aaa", "bbb,ccc", "ddd", "xxx,yyy"), opts.jvmParams) System.setProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY, "-Xmx300m,-XX:MaxPermSize=10k,-XX:ReservedCodeCacheSize=100") - val opts2 = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false) + val opts2 = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false, inheritOtherJvmOptions = false) assertEquals("300m", opts2.maxMemory) assertEquals( -1, DaemonJVMOptionsMemoryComparator().compare(opts, opts2)) assertEquals("300m", listOf(opts, opts2).maxWith(DaemonJVMOptionsMemoryComparator())?.maxMemory) @@ -161,6 +162,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { val myXmxVal = myXmxParam.substring(4) System.clearProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY) val opts3 = configureDaemonJVMOptions(inheritMemoryLimits = true, + inheritOtherJvmOptions = true, inheritAdditionalProperties = false) assertEquals(myXmxVal, opts3.maxMemory) } @@ -229,7 +231,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) - val daemonJVMOptions = configureDaemonJVMOptions("-abracadabra", inheritMemoryLimits = false, inheritAdditionalProperties = false) + val daemonJVMOptions = configureDaemonJVMOptions("-abracadabra", inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false) val messageCollector = TestMessageCollector() @@ -238,8 +240,30 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() { assertNull(daemon) - assertTrue("Expecting error message, actual:\n${messageCollector.messages.joinToString("\n") { it.message }}", - messageCollector.messages.any { it.message == "Unrecognized option: --abracadabra" }) + messageCollector.assertHasMessage("Unrecognized option: --abracadabra") + } + } + + // TODO: find out how to reliably cause the retry + fun ignore_testDaemonStartRetry() { + withFlagFile(getTestName(true), ".alive") { flagFile -> + val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, verbose = true, runFilesPath = File(tmpdir, getTestName(true)).absolutePath) + + KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) + + val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false) + + val messageCollector = TestMessageCollector() + + val daemon = KotlinCompilerClient.connectToCompileService(compilerId, flagFile, daemonJVMOptions, daemonOptions, + DaemonReportingTargets(messageCollector = messageCollector), autostart = true) + + assertNull(daemon) + + messageCollector.assertHasMessage("retrying(0) on:") + messageCollector.assertHasMessage("retrying(1) on:") + // TODO: messageCollector.assertHasNoMessage("retrying(2) on:") + messageCollector.assertHasMessage("no more retries on:") } } diff --git a/plugins/source-sections/source-sections-compiler/tests/org/jetbrains/kotlin/sourceSections/SourceSectionsTest.kt b/plugins/source-sections/source-sections-compiler/tests/org/jetbrains/kotlin/sourceSections/SourceSectionsTest.kt index c0294d3430b..a47996a4b7f 100644 --- a/plugins/source-sections/source-sections-compiler/tests/org/jetbrains/kotlin/sourceSections/SourceSectionsTest.kt +++ b/plugins/source-sections/source-sections-compiler/tests/org/jetbrains/kotlin/sourceSections/SourceSectionsTest.kt @@ -196,7 +196,7 @@ class SourceSectionsTest : TestCaseWithTmpdir() { withFlagFile("sourceSections", ".alive") { aliveFile -> val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath, verbose = true, reportPerf = true) - val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false) + val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false) val messageCollector = TestMessageCollector() val daemonWithSession = KotlinCompilerClient.connectAndLease(compilerId, aliveFile, daemonJVMOptions, daemonOptions,