Introduce other JVM options inheritance for daemon

compiler/compiler-runner/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.kt

@@ -60,7 +60,7 @@ abstract class KotlinCompilerRunner<in Env : CompilerEnvironment> {
             environment: Env,
             daemonOptions: DaemonOptions = configureDaemonOptions()
     ): CompileServiceSession? {
-        val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritAdditionalProperties = true)
+        val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritOtherJvmOptions = false, inheritAdditionalProperties = true)
 
         val daemonReportMessages = ArrayList<DaemonReportMessage>()
         val daemonReportingTargets = DaemonReportingTargets(messages = daemonReportMessages)

compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/KotlinCompilerClient.kt

@@ -102,7 +102,7 @@ object KotlinCompilerClient {
                         autostart: Boolean,
                         leaseSession: Boolean,
                         sessionAliveFlagFile: File? = null
-    ): CompileServiceSession? = connectLoop(reportingTargets) {
+    ): CompileServiceSession? = connectLoop(reportingTargets, autostart) { isLastAttempt ->
         ensureServerHostnameIsSetUp()
         val (service, newJVMOptions) = tryFindSuitableDaemonOrNewOpts(File(daemonOptions.runFilesPath), compilerId, daemonJVMOptions, { cat, msg -> reportingTargets.report(cat, msg) })
         if (service != null) {
@@ -119,7 +119,7 @@ object KotlinCompilerClient {
             }
         } else {
             reportingTargets.report(DaemonReportCategory.DEBUG, "no suitable daemon found")
-            if (autostart) {
+            if (!isLastAttempt && autostart) {
                 startDaemon(compilerId, newJVMOptions, daemonOptions, reportingTargets)
                 reportingTargets.report(DaemonReportCategory.DEBUG, "new daemon started, trying to find it")
             }
@@ -236,7 +236,7 @@ object KotlinCompilerClient {
     fun main(vararg args: String) {
         val compilerId = CompilerId()
         val daemonOptions = configureDaemonOptions()
-        val daemonLaunchingOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritAdditionalProperties = true)
+        val daemonLaunchingOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritOtherJvmOptions = false, inheritAdditionalProperties = true)
         val clientOptions = configureClientOptions()
         val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, daemonLaunchingOptions, clientOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX)
 
@@ -311,24 +311,26 @@ object KotlinCompilerClient {
     // --- Implementation ---------------------------------------
 
     @Synchronized
-    private inline fun <R> connectLoop(reportingTargets: DaemonReportingTargets, body: () -> R?): R? {
+    private inline fun <R> connectLoop(reportingTargets: DaemonReportingTargets, autostart: Boolean, body: (Boolean) -> R?): R? {
         try {
-            var attempts = 0
+            var attempts = 1
-            while (attempts < DAEMON_CONNECT_CYCLE_ATTEMPTS) {
+            while (true) {
-                attempts += 1
                 val (res, err) = try {
-                    body() to null
+                    body(attempts >= DAEMON_CONNECT_CYCLE_ATTEMPTS) to null
                 }
                 catch (e: SocketException) { null to e }
                 catch (e: ConnectException) { null to e }
                 catch (e: ConnectIOException) { null to e }
                 catch (e: UnmarshalException) { null to e }
 
-                when {
+                if (res != null) return res
-                    res != null -> return res
+
-                    err != null && attempts >= DAEMON_CONNECT_CYCLE_ATTEMPTS -> throw err
+                reportingTargets.report(DaemonReportCategory.INFO,
-                    err != null ->
+                                        (if (attempts >= DAEMON_CONNECT_CYCLE_ATTEMPTS || !autostart) "no more retries on: " else "retrying($attempts) on: ")
-                        reportingTargets.report(DaemonReportCategory.INFO, "retrying($attempts) on: " + err.toString())
+                                        + err?.toString())
+
+                if (attempts++ > DAEMON_CONNECT_CYCLE_ATTEMPTS || !autostart) {
+                    return null
                 }
             }
         }

compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ClientUtils.kt

@@ -44,6 +44,10 @@ private const val ORPHANED_RUN_FILE_AGE_THRESHOLD_MS = 1000000L
 
 data class DaemonWithMetadata(val daemon: CompileService, val runFile: File, val jvmOptions: DaemonJVMOptions)
 
+// TODO: write metadata into discovery file to speed up selection
+// TODO: consider using compiler jar signature (checksum) as a CompilerID (plus java version, plus ???) instead of classpath checksum
+//    would allow to use same compiler from taken different locations
+//    reqs: check that plugins (or anything els) should not be part of the CP
 fun walkDaemons(registryDir: File,
                 compilerId: CompilerId,
                 fileToCompareTimestamp: File,

compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonParams.kt

@@ -257,15 +257,44 @@ data class CompilerId(
 
 fun isDaemonEnabled(): Boolean = System.getProperty(COMPILE_DAEMON_ENABLED_PROPERTY) != null
 
-
 fun configureDaemonJVMOptions(opts: DaemonJVMOptions,
                               vararg additionalParams: String,
                               inheritMemoryLimits: Boolean,
+                              inheritOtherJvmOptions: Boolean,
+                              inheritAdditionalProperties: Boolean
+): DaemonJVMOptions =
+        configureDaemonJVMOptions(opts, additionalParams.asIterable(), inheritMemoryLimits, inheritOtherJvmOptions, inheritAdditionalProperties)
+
+// TODO: expose sources for testability and test properly
+fun configureDaemonJVMOptions(opts: DaemonJVMOptions,
+                              additionalParams: Iterable<String>,
+                              inheritMemoryLimits: Boolean,
+                              inheritOtherJvmOptions: Boolean,
                               inheritAdditionalProperties: Boolean
 ): DaemonJVMOptions {
-    // note: sequence matters, explicit override in COMPILE_DAEMON_JVM_OPTIONS_PROPERTY should be done after inputArguments processing
+    // note: sequence matters, e.g. explicit override in COMPILE_DAEMON_JVM_OPTIONS_PROPERTY should be done after inputArguments processing
-    if (inheritMemoryLimits) {
+    if (inheritMemoryLimits || inheritOtherJvmOptions) {
-        ManagementFactory.getRuntimeMXBean().inputArguments.filterExtractProps(opts.mappers, "-")
+        val jvmArguments = ManagementFactory.getRuntimeMXBean().inputArguments
+        val targetOptions = if (inheritMemoryLimits) opts else DaemonJVMOptions()
+        val otherArgs = jvmArguments.filterExtractProps(targetOptions.mappers, prefix = "-")
+
+        if (inheritMemoryLimits && opts.maxMemory.isBlank()) {
+            val maxMemBytes = Runtime.getRuntime().maxMemory()
+            // rounding up
+            val maxMemMegabytes = maxMemBytes / (1024 * 1024) + if (maxMemBytes % (1024 * 1024) == 0L) 0 else 1
+            opts.maxMemory = "${maxMemMegabytes}m"
+        }
+
+        if (inheritOtherJvmOptions) {
+            opts.jvmParams.addAll(
+                    otherArgs.filterNot {
+                        it.startsWith("agentlib") ||
+                        it.startsWith("D" + COMPILE_DAEMON_LOG_PATH_PROPERTY) ||
+                        it.startsWith("D" + KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY) ||
+                        it.startsWith("D" + COMPILE_DAEMON_JVM_OPTIONS_PROPERTY) ||
+                        it.startsWith("D" + COMPILE_DAEMON_OPTIONS_PROPERTY)
+                    })
+        }
     }
     System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)?.let {
         opts.jvmParams.addAll(
@@ -275,7 +304,10 @@ fun configureDaemonJVMOptions(opts: DaemonJVMOptions,
                   .filterExtractProps(opts.mappers, "-", opts.restMapper))
     }
 
+    // assuming that from the conflicting options the last one is taken
+    // TODO: compare and override
     opts.jvmParams.addAll(additionalParams)
+
     if (inheritAdditionalProperties) {
         System.getProperty(COMPILE_DAEMON_LOG_PATH_PROPERTY)?.let { opts.jvmParams.add("D$COMPILE_DAEMON_LOG_PATH_PROPERTY=\"$it\"") }
         System.getProperty(KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY)?.let { opts.jvmParams.add("D$KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY") }
@@ -285,12 +317,14 @@ fun configureDaemonJVMOptions(opts: DaemonJVMOptions,
 
 
 fun configureDaemonJVMOptions(vararg additionalParams: String,
-                                     inheritMemoryLimits: Boolean,
+                              inheritMemoryLimits: Boolean,
-                                     inheritAdditionalProperties: Boolean
+                              inheritOtherJvmOptions: Boolean,
+                              inheritAdditionalProperties: Boolean
 ): DaemonJVMOptions =
         configureDaemonJVMOptions(DaemonJVMOptions(),
                                   additionalParams = *additionalParams,
                                   inheritMemoryLimits = inheritMemoryLimits,
+                                  inheritOtherJvmOptions = inheritOtherJvmOptions,
                                   inheritAdditionalProperties = inheritAdditionalProperties)
 
 

compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt

@@ -107,6 +107,8 @@ class CompileServiceImpl(
         val onShutdown: () -> Unit
 ) : CompileService {
 
+    private val log by lazy { Logger.getLogger("compiler") }
+
     init {
         System.setProperty(KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY, "true")
     }
@@ -184,8 +186,6 @@ class CompileServiceImpl(
     @Volatile private var _lastUsedSeconds = nowSeconds()
     val lastUsedSeconds: Long get() = if (rwlock.isWriteLocked || rwlock.readLockCount - rwlock.readHoldCount > 0) nowSeconds() else _lastUsedSeconds
 
-    private val log by lazy { Logger.getLogger("compiler") }
-
     private val rwlock = ReentrantReadWriteLock()
 
     private var runFile: File
@@ -212,6 +212,8 @@ class CompileServiceImpl(
     }
 
     override fun getDaemonJVMOptions(): CompileService.CallResult<DaemonJVMOptions> = ifAlive {
+        log.info("getDaemonJVMOptions: $daemonJVMOptions")// + daemonJVMOptions.mappers.flatMap { it.toArgs("-") })
+
         CompileService.CallResult.Good(daemonJVMOptions)
     }
 

compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinCompileDaemon.kt

@@ -91,8 +91,10 @@ object KotlinCompileDaemon {
     fun main(args: Array<String>) {
         ensureServerHostnameIsSetUp()
 
+        val jvmArguments = ManagementFactory.getRuntimeMXBean().inputArguments
+
         log.info("Kotlin compiler daemon version " + (loadVersionFromResource() ?: "<unknown>"))
-        log.info("daemon JVM args: " + ManagementFactory.getRuntimeMXBean().inputArguments.joinToString(" "))
+        log.info("daemon JVM args: " + jvmArguments.joinToString(" "))
         log.info("daemon args: " + args.joinToString(" "))
 
         setIdeaIoUseFallback()
@@ -101,7 +103,9 @@ object KotlinCompileDaemon {
         val daemonOptions = DaemonOptions()
 
         try {
-            val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritAdditionalProperties = true)
+            val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true,
+                                                             inheritOtherJvmOptions = true,
+                                                             inheritAdditionalProperties = true)
 
             val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX)
 

compiler/tests/org/jetbrains/kotlin/daemon/CompilerApiTest.kt

@@ -128,7 +128,7 @@ class CompilerApiTest : KotlinIntegrationTestBase() {
             val logFile = createTempFile("kotlin-daemon-test.", ".log")
 
             val daemonJVMOptions = configureDaemonJVMOptions("D$COMPILE_DAEMON_LOG_PATH_PROPERTY=\"${logFile.loggerCompatiblePath}\"",
-                                                             inheritMemoryLimits = false, inheritAdditionalProperties = false)
+                                                             inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false)
             val jar = tmpdir.absolutePath + File.separator + "hello.jar"
 
             try {
@@ -169,7 +169,7 @@ class CompilerApiTest : KotlinIntegrationTestBase() {
             val logFile = createTempFile("kotlin-daemon-test.", ".log")
 
             val daemonJVMOptions = configureDaemonJVMOptions("D$COMPILE_DAEMON_LOG_PATH_PROPERTY=\"${logFile.loggerCompatiblePath}\"",
-                                                             inheritMemoryLimits = false, inheritAdditionalProperties = false)
+                                                             inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false)
             try {
                 val (code, outputs) = compileOnDaemon(
                         flagFile, compilerId, daemonJVMOptions, daemonOptions, TestMessageCollector(),
@@ -203,3 +203,10 @@ class TestMessageCollector : MessageCollector {
 
     override fun hasErrors(): Boolean = messages.any { it.severity == CompilerMessageSeverity.EXCEPTION || it.severity == CompilerMessageSeverity.ERROR }
 }
+
+fun TestMessageCollector.assertHasMessage(msg: String) {
+    assert(messages.any { it.message.contains(msg) }) {
+        "Expecting message \"$msg\", actual:\n${messages.joinToString("\n") { it.message }}"
+    }
+}
+

compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt

@@ -97,7 +97,8 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
                 baseOpts,
                 *additionalArgs.toTypedArray(),
                 inheritMemoryLimits = xmx > 0,
-                inheritAdditionalProperties = false)
+                inheritAdditionalProperties = false,
+                inheritOtherJvmOptions = false)
     }
 
     fun testHelloApp() {
@@ -144,14 +145,14 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
         val backupJvmOptions = System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)
         try {
             System.setProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY, "-aaa,-bbb\\,ccc,-ddd,-Xmx200m,-XX:MaxPermSize=10k,-XX:ReservedCodeCacheSize=100,-xxx\\,yyy")
-            val opts = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false)
+            val opts = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false, inheritOtherJvmOptions = false)
             assertEquals("200m", opts.maxMemory)
             assertEquals("10k", opts.maxPermSize)
             assertEquals("100", opts.reservedCodeCacheSize)
             assertEquals(arrayListOf("aaa", "bbb,ccc", "ddd", "xxx,yyy"), opts.jvmParams)
 
             System.setProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY, "-Xmx300m,-XX:MaxPermSize=10k,-XX:ReservedCodeCacheSize=100")
-            val opts2 = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false)
+            val opts2 = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false, inheritOtherJvmOptions = false)
             assertEquals("300m", opts2.maxMemory)
             assertEquals( -1, DaemonJVMOptionsMemoryComparator().compare(opts, opts2))
             assertEquals("300m", listOf(opts, opts2).maxWith(DaemonJVMOptionsMemoryComparator())?.maxMemory)
@@ -161,6 +162,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
             val myXmxVal = myXmxParam.substring(4)
             System.clearProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)
             val opts3 = configureDaemonJVMOptions(inheritMemoryLimits = true,
+                                                  inheritOtherJvmOptions = true,
                                                   inheritAdditionalProperties = false)
             assertEquals(myXmxVal, opts3.maxMemory)
         }
@@ -229,7 +231,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
 
             KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions)
 
-            val daemonJVMOptions = configureDaemonJVMOptions("-abracadabra", inheritMemoryLimits = false, inheritAdditionalProperties = false)
+            val daemonJVMOptions = configureDaemonJVMOptions("-abracadabra", inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false)
 
             val messageCollector = TestMessageCollector()
 
@@ -238,8 +240,30 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
 
             assertNull(daemon)
 
-            assertTrue("Expecting error message, actual:\n${messageCollector.messages.joinToString("\n") { it.message }}",
+            messageCollector.assertHasMessage("Unrecognized option: --abracadabra")
-                       messageCollector.messages.any { it.message == "Unrecognized option: --abracadabra" })
+        }
+    }
+
+    // TODO: find out how to reliably cause the retry
+    fun ignore_testDaemonStartRetry() {
+        withFlagFile(getTestName(true), ".alive") { flagFile ->
+            val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, verbose = true, runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
+
+            KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions)
+
+            val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false)
+
+            val messageCollector = TestMessageCollector()
+
+            val daemon = KotlinCompilerClient.connectToCompileService(compilerId, flagFile, daemonJVMOptions, daemonOptions,
+                                                                      DaemonReportingTargets(messageCollector = messageCollector), autostart = true)
+
+            assertNull(daemon)
+
+            messageCollector.assertHasMessage("retrying(0) on:")
+            messageCollector.assertHasMessage("retrying(1) on:")
+            // TODO: messageCollector.assertHasNoMessage("retrying(2) on:")
+            messageCollector.assertHasMessage("no more retries on:")
         }
     }
 

plugins/source-sections/source-sections-compiler/tests/org/jetbrains/kotlin/sourceSections/SourceSectionsTest.kt

@@ -196,7 +196,7 @@ class SourceSectionsTest : TestCaseWithTmpdir() {
         withFlagFile("sourceSections", ".alive") { aliveFile ->
 
             val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath, verbose = true, reportPerf = true)
-            val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false)
+            val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false)
             val messageCollector = TestMessageCollector()
 
             val daemonWithSession = KotlinCompilerClient.connectAndLease(compilerId, aliveFile, daemonJVMOptions, daemonOptions,