diff --git a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt index 1e370e2c273..22ca0044e9e 100644 --- a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt +++ b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt @@ -32,6 +32,8 @@ public object KotlinCompilerClient { val DAEMON_DEFAULT_STARTUP_TIMEOUT_MS = 10000L val DAEMON_CONNECT_CYCLE_ATTEMPTS = 3 + val verboseReporting = System.getProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) != null + // TODO: remove jvmStatic after all use sites will switch to kotlin jvmStatic @@ -45,6 +47,7 @@ public object KotlinCompilerClient { var attempts = 0 var fileLock: FileBasedLock? = null + var shutdonwnPerformed = false try { while (attempts++ < DAEMON_CONNECT_CYCLE_ATTEMPTS) { val service = tryFindDaemon(File(daemonOptions.runFilesPath), compilerId, reportingTargets) @@ -59,11 +62,12 @@ public object KotlinCompilerClient { service.shutdown() // TODO: find more reliable way Thread.sleep(1000) - reportingTargets.report(DaemonReportCategory.DEBUG, "daemon shut down correctly, restarting search") + reportingTargets.report(DaemonReportCategory.DEBUG, "daemon shut down correctly") + shutdonwnPerformed = true } else { if (!autostart) return null - reportingTargets.report(DaemonReportCategory.DEBUG, "no suitable daemon found, starting a new one") + reportingTargets.report(DaemonReportCategory.DEBUG, if (shutdonwnPerformed) "starting a new daemon" else "no suitable daemon found, starting a new one") } if (fileLock == null || !fileLock.isLocked()) { @@ -75,7 +79,7 @@ public object KotlinCompilerClient { } else { startDaemon(compilerId, daemonJVMOptions, daemonOptions, reportingTargets) - reportingTargets.report(DaemonReportCategory.DEBUG, "daemon started, trying to resolve") + reportingTargets.report(DaemonReportCategory.DEBUG, "daemon started, trying to find it") } } } @@ -208,20 +212,28 @@ public object KotlinCompilerClient { // --- Implementation --------------------------------------- - val verboseReporting = System.getProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) != null - fun DaemonReportingTargets.report(category: DaemonReportCategory, message: String, source: String = "daemon client") { if (category == DaemonReportCategory.DEBUG && !verboseReporting) return out?.println("[$source] ${category.name()}: $message") messages?.add(DaemonReportMessage(category, "[$source] $message")) } + + private fun String.extractPortFromRunFilename(digest: String): Int = + makeRunFilenameString(timestamp = "[0-9TZ:\\.\\+-]+", digest = digest, port = "(\\d+)", escapeSequence = "\\").toRegex() + .match(this) + ?.groups?.get(1) + ?.value?.toInt() + ?: 0 + + private fun tryFindDaemon(registryDir: File, compilerId: CompilerId, reportingTargets: DaemonReportingTargets): CompileService? { val classPathDigest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest() val daemons = registryDir.walk() - .map { Pair(it, makeRunFilenameRegex(digest = classPathDigest, port = "(\\d+)").match(it.name)?.groups?.get(1)?.value?.toInt() ?: 0) } + .map { Pair(it, it.name.extractPortFromRunFilename(classPathDigest)) } .filter { it.second != 0 } .map { + assert(it.second > 0 && it.second < 0xffff) reportingTargets.report(DaemonReportCategory.DEBUG, "found suitable daemon on port ${it.second}, trying to connect") val daemon = tryConnectToDaemon(it.second, reportingTargets) // cleaning orphaned file; note: daemon should shut itself down if it detects that the run file is deleted @@ -240,9 +252,10 @@ public object KotlinCompilerClient { } } + private fun tryConnectToDaemon(port: Int, reportingTargets: DaemonReportingTargets): CompileService? { try { - val daemon = LocateRegistry.getRegistry(loopbackAddrName, port) + val daemon = LocateRegistry.getRegistry(LoopbackNetworkInterface.loopbackInetAddressName, port) ?.lookup(COMPILER_SERVICE_RMI_NAME) if (daemon != null) return daemon as? CompileService ?: @@ -330,13 +343,15 @@ public object KotlinCompilerClient { class FileBasedLock(compilerId: CompilerId, daemonOptions: DaemonOptions) { + private val lockFile: File = File(daemonOptions.runFilesPath, - makeRunFilenameString(ts = "lock", + makeRunFilenameString(timestamp = "lock", digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest(), port = "0")) - - private var locked: Boolean = acquireLockFile(lockFile) + private volatile var locked = acquireLockFile(lockFile) + private val channel = if (locked) RandomAccessFile(lockFile, "rw").channel else null + private val lock = channel?.lock() public fun isLocked(): Boolean = locked @@ -349,42 +364,26 @@ public object KotlinCompilerClient { } } - private val channel = if (locked) RandomAccessFile(lockFile, "rw").channel else null - private val lock = channel?.lock() - synchronized private fun acquireLockFile(lockFile: File): Boolean { - if (lockFile.createNewFile()) return true - try { - // attempt to delete if file is orphaned - if (lockFile.delete() && lockFile.createNewFile()) - return true // if orphaned file deleted assuming that the probability of - } - catch (e: IOException) { - // Ignoring it - assuming it is another client process owning it - } - var attempts = 0L - while (lockFile.exists() && attempts++ * COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_CHECK_MS < COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_MS) { + val maxAttempts = COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_MS / COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_CHECK_MS + 1 + var attempt = 0L + while (true) { + try { + if (lockFile.createNewFile()) break + // attempt to delete if file is orphaned + if (lockFile.delete() && lockFile.createNewFile()) break + } + catch (e: IOException) {} // Ignoring it - assuming it is another client process owning it + if (lockFile.exists() && ++attempt >= maxAttempts) + throw IOException("Timeout waiting the release of the lock file '${lockFile.absolutePath}") Thread.sleep(COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_CHECK_MS) } - if (lockFile.exists()) - throw IOException("Timeout waiting the release of the lock file '${lockFile.absolutePath}") - - return lockFile.createNewFile() + return true } } } -fun Process.isAlive() = - try { - this.exitValue() - false - } - catch (e: IllegalThreadStateException) { - true - } - - public enum class DaemonReportCategory { DEBUG, INFO, EXCEPTION; } @@ -392,3 +391,13 @@ public enum class DaemonReportCategory { public data class DaemonReportMessage(public val category: DaemonReportCategory, public val message: String) public class DaemonReportingTargets(public val out: PrintStream? = null, public val messages: MutableCollection? = null) + + +internal fun Process.isAlive() = + try { + this.exitValue() + false + } + catch (e: IllegalThreadStateException) { + true + } diff --git a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteIncrementalCacheServer.kt b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteIncrementalCacheServer.kt index 53fa674b419..9dee971c862 100644 --- a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteIncrementalCacheServer.kt +++ b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteIncrementalCacheServer.kt @@ -18,15 +18,15 @@ package org.jetbrains.kotlin.rmi.kotlinr import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache import org.jetbrains.kotlin.rmi.CompileService -import org.jetbrains.kotlin.rmi.clientLoopbackSocketFactory -import org.jetbrains.kotlin.rmi.serverLoopbackSocketFactory +import org.jetbrains.kotlin.rmi.LoopbackNetworkInterface +import org.jetbrains.kotlin.rmi.SOCKET_ANY_FREE_PORT import java.rmi.server.UnicastRemoteObject -public class RemoteIncrementalCacheServer(val cache: IncrementalCache, port: Int = 0) : CompileService.RemoteIncrementalCache { +public class RemoteIncrementalCacheServer(val cache: IncrementalCache, port: Int = SOCKET_ANY_FREE_PORT) : CompileService.RemoteIncrementalCache { init { - UnicastRemoteObject.exportObject(this, port, clientLoopbackSocketFactory, serverLoopbackSocketFactory) + UnicastRemoteObject.exportObject(this, port, LoopbackNetworkInterface.clientLoopbackSocketFactory, LoopbackNetworkInterface.serverLoopbackSocketFactory) } override fun getObsoletePackageParts(): Collection = cache.getObsoletePackageParts() diff --git a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteOutputStreamServer.kt b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteOutputStreamServer.kt index 19dd8c136ee..8b00b25c215 100644 --- a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteOutputStreamServer.kt +++ b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteOutputStreamServer.kt @@ -16,17 +16,17 @@ package org.jetbrains.kotlin.rmi.kotlinr +import org.jetbrains.kotlin.rmi.LoopbackNetworkInterface import org.jetbrains.kotlin.rmi.RemoteOutputStream -import org.jetbrains.kotlin.rmi.clientLoopbackSocketFactory -import org.jetbrains.kotlin.rmi.serverLoopbackSocketFactory +import org.jetbrains.kotlin.rmi.SOCKET_ANY_FREE_PORT import java.io.OutputStream import java.rmi.server.UnicastRemoteObject -class RemoteOutputStreamServer(val out: OutputStream, port: Int = 0) : RemoteOutputStream { +class RemoteOutputStreamServer(val out: OutputStream, port: Int = SOCKET_ANY_FREE_PORT) : RemoteOutputStream { init { - UnicastRemoteObject.exportObject(this, port, clientLoopbackSocketFactory, serverLoopbackSocketFactory) + UnicastRemoteObject.exportObject(this, port, LoopbackNetworkInterface.clientLoopbackSocketFactory, LoopbackNetworkInterface.serverLoopbackSocketFactory) } public fun disconnect() { diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt index 0f04bd33b10..f61ab210dc9 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt @@ -22,7 +22,6 @@ import java.lang.management.ManagementFactory import java.security.DigestInputStream import java.security.MessageDigest import kotlin.reflect.KMutableProperty1 -import kotlin.text.Regex public val COMPILER_JAR_NAME: String = "kotlin-compiler.jar" @@ -54,9 +53,7 @@ public val COMPILE_DAEMON_DEFAULT_RUN_DIR_PATH: String get() = val COMPILER_ID_DIGEST = "MD5" -public fun makeRunFilenameString(ts: String, digest: String, port: String, esc: String = ""): String = "$COMPILE_DAEMON_DEFAULT_FILES_PREFIX$esc.$ts$esc.$digest$esc.$port$esc.run" - -public fun makeRunFilenameRegex(ts: String = "[0-9TZ:\\.\\+-]+", digest: String = "[0-9a-f]+", port: String = "\\d+"): Regex = makeRunFilenameString(ts, digest, port, esc = "\\").toRegex() +public fun makeRunFilenameString(timestamp: String, digest: String, port: String, escapeSequence: String = ""): String = "$COMPILE_DAEMON_DEFAULT_FILES_PREFIX$escapeSequence.$timestamp$escapeSequence.$digest$escapeSequence.$port$escapeSequence.run" open class PropMapper>(val dest: C, @@ -114,6 +111,8 @@ class RestPropMapper>>(des } +// helper function combining find with map, useful for the cases then there is a calculation performed in find, which is nice to return along with +// found value; mappingPredicate should return the pair of boolean compare predicate result and transformation value, we want to get along with found value inline fun Iterable.findWithTransform(mappingPredicate: (T) -> Pair): R? { for (element in this) { val (found, mapped) = mappingPredicate(element) @@ -123,6 +122,9 @@ inline fun Iterable.findWithTransform(mappingPredicate: (T) -> P } +// filter-like function, takes list of propmappers, bound to properties of concrete objects, iterates over receiver, extract matching values via appropriate +// mappers into bound properties; if restParser is given, adds all non-matching elements to it, otherwise return them as an iterable +// note bound properties mutation! fun Iterable.filterExtractProps(propMappers: List>, prefix: String, restParser: RestPropMapper<*, *>? = null): Iterable { val iter = iterator() @@ -169,6 +171,9 @@ fun Iterable.filterExtractProps(propMappers: List>, } +public fun String.trimQuotes() = trim('"','\'') + + public interface OptionsGroup : Serializable { public val mappers: List> } @@ -203,10 +208,10 @@ public data class DaemonOptions( ) : OptionsGroup { override val mappers: List> - get() = listOf(PropMapper(this, ::runFilesPath, fromString = { it.trim('"') }), + get() = listOf(PropMapper(this, ::runFilesPath, fromString = { it.trimQuotes() }), PropMapper(this, ::autoshutdownMemoryThreshold, fromString = { it.toLong() }, skipIf = { it == 0L }, mergeDelimiter = "="), PropMapper(this, ::autoshutdownIdleSeconds, fromString = { it.toInt() }, skipIf = { it == 0 }, mergeDelimiter = "="), - NullablePropMapper(this, ::clientAliveFlagPath, fromString = { it }, toString = { "${it?.trim('\"', '\'')}" }, mergeDelimiter = "=")) + NullablePropMapper(this, ::clientAliveFlagPath, fromString = { it }, toString = { "${it?.trimQuotes()}" }, mergeDelimiter = "=")) } @@ -257,7 +262,7 @@ public data class CompilerId( ) : OptionsGroup { override val mappers: List> - get() = listOf(PropMapper(this, ::compilerClasspath, toString = { it.joinToString(File.pathSeparator) }, fromString = { it.trim('"').split(File.pathSeparator) }), + get() = listOf(PropMapper(this, ::compilerClasspath, toString = { it.joinToString(File.pathSeparator) }, fromString = { it.trimQuotes().split(File.pathSeparator) }), StringPropMapper(this, ::compilerDigest), StringPropMapper(this, ::compilerVersion)) @@ -283,10 +288,11 @@ public fun configureDaemonJVMOptions(opts: DaemonJVMOptions, inheritMemoryLimits ManagementFactory.getRuntimeMXBean().inputArguments.filterExtractProps(opts.mappers, "-") } System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)?.let { - opts.jvmParams.addAll(it.trim('"', '\'') - .split("(?>( // ignoring if object already exported } - val stub = UnicastRemoteObject.exportObject(this, port, clientLoopbackSocketFactory, serverLoopbackSocketFactory) as CompileService + val stub = UnicastRemoteObject.exportObject(this, port, LoopbackNetworkInterface.clientLoopbackSocketFactory, LoopbackNetworkInterface.serverLoopbackSocketFactory) as CompileService registry.rebind (COMPILER_SERVICE_RMI_NAME, stub); alive = true } diff --git a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt index 2a8c94f793a..43efba74df2 100644 --- a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt +++ b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt @@ -19,10 +19,7 @@ package org.jetbrains.kotlin.daemon import junit.framework.TestCase import org.jetbrains.kotlin.cli.CliBaseTest import org.jetbrains.kotlin.integration.KotlinIntegrationTestBase -import org.jetbrains.kotlin.rmi.CompileService -import org.jetbrains.kotlin.rmi.CompilerId -import org.jetbrains.kotlin.rmi.DaemonJVMOptions -import org.jetbrains.kotlin.rmi.DaemonOptions +import org.jetbrains.kotlin.rmi.* import org.jetbrains.kotlin.rmi.kotlinr.DaemonReportingTargets import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient import org.jetbrains.kotlin.test.JetTestUtils @@ -75,4 +72,33 @@ public class CompilerDaemonTest : KotlinIntegrationTestBase() { runDaemonCompilerTwice("hello.compile", "-include-runtime", File(getTestBaseDir(), "hello.kt").absolutePath, "-d", jar) run("hello.run", "-cp", jar, "Hello.HelloPackage") } + + public fun testDaemonJvmOptionsParsing() { + val backupJvmOptions = System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY) + try { + System.setProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY, "-aaa,-bbb\\,ccc,-ddd,-Xmx200m,-XX:MaxPermSize=10k,-XX:ReservedCodeCacheSize=100,-xxx\\,yyy") + val opts = configureDaemonJVMOptions(inheritMemoryLimits = false) + TestCase.assertEquals("200m", opts.maxMemory) + TestCase.assertEquals("10k", opts.maxPermSize) + TestCase.assertEquals("100", opts.reservedCodeCacheSize) + TestCase.assertEquals(arrayListOf("aaa", "bbb,ccc", "ddd", "xxx,yyy"), opts.jvmParams) + } + finally { + backupJvmOptions?.let { System.setProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY, it) } + } + } + + public fun testDaemonOptionsParsing() { + val backupJvmOptions = System.getProperty(COMPILE_DAEMON_OPTIONS_PROPERTY) + try { + System.setProperty(COMPILE_DAEMON_OPTIONS_PROPERTY, "runFilesPath=abcd,clientAliveFlagPath=efgh,autoshutdownIdleSeconds=1111") + val opts = configureDaemonOptions() + TestCase.assertEquals("abcd", opts.runFilesPath) + TestCase.assertEquals("efgh", opts.clientAliveFlagPath) + TestCase.assertEquals(1111, opts.autoshutdownIdleSeconds) + } + finally { + backupJvmOptions?.let { System.setProperty(COMPILE_DAEMON_OPTIONS_PROPERTY, it) } + } + } }