Refactorings, reformatting code, applying code style and other cleanup

This commit is contained in:
Ilya Chernikov
2015-09-08 17:28:31 +02:00
parent d448602cb2
commit 96558c52ff
7 changed files with 570 additions and 576 deletions
@@ -26,6 +26,355 @@ import java.util.concurrent.Semaphore
import java.util.concurrent.TimeUnit
import kotlin.concurrent.thread
public object KotlinCompilerClient {
val DAEMON_DEFAULT_STARTUP_TIMEOUT_MS = 10000L
val DAEMON_CONNECT_CYCLE_ATTEMPTS = 3
// TODO: remove jvmStatic after all use sites will switch to kotlin
jvmStatic
public fun connectToCompileService(compilerId: CompilerId,
daemonJVMOptions: DaemonJVMOptions,
daemonOptions: DaemonOptions,
reportingTargets: DaemonReportingTargets,
autostart: Boolean = true,
checkId: Boolean = true
): CompileService? {
var attempts = 0
var fileLock: FileBasedLock? = null
try {
while (attempts++ < DAEMON_CONNECT_CYCLE_ATTEMPTS) {
val service = tryFindDaemon(File(daemonOptions.runFilesPath), compilerId, reportingTargets)
if (service != null) {
if (!checkId || checkCompilerId(service, compilerId)) {
reportingTargets.report(DaemonReportCategory.DEBUG, "connected to the daemon")
return service
}
reportingTargets.report(DaemonReportCategory.DEBUG, "compiler identity don't match: " + compilerId.mappers.flatMap { it.toArgs("") }.joinToString(" "))
if (!autostart) return null
reportingTargets.report(DaemonReportCategory.DEBUG, "shutdown the daemon")
service.shutdown()
// TODO: find more reliable way
Thread.sleep(1000)
reportingTargets.report(DaemonReportCategory.DEBUG, "daemon shut down correctly, restarting search")
}
else {
if (!autostart) return null
reportingTargets.report(DaemonReportCategory.DEBUG, "no suitable daemon found, starting a new one")
}
if (fileLock == null || !fileLock.isLocked()) {
File(daemonOptions.runFilesPath).mkdirs()
fileLock = FileBasedLock(compilerId, daemonOptions)
// need to check the daemons again here, because of possible racing conditions
// note: the algorithm could be simpler if we'll acquire lock right from the beginning, but it may be costly
attempts--
}
else {
startDaemon(compilerId, daemonJVMOptions, daemonOptions, reportingTargets)
reportingTargets.report(DaemonReportCategory.DEBUG, "daemon started, trying to resolve")
}
}
}
catch (e: Exception) {
reportingTargets.report(DaemonReportCategory.EXCEPTION, e.toString())
}
finally {
fileLock?.release()
}
return null
}
public fun shutdownCompileService(daemonOptions: DaemonOptions): Unit {
KotlinCompilerClient.connectToCompileService(CompilerId(), DaemonJVMOptions(), daemonOptions, DaemonReportingTargets(out = System.out), autostart = false, checkId = false)
?.shutdown()
}
public fun shutdownCompileService(): Unit {
shutdownCompileService(DaemonOptions())
}
public fun compile(compiler: CompileService, args: Array<out String>, out: OutputStream): Int {
val outStrm = RemoteOutputStreamServer(out)
try {
return compiler.remoteCompile(args, outStrm, CompileService.OutputFormat.PLAIN, outStrm)
}
finally {
outStrm.disconnect()
}
}
// TODO: remove jvmStatic after all use sites will switch to kotlin
jvmStatic
public fun incrementalCompile(compiler: CompileService, args: Array<out String>, caches: Map<TargetId, IncrementalCache>, compilerOut: OutputStream, daemonOut: OutputStream): Int {
val compilerOutStreamServer = RemoteOutputStreamServer(compilerOut)
val daemonOutStreamServer = RemoteOutputStreamServer(daemonOut)
val cacheServers = hashMapOf<TargetId, RemoteIncrementalCacheServer>()
try {
caches.mapValuesTo(cacheServers, { RemoteIncrementalCacheServer(it.getValue()) })
return compiler.remoteIncrementalCompile(args, cacheServers, compilerOutStreamServer, CompileService.OutputFormat.XML, daemonOutStreamServer)
}
finally {
cacheServers.forEach { it.getValue().disconnect() }
compilerOutStreamServer.disconnect()
daemonOutStreamServer.disconnect()
}
}
data class ClientOptions(
public var stop: Boolean = false
) : OptionsGroup {
override val mappers: List<PropMapper<*, *, *>>
get() = listOf(BoolPropMapper(this, ::stop))
}
jvmStatic public fun main(vararg args: String) {
val compilerId = CompilerId()
val daemonOptions = DaemonOptions()
val daemonLaunchingOptions = DaemonJVMOptions()
val clientOptions = ClientOptions()
val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, daemonLaunchingOptions, clientOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX)
if (!clientOptions.stop) {
if (compilerId.compilerClasspath.none()) {
// attempt to find compiler to use
System.err.println("compiler wasn't explicitly specified, attempt to find appropriate jar")
System.getProperty("java.class.path")
?.split(File.pathSeparator)
?.map { File(it).parentFile }
?.distinct()
?.map {
it?.walk()
?.firstOrNull { it.name.equals(COMPILER_JAR_NAME, ignoreCase = true) }
}
?.filterNotNull()
?.firstOrNull()
?.let { compilerId.compilerClasspath = listOf(it.absolutePath) }
}
if (compilerId.compilerClasspath.none())
throw IllegalArgumentException("Cannot find compiler jar")
else
println("desired compiler classpath: " + compilerId.compilerClasspath.joinToString(File.pathSeparator))
compilerId.updateDigest()
}
val daemon = connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, DaemonReportingTargets(out = System.out), autostart = !clientOptions.stop, checkId = !clientOptions.stop)
if (daemon == null) {
if (clientOptions.stop) {
System.err.println("No daemon found to shut down")
}
else throw Exception("Unable to connect to daemon")
}
else when {
clientOptions.stop -> {
println("Shutdown the daemon")
daemon.shutdown()
println("Daemon shut down successfully")
}
else -> {
println("Executing daemon compilation with args: " + filteredArgs.joinToString(" "))
val outStrm = RemoteOutputStreamServer(System.out)
try {
val memBefore = daemon.getUsedMemory() / 1024
val startTime = System.nanoTime()
val res = daemon.remoteCompile(filteredArgs.toArrayList().toTypedArray(), outStrm, CompileService.OutputFormat.PLAIN, outStrm)
val endTime = System.nanoTime()
println("Compilation result code: $res")
val memAfter = daemon.getUsedMemory() / 1024
println("Compilation time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms")
println("Used memory $memAfter (${"%+d".format(memAfter - memBefore)} kb)")
}
finally {
outStrm.disconnect()
}
}
}
}
// --- Implementation ---------------------------------------
val verboseReporting = System.getProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) != null
fun DaemonReportingTargets.report(category: DaemonReportCategory, message: String, source: String = "daemon client") {
if (category == DaemonReportCategory.DEBUG && !verboseReporting) return
out?.println("[$source] ${category.name()}: $message")
messages?.add(DaemonReportMessage(category, "[$source] $message"))
}
private fun tryFindDaemon(registryDir: File, compilerId: CompilerId, reportingTargets: DaemonReportingTargets): CompileService? {
val classPathDigest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest()
val daemons = registryDir.walk()
.map { Pair(it, makeRunFilenameRegex(digest = classPathDigest, port = "(\\d+)").match(it.name)?.groups?.get(1)?.value?.toInt() ?: 0) }
.filter { it.second != 0 }
.map {
reportingTargets.report(DaemonReportCategory.DEBUG, "found suitable daemon on port ${it.second}, trying to connect")
val daemon = tryConnectToDaemon(it.second, reportingTargets)
// cleaning orphaned file; note: daemon should shut itself down if it detects that the run file is deleted
if (daemon == null && !it.first.delete()) {
reportingTargets.report(DaemonReportCategory.INFO, "WARNING: unable to delete seemingly orphaned file '${it.first.absolutePath}', cleanup recommended")
}
daemon
}
.filterNotNull()
.toList()
return when (daemons.size()) {
0 -> null
1 -> daemons.first()
else -> throw IllegalStateException("Multiple daemons serving the same compiler, reset with the cleanup required")
// TODO: consider implementing automatic recovery instead, e.g. getting the youngest or least used daemon and shut down others
}
}
private fun tryConnectToDaemon(port: Int, reportingTargets: DaemonReportingTargets): CompileService? {
try {
val daemon = LocateRegistry.getRegistry(loopbackAddrName, port)
?.lookup(COMPILER_SERVICE_RMI_NAME)
if (daemon != null)
return daemon as? CompileService ?:
throw ClassCastException("Unable to cast compiler service, actual class received: ${daemon.javaClass}")
reportingTargets.report(DaemonReportCategory.EXCEPTION, "daemon not found")
}
catch (e: ConnectException) {
reportingTargets.report(DaemonReportCategory.EXCEPTION, "cannot connect to registry: " + (e.getCause()?.getMessage() ?: e.getMessage() ?: "unknown exception"))
// ignoring it - processing below
}
return null
}
private fun startDaemon(compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, daemonOptions: DaemonOptions, reportingTargets: DaemonReportingTargets) {
val javaExecutable = File(System.getProperty("java.home"), "bin").let {
val javaw = File(it, "javaw.exe")
// TODO: doesn't seem reliable enough, consider more checks if OS is of windows flavor, etc.
if (javaw.exists() && javaw.isFile && javaw.canExecute()) javaw else File(it, "java")
}
val args = listOf(javaExecutable.absolutePath,
"-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator)) +
daemonJVMOptions.mappers.flatMap { it.toArgs("-") } +
COMPILER_DAEMON_CLASS_FQN +
daemonOptions.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } +
compilerId.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) }
reportingTargets.report(DaemonReportCategory.DEBUG, "starting the daemon as: " + args.joinToString(" "))
val processBuilder = ProcessBuilder(args).redirectErrorStream(true)
// assuming daemon process is deaf and (mostly) silent, so do not handle streams
val daemon = processBuilder.start()
var isEchoRead = Semaphore(1)
isEchoRead.acquire()
val stdoutThread =
thread {
daemon.inputStream
.reader()
.forEachLine {
if (daemonOptions.runFilesPath.isNotEmpty() && it.contains(daemonOptions.runFilesPath)) {
isEchoRead.release()
return@forEachLine
}
reportingTargets.report(DaemonReportCategory.DEBUG, it, "daemon")
}
}
try {
// trying to wait for process
val daemonStartupTimeout = System.getProperty(COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY)?.let {
try {
it.toLong()
}
catch (e: Exception) {
reportingTargets.report(DaemonReportCategory.INFO, "unable to interpret $COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY property ('$it'); using default timeout $DAEMON_DEFAULT_STARTUP_TIMEOUT_MS ms")
null
}
} ?: DAEMON_DEFAULT_STARTUP_TIMEOUT_MS
if (daemonOptions.runFilesPath.isNotEmpty()) {
val succeeded = isEchoRead.tryAcquire(daemonStartupTimeout, TimeUnit.MILLISECONDS)
if (!daemon.isAlive())
throw Exception("Daemon terminated unexpectedly")
if (!succeeded)
throw Exception("Unable to get response from daemon in $daemonStartupTimeout ms")
}
else
// without startEcho defined waiting for max timeout
Thread.sleep(daemonStartupTimeout)
}
finally {
// assuming that all important output is already done, the rest should be routed to the log by the daemon itself
if (stdoutThread.isAlive) {
// TODO: find better method to stop the thread, but seems it will require asynchronous consuming of the stream
stdoutThread.stop()
}
}
}
private fun checkCompilerId(compiler: CompileService, localId: CompilerId): Boolean {
val remoteId = compiler.getCompilerId()
return (localId.compilerVersion.isEmpty() || localId.compilerVersion == remoteId.compilerVersion) &&
(localId.compilerClasspath.all { remoteId.compilerClasspath.contains(it) }) &&
(localId.compilerDigest.isEmpty() || remoteId.compilerDigest.isEmpty() || localId.compilerDigest == remoteId.compilerDigest)
}
class FileBasedLock(compilerId: CompilerId, daemonOptions: DaemonOptions) {
private val lockFile: File =
File(daemonOptions.runFilesPath,
makeRunFilenameString(ts = "lock",
digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest(),
port = "0"))
private var locked: Boolean = acquireLockFile(lockFile)
public fun isLocked(): Boolean = locked
synchronized public fun release(): Unit {
if (locked) {
lock?.release()
channel?.close()
lockFile.delete()
locked = false
}
}
private val channel = if (locked) RandomAccessFile(lockFile, "rw").channel else null
private val lock = channel?.lock()
synchronized private fun acquireLockFile(lockFile: File): Boolean {
if (lockFile.createNewFile()) return true
try {
// attempt to delete if file is orphaned
if (lockFile.delete() && lockFile.createNewFile())
return true // if orphaned file deleted assuming that the probability of
}
catch (e: IOException) {
// Ignoring it - assuming it is another client process owning it
}
var attempts = 0L
while (lockFile.exists() && attempts++ * COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_CHECK_MS < COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_MS) {
Thread.sleep(COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_CHECK_MS)
}
if (lockFile.exists())
throw IOException("Timeout waiting the release of the lock file '${lockFile.absolutePath}")
return lockFile.createNewFile()
}
}
}
fun Process.isAlive() =
try {
this.exitValue()
@@ -43,339 +392,3 @@ public enum class DaemonReportCategory {
public data class DaemonReportMessage(public val category: DaemonReportCategory, public val message: String)
public class DaemonReportingTargets(public val out: PrintStream? = null, public val messages: MutableCollection<DaemonReportMessage>? = null)
public class KotlinCompilerClient {
companion object {
val DAEMON_DEFAULT_STARTUP_TIMEOUT_MS = 10000L
val DAEMON_CONNECT_CYCLE_ATTEMPTS = 3
val verboseReporting = System.getProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) != null
fun DaemonReportingTargets.report(category: DaemonReportCategory, message: String, source: String = "daemon client") {
if (category == DaemonReportCategory.DEBUG && !verboseReporting) return
out?.println("[$source] ${category.name()}: $message")
messages?.add(DaemonReportMessage(category, "[$source] $message"))
}
private fun tryFindDaemon(registryDir: File, compilerId: CompilerId, reportingTargets: DaemonReportingTargets): CompileService? {
val classPathDigest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest()
val daemons = registryDir.walk()
.map { Pair(it, makeRunFilenameRegex(digest = classPathDigest, port = "(\\d+)").match(it.name)?.groups?.get(1)?.value?.toInt() ?: 0) }
.filter { it.second != 0 }
.map {
reportingTargets.report(DaemonReportCategory.DEBUG, "found suitable daemon on port ${it.second}, trying to connect")
val daemon = tryConnectToDaemon(it.second, reportingTargets)
// cleaning orphaned file; note: daemon should shut itself down if it detects that the run file is deleted
if (daemon == null && !it.first.delete()) {
reportingTargets.report(DaemonReportCategory.INFO, "WARNING: unable to delete seemingly orphaned file '${it.first.absolutePath}', cleanup recommended")
}
daemon
}
.filterNotNull()
.toList()
return when (daemons.size()) {
0 -> null
1 -> daemons.first()
else -> throw IllegalStateException("Multiple daemons serving the same compiler, reset with the cleanup required")
// TODO: consider implementing automatic recovery instead, e.g. getting the youngest or least used daemon and shut down others
}
}
private fun tryConnectToDaemon(port: Int, reportingTargets: DaemonReportingTargets): CompileService? {
try {
val daemon = LocateRegistry.getRegistry(loopbackAddrName, port)
?.lookup(COMPILER_SERVICE_RMI_NAME)
if (daemon != null)
return daemon as? CompileService ?:
throw ClassCastException("Unable to cast compiler service, actual class received: ${daemon.javaClass}")
reportingTargets.report(DaemonReportCategory.EXCEPTION, "daemon not found")
}
catch (e: ConnectException) {
reportingTargets.report(DaemonReportCategory.EXCEPTION, "cannot connect to registry: " + (e.getCause()?.getMessage() ?: e.getMessage() ?: "unknown exception"))
// ignoring it - processing below
}
return null
}
private fun startDaemon(compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, daemonOptions: DaemonOptions, reportingTargets: DaemonReportingTargets) {
val javaExecutable = File(System.getProperty("java.home"), "bin").let {
val javaw = File(it, "javaw.exe")
if (javaw.exists()) javaw
else File(it, "java")
}
// TODO: add some specific environment variables to the cp and may be command line, to allow some specific startup configs
val args = listOf(javaExecutable.absolutePath,
"-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator)) +
daemonJVMOptions.mappers.flatMap { it.toArgs("-") } +
COMPILER_DAEMON_CLASS_FQN +
daemonOptions.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } +
compilerId.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) }
reportingTargets.report(DaemonReportCategory.DEBUG, "starting the daemon as: " + args.joinToString(" "))
val processBuilder = ProcessBuilder(args).redirectErrorStream(true)
// assuming daemon process is deaf and (mostly) silent, so do not handle streams
val daemon = processBuilder.start()
var isEchoRead = Semaphore(1)
isEchoRead.acquire()
val stdoutThread =
thread {
daemon.getInputStream()
.reader()
.forEachLine {
if (daemonOptions.runFilesPath.isNotEmpty() && it.contains(daemonOptions.runFilesPath)) {
isEchoRead.release()
return@forEachLine
}
reportingTargets.report(DaemonReportCategory.DEBUG, it, "daemon")
}
}
try {
// trying to wait for process
val daemonStartupTimeout = System.getProperty(COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY)?.let {
try {
it.toLong()
}
catch (e: Exception) {
reportingTargets.report(DaemonReportCategory.INFO, "unable to interpret $COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY property ('$it'); using default timeout $DAEMON_DEFAULT_STARTUP_TIMEOUT_MS ms")
null
}
} ?: DAEMON_DEFAULT_STARTUP_TIMEOUT_MS
if (daemonOptions.runFilesPath.isNotEmpty()) {
val succeeded = isEchoRead.tryAcquire(daemonStartupTimeout, TimeUnit.MILLISECONDS)
if (!daemon.isAlive())
throw Exception("Daemon terminated unexpectedly")
if (!succeeded)
throw Exception("Unable to get response from daemon in $daemonStartupTimeout ms")
}
else
// without startEcho defined waiting for max timeout
Thread.sleep(daemonStartupTimeout)
}
finally {
// assuming that all important output is already done, the rest should be routed to the log by the daemon itself
if (stdoutThread.isAlive)
// TODO: find better method to stop the thread, but seems it will require asynchronous consuming of the stream
stdoutThread.stop()
}
}
public fun checkCompilerId(compiler: CompileService, localId: CompilerId): Boolean {
val remoteId = compiler.getCompilerId()
return (localId.compilerVersion.isEmpty() || localId.compilerVersion == remoteId.compilerVersion) &&
(localId.compilerClasspath.all { remoteId.compilerClasspath.contains(it) }) &&
(localId.compilerDigest.isEmpty() || remoteId.compilerDigest.isEmpty() || localId.compilerDigest == remoteId.compilerDigest)
}
class FileBasedLock(compilerId: CompilerId, daemonOptions: DaemonOptions) {
private val lockFile: File =
File(daemonOptions.runFilesPath,
makeRunFilenameString(ts = "lock",
digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest(),
port = "0"))
private var locked: Boolean = acquireLockFile(lockFile)
public fun isLocked(): Boolean = locked
synchronized public fun release(): Unit {
if (locked) {
lock?.release()
channel?.close()
lockFile.delete()
locked = false
}
}
private val channel = if (locked) RandomAccessFile(lockFile, "rw").channel else null
private val lock = channel?.lock()
synchronized private fun acquireLockFile(lockFile: File): Boolean {
if (lockFile.createNewFile()) return true
try {
// attempt to delete if file is orphaned
if (lockFile.delete() && lockFile.createNewFile())
return true // if orphaned file deleted assuming that the probability of
}
catch (e: IOException) {
// Ignoring it - assuming it is another client process owning it
}
var attempts = 0L
while (lockFile.exists() && attempts++ * COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_CHECK_MS < COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_MS) {
Thread.sleep(COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_CHECK_MS)
}
if (lockFile.exists())
throw IOException("Timeout waiting the release of the lock file '${lockFile.absolutePath}")
return lockFile.createNewFile()
}
}
public fun connectToCompileService(compilerId: CompilerId,
daemonJVMOptions: DaemonJVMOptions,
daemonOptions: DaemonOptions,
reportingTargets: DaemonReportingTargets,
autostart: Boolean = true,
checkId: Boolean = true
): CompileService? {
var attempts = 0
var fileLock: FileBasedLock? = null
try {
while (attempts++ < DAEMON_CONNECT_CYCLE_ATTEMPTS) {
val service = tryFindDaemon(File(daemonOptions.runFilesPath), compilerId, reportingTargets)
if (service != null) {
if (!checkId || checkCompilerId(service, compilerId)) {
reportingTargets.report(DaemonReportCategory.DEBUG, "connected to the daemon")
return service
}
reportingTargets.report(DaemonReportCategory.DEBUG, "compiler identity don't match: " + compilerId.mappers.flatMap { it.toArgs("") }.joinToString(" "))
if (!autostart) return null
reportingTargets.report(DaemonReportCategory.DEBUG, "shutdown the daemon")
service.shutdown()
// TODO: find more reliable way
Thread.sleep(1000)
reportingTargets.report(DaemonReportCategory.DEBUG, "daemon shut down correctly, restarting search")
}
else {
if (!autostart) return null
reportingTargets.report(DaemonReportCategory.DEBUG, "no suitable daemon found, starting a new one")
}
if (fileLock == null || !fileLock.isLocked()) {
File(daemonOptions.runFilesPath).mkdirs()
fileLock = FileBasedLock(compilerId, daemonOptions)
// need to check the daemons again here, because of possible racing conditions
// note: the algorithm could be simpler if we'll acquire lock right from the beginning, but it may be costly
attempts--
}
else {
startDaemon(compilerId, daemonJVMOptions, daemonOptions, reportingTargets)
reportingTargets.report(DaemonReportCategory.DEBUG, "daemon started, trying to resolve")
}
}
}
catch (e: Exception) {
reportingTargets.report(DaemonReportCategory.EXCEPTION, e.toString())
}
finally {
fileLock?.release()
}
return null
}
public fun shutdownCompileService(daemonOptions: DaemonOptions): Unit {
KotlinCompilerClient.connectToCompileService(CompilerId(), DaemonJVMOptions(), daemonOptions, DaemonReportingTargets(out = System.out), autostart = false, checkId = false)
?.shutdown()
}
public fun shutdownCompileService(): Unit {
shutdownCompileService(DaemonOptions())
}
public fun compile(compiler: CompileService, args: Array<out String>, out: OutputStream): Int {
val outStrm = RemoteOutputStreamServer(out)
try {
return compiler.remoteCompile(args, outStrm, CompileService.OutputFormat.PLAIN, outStrm)
}
finally {
outStrm.disconnect()
}
}
public fun incrementalCompile(compiler: CompileService, args: Array<out String>, caches: Map<TargetId, IncrementalCache>, compilerOut: OutputStream, daemonOut: OutputStream): Int {
val compilerOutStreamServer = RemoteOutputStreamServer(compilerOut)
val daemonOutStreamServer = RemoteOutputStreamServer(daemonOut)
val cacheServers = hashMapOf<TargetId, RemoteIncrementalCacheServer>()
try {
caches.mapValuesTo(cacheServers, { RemoteIncrementalCacheServer( it.getValue()) })
return compiler.remoteIncrementalCompile(args, cacheServers, compilerOutStreamServer, CompileService.OutputFormat.XML, daemonOutStreamServer)
}
finally {
cacheServers.forEach { it.getValue().disconnect() }
compilerOutStreamServer.disconnect()
daemonOutStreamServer.disconnect()
}
}
data class ClientOptions(
public var stop: Boolean = false
) : OptionsGroup {
override val mappers: List<PropMapper<*, *, *>>
get() = listOf( BoolPropMapper(this, ::stop))
}
jvmStatic public fun main(vararg args: String) {
val compilerId = CompilerId()
val daemonOptions = DaemonOptions()
val daemonLaunchingOptions = DaemonJVMOptions()
val clientOptions = ClientOptions()
val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, daemonLaunchingOptions, clientOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX)
if (!clientOptions.stop) {
if (compilerId.compilerClasspath.none()) {
// attempt to find compiler to use
System.err.println("compiler wasn't explicitly specified, attempt to find appropriate jar")
System.getProperty("java.class.path")
?.split(File.pathSeparator)
?.map { File(it).parentFile }
?.distinct()
?.map {
it?.walk()
?.firstOrNull { it.getName().equals(COMPILER_JAR_NAME, ignoreCase = true) }
}
?.filterNotNull()
?.firstOrNull()
?.let { compilerId.compilerClasspath = listOf(it.absolutePath) }
}
if (compilerId.compilerClasspath.none())
throw IllegalArgumentException("Cannot find compiler jar")
else
println("desired compiler classpath: " + compilerId.compilerClasspath.joinToString(File.pathSeparator))
compilerId.updateDigest()
}
val daemon = connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, DaemonReportingTargets(out = System.out), autostart = !clientOptions.stop, checkId = !clientOptions.stop)
if (daemon == null) {
if (clientOptions.stop) System.err.println("No daemon found to shut down")
else throw Exception("Unable to connect to daemon")
}
else when {
clientOptions.stop -> {
println("Shutdown the daemon")
daemon.shutdown()
println("Daemon shut down successfully")
}
else -> {
println("Executing daemon compilation with args: " + filteredArgs.joinToString(" "))
val outStrm = RemoteOutputStreamServer(System.out)
try {
val memBefore = daemon.getUsedMemory() / 1024
val startTime = System.nanoTime()
val res = daemon.remoteCompile(filteredArgs.toArrayList().toTypedArray(), outStrm, CompileService.OutputFormat.PLAIN, outStrm)
val endTime = System.nanoTime()
println("Compilation result code: $res")
val memAfter = daemon.getUsedMemory() / 1024
println("Compilation time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms")
println("Used memory $memAfter (${"%+d".format(memAfter - memBefore)} kb)")
}
finally {
outStrm.disconnect()
}
}
}
}
}
}
@@ -40,7 +40,7 @@ public val COMPILE_DAEMON_CLIENT_ALIVE_PATH_PROPERTY: String = "kotlin.daemon.cl
public val COMPILE_DAEMON_REPORT_PERF_PROPERTY: String = "kotlin.daemon.perf"
public val COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY: String = "kotlin.daemon.verbose"
public val COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX: String = "--daemon-"
public val COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY: String ="kotlin.daemon.startup.timeout"
public val COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY: String = "kotlin.daemon.startup.timeout"
public val COMPILE_DAEMON_DEFAULT_FILES_PREFIX: String = "kotlin-daemon"
public val COMPILE_DAEMON_DATA_DIRECTORY_NAME: String = "." + COMPILE_DAEMON_DEFAULT_FILES_PREFIX
public val COMPILE_DAEMON_TIMEOUT_INFINITE_S: Int = 0
@@ -51,67 +51,70 @@ public val COMPILE_DAEMON_DEFAULT_RUN_DIR_PATH: String get() =
// TODO consider special case for windows - local appdata
File(System.getProperty("user.home"), COMPILE_DAEMON_DATA_DIRECTORY_NAME).absolutePath
val COMPILER_ID_DIGEST = "MD5"
public fun makeRunFilenameString(ts: String, digest: String, port: String, esc: String = ""): String = "$COMPILE_DAEMON_DEFAULT_FILES_PREFIX$esc.$ts$esc.$digest$esc.$port$esc.run"
public fun makeRunFilenameRegex(ts: String = "[0-9TZ:\\.\\+-]+", digest: String = "[0-9a-f]+", port: String = "\\d+"): Regex = makeRunFilenameString(ts, digest, port, esc = "\\").toRegex()
open class PropMapper<C, V, P: KMutableProperty1<C, V>>(val dest: C,
val prop: P,
val names: List<String> = listOf(prop.name),
val fromString: (String) -> V,
val toString: ((V) -> String?) = { it.toString() },
val skipIf: ((V) -> Boolean) = { false },
val mergeDelimiter: String? = null)
{
open class PropMapper<C, V, P : KMutableProperty1<C, V>>(val dest: C,
val prop: P,
val names: List<String> = listOf(prop.name),
val fromString: (String) -> V,
val toString: ((V) -> String?) = { it.toString() },
val skipIf: ((V) -> Boolean) = { false },
val mergeDelimiter: String? = null) {
open fun toArgs(prefix: String = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX): List<String> =
when {
skipIf(prop.get(dest)) -> listOf<String>()
mergeDelimiter != null -> listOf( listOf(prefix + names.first(), toString(prop.get(dest))).filterNotNull().joinToString(mergeDelimiter))
mergeDelimiter != null -> listOf(listOf(prefix + names.first(), toString(prop.get(dest))).filterNotNull().joinToString(mergeDelimiter))
else -> listOf(prefix + names.first(), toString(prop.get(dest))).filterNotNull()
}
open fun apply(s: String) = prop.set(dest, fromString(s))
}
class NullablePropMapper<C, V: Any?, P: KMutableProperty1<C, V>>(dest: C,
prop: P,
names: List<String> = listOf(),
fromString: ((String) -> V),
toString: ((V) -> String?) = { it.toString() },
skipIf: ((V) -> Boolean) = { it == null },
mergeDelimiter: String? = null)
class NullablePropMapper<C, V : Any?, P : KMutableProperty1<C, V>>(dest: C,
prop: P,
names: List<String> = listOf(),
fromString: ((String) -> V),
toString: ((V) -> String?) = { it.toString() },
skipIf: ((V) -> Boolean) = { it == null },
mergeDelimiter: String? = null)
: PropMapper<C, V, P>(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name),
fromString = fromString, toString = toString, skipIf = skipIf, mergeDelimiter = mergeDelimiter)
fromString = fromString, toString = toString, skipIf = skipIf, mergeDelimiter = mergeDelimiter)
class StringPropMapper<C, P: KMutableProperty1<C, String>>(dest: C,
prop: P,
names: List<String> = listOf(),
fromString: ((String) -> String) = { it },
toString: ((String) -> String?) = { it.toString() },
skipIf: ((String) -> Boolean) = { it.isEmpty() },
mergeDelimiter: String? = null)
class StringPropMapper<C, P : KMutableProperty1<C, String>>(dest: C,
prop: P,
names: List<String> = listOf(),
fromString: ((String) -> String) = { it },
toString: ((String) -> String?) = { it.toString() },
skipIf: ((String) -> Boolean) = { it.isEmpty() },
mergeDelimiter: String? = null)
: PropMapper<C, String, P>(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name),
fromString = fromString, toString = toString, skipIf = skipIf, mergeDelimiter = mergeDelimiter)
fromString = fromString, toString = toString, skipIf = skipIf, mergeDelimiter = mergeDelimiter)
class BoolPropMapper<C, P: KMutableProperty1<C, Boolean>>(dest: C, prop: P, names: List<String> = listOf())
: PropMapper<C, Boolean, P>(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name),
fromString = { true }, toString = { null }, skipIf = { !prop.get(dest) })
class BoolPropMapper<C, P : KMutableProperty1<C, Boolean>>(dest: C, prop: P, names: List<String> = listOf())
: PropMapper<C, Boolean, P>(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name),
fromString = { true }, toString = { null }, skipIf = { !prop.get(dest) })
class RestPropMapper<C, P: KMutableProperty1<C, MutableCollection<String>>>(dest: C, prop: P)
: PropMapper<C, MutableCollection<String>, P>(dest = dest, prop = prop, toString = { null }, fromString = { arrayListOf() })
{
class RestPropMapper<C, P : KMutableProperty1<C, MutableCollection<String>>>(dest: C, prop: P)
: PropMapper<C, MutableCollection<String>, P>(dest = dest, prop = prop, toString = { null }, fromString = { arrayListOf() }) {
override fun toArgs(prefix: String): List<String> = prop.get(dest).map { prefix + it }
override fun apply(s: String) = add(s)
fun add(s: String) { prop.get(dest).add(s) }
fun add(s: String) {
prop.get(dest).add(s)
}
}
inline fun <T, R: Any> Iterable<T>.findWithTransform(mappingPredicate: (T) -> Pair<Boolean, R?>): R? {
inline fun <T, R : Any> Iterable<T>.findWithTransform(mappingPredicate: (T) -> Pair<Boolean, R?>): R? {
for (element in this) {
val (found, mapped) = mappingPredicate(element)
if (found) return mapped
@@ -120,7 +123,7 @@ inline fun <T, R: Any> Iterable<T>.findWithTransform(mappingPredicate: (T) -> Pa
}
fun Iterable<String>.filterExtractProps(propMappers: List<PropMapper<*,*,*>>, prefix: String, restParser: RestPropMapper<*,*>? = null) : Iterable<String> {
fun Iterable<String>.filterExtractProps(propMappers: List<PropMapper<*, *, *>>, prefix: String, restParser: RestPropMapper<*, *>? = null): Iterable<String> {
val iter = iterator()
val rest = arrayListOf<String>()
@@ -137,7 +140,7 @@ fun Iterable<String>.filterExtractProps(propMappers: List<PropMapper<*,*,*>>, pr
propMapper != null -> {
val optionLength = prefix.length() + matchingOption!!.length()
when {
propMapper is BoolPropMapper<*,*> -> {
propMapper is BoolPropMapper<*, *> -> {
if (param.length() > optionLength)
throw IllegalArgumentException("Invalid switch option '$param', expecting $prefix$matchingOption without arguments")
propMapper.apply("")
@@ -166,19 +169,11 @@ fun Iterable<String>.filterExtractProps(propMappers: List<PropMapper<*,*,*>>, pr
}
// TODO: find out how to create more generic variant using first constructor
//fun<C> C.propsToParams() {
// val kc = C::class
// kc.constructors.first().
//}
public interface OptionsGroup : Serializable {
public val mappers: List<PropMapper<*,*,*>>
public val mappers: List<PropMapper<*, *, *>>
}
public fun Iterable<String>.filterExtractProps(vararg groups: OptionsGroup, prefix: String) : Iterable<String> =
public fun Iterable<String>.filterExtractProps(vararg groups: OptionsGroup, prefix: String): Iterable<String> =
filterExtractProps(groups.flatMap { it.mappers }, prefix)
@@ -189,13 +184,13 @@ public data class DaemonJVMOptions(
public var jvmParams: MutableCollection<String> = arrayListOf()
) : OptionsGroup {
override val mappers: List<PropMapper<*,*,*>>
get() = listOf( StringPropMapper(this, ::maxMemory, listOf("Xmx"), mergeDelimiter = ""),
StringPropMapper(this, ::maxPermSize, listOf("XX:MaxPermSize"), mergeDelimiter = "="),
StringPropMapper(this, ::reservedCodeCacheSize, listOf("XX:ReservedCodeCacheSize"), mergeDelimiter = "="),
restMapper)
override val mappers: List<PropMapper<*, *, *>>
get() = listOf(StringPropMapper(this, ::maxMemory, listOf("Xmx"), mergeDelimiter = ""),
StringPropMapper(this, ::maxPermSize, listOf("XX:MaxPermSize"), mergeDelimiter = "="),
StringPropMapper(this, ::reservedCodeCacheSize, listOf("XX:ReservedCodeCacheSize"), mergeDelimiter = "="),
restMapper)
val restMapper: RestPropMapper<*,*>
val restMapper: RestPropMapper<*, *>
get() = RestPropMapper(this, ::jvmParams)
}
@@ -208,17 +203,17 @@ public data class DaemonOptions(
) : OptionsGroup {
override val mappers: List<PropMapper<*, *, *>>
get() = listOf( PropMapper(this, ::runFilesPath, fromString = { it.trim('"') }),
PropMapper(this, ::autoshutdownMemoryThreshold, fromString = { it.toLong() }, skipIf = { it == 0L }, mergeDelimiter = "="),
PropMapper(this, ::autoshutdownIdleSeconds, fromString = { it.toInt() }, skipIf = { it == 0 }, mergeDelimiter = "="),
NullablePropMapper(this, ::clientAliveFlagPath, fromString = { it }, toString = { "${it?.trim('\"','\'')}" }, mergeDelimiter = "="))
get() = listOf(PropMapper(this, ::runFilesPath, fromString = { it.trim('"') }),
PropMapper(this, ::autoshutdownMemoryThreshold, fromString = { it.toLong() }, skipIf = { it == 0L }, mergeDelimiter = "="),
PropMapper(this, ::autoshutdownIdleSeconds, fromString = { it.toInt() }, skipIf = { it == 0 }, mergeDelimiter = "="),
NullablePropMapper(this, ::clientAliveFlagPath, fromString = { it }, toString = { "${it?.trim('\"', '\'')}" }, mergeDelimiter = "="))
}
fun updateSingleFileDigest(file: File, md: MessageDigest) {
DigestInputStream(file.inputStream(), md).use {
val buf = ByteArray(1024)
while (it.read(buf) != -1) { }
while (it.read(buf) != -1) {}
it.close()
}
}
@@ -232,10 +227,10 @@ fun updateEntryDigest(entry: File, md: MessageDigest) {
entry.isDirectory
-> updateForAllClasses(entry, md)
entry.isFile &&
(entry.getName().endsWith(".class", ignoreCase = true) ||
entry.getName().endsWith(".jar", ignoreCase = true))
(entry.extension.equals("class", ignoreCase = true) ||
entry.extension.equals("jar", ignoreCase = true))
-> updateSingleFileDigest(entry, md)
// else skip
// else skip
}
}
@@ -250,22 +245,21 @@ jvmName("getFilesClasspathDigest_Strings")
fun Iterable<String>.getFilesClasspathDigest(): String = map { File(it) }.getFilesClasspathDigest()
fun Iterable<String>.distinctStringsDigest(): String =
MessageDigest.getInstance(COMPILER_ID_DIGEST)
.digest(this.distinct().sort().joinToString("").toByteArray())
.joinToString("", transform = { "%02x".format(it) })
MessageDigest.getInstance(COMPILER_ID_DIGEST)
.digest(this.distinct().sorted().joinToString("").toByteArray())
.joinToString("", transform = { "%02x".format(it) })
public data class CompilerId(
public var compilerClasspath: List<String> = listOf(),
public var compilerDigest: String = "",
public var compilerVersion: String = ""
// TODO: checksum
) : OptionsGroup {
override val mappers: List<PropMapper<*, *, *>>
get() = listOf( PropMapper(this, ::compilerClasspath, toString = { it.joinToString(File.pathSeparator) }, fromString = { it.trim('"').split(File.pathSeparator)}),
StringPropMapper(this, ::compilerDigest),
StringPropMapper(this, ::compilerVersion))
get() = listOf(PropMapper(this, ::compilerClasspath, toString = { it.joinToString(File.pathSeparator) }, fromString = { it.trim('"').split(File.pathSeparator) }),
StringPropMapper(this, ::compilerDigest),
StringPropMapper(this, ::compilerVersion))
public fun updateDigest() {
compilerDigest = compilerClasspath.getFilesClasspathDigest()
@@ -275,7 +269,6 @@ public data class CompilerId(
public jvmStatic fun makeCompilerId(vararg paths: File): CompilerId = makeCompilerId(paths.asIterable())
public jvmStatic fun makeCompilerId(paths: Iterable<File>): CompilerId =
// TODO consider reading version here
CompilerId(compilerClasspath = paths.map { it.absolutePath }, compilerDigest = paths.getFilesClasspathDigest())
}
}
@@ -286,14 +279,14 @@ public fun isDaemonEnabled(): Boolean = System.getProperty(COMPILE_DAEMON_ENABLE
public fun configureDaemonJVMOptions(opts: DaemonJVMOptions, inheritMemoryLimits: Boolean): DaemonJVMOptions {
// note: sequence matters, explicit override in COMPILE_DAEMON_JVM_OPTIONS_PROPERTY should be done after inputArguments processing
if (inheritMemoryLimits)
if (inheritMemoryLimits) {
ManagementFactory.getRuntimeMXBean().inputArguments.filterExtractProps(opts.mappers, "-")
}
System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)?.let {
opts.jvmParams.addAll( it.trim('"', '\'')
.split("(?<!\\\\),".toRegex())
.map { it.replace("[\\\\](.)".toRegex(), "$1") }
.filterExtractProps(opts.mappers, "-", opts.restMapper))
opts.jvmParams.addAll(it.trim('"', '\'')
.split("(?<!\\\\),".toRegex())
.map { it.replace("[\\\\](.)".toRegex(), "$1") }
.filterExtractProps(opts.mappers, "-", opts.restMapper))
}
System.getProperty(COMPILE_DAEMON_REPORT_PERF_PROPERTY)?.let { opts.jvmParams.add("D" + COMPILE_DAEMON_REPORT_PERF_PROPERTY) }
@@ -302,7 +295,7 @@ public fun configureDaemonJVMOptions(opts: DaemonJVMOptions, inheritMemoryLimits
}
public fun configureDaemonJVMOptions(inheritMemoryLimits: Boolean): DaemonJVMOptions =
configureDaemonJVMOptions(DaemonJVMOptions(), inheritMemoryLimits = inheritMemoryLimits)
configureDaemonJVMOptions(DaemonJVMOptions(), inheritMemoryLimits = inheritMemoryLimits)
public fun configureDaemonOptions(opts: DaemonOptions): DaemonOptions {
System.getProperty(COMPILE_DAEMON_OPTIONS_PROPERTY)?.let {
@@ -313,7 +306,7 @@ public fun configureDaemonOptions(opts: DaemonOptions): DaemonOptions {
"\nSupported options: " + opts.mappers.joinToString(", ", transform = { it.names.first() }))
}
System.getProperty(COMPILE_DAEMON_CLIENT_ALIVE_PATH_PROPERTY)?.let {
val trimmed = it.trim('"','\'')
val trimmed = it.trim('"', '\'')
if (!trimmed.isBlank()) {
opts.clientAliveFlagPath = trimmed
}
@@ -26,28 +26,23 @@ import java.rmi.server.RMIServerSocketFactory
// TODO switch to InetAddress.getLoopbackAddress on java 7+
val loopbackAddrName = if (java.net.InetAddress.getLocalHost() is java.net.Inet6Address) "::1" else "127.0.0.1"
val loopbackAddrName by lazy { if (java.net.InetAddress.getLocalHost() is java.net.Inet6Address) "::1" else "127.0.0.1" }
val loopbackAddr by lazy { InetAddress.getByName(loopbackAddrName) }
val serverLoopbackSocketFactory by lazy { ServerLoopbackSocketFactory() }
val clientLoopbackSocketFactory by lazy { ClientLoopbackSocketFactory() }
data class ServerLoopbackSocketFactory : RMIServerSocketFactory, Serializable {
throws(IOException::class)
override fun createServerSocket(port: Int): ServerSocket {
return ServerSocket(port, 5, loopbackAddr)
}
override fun createServerSocket(port: Int): ServerSocket = ServerSocket(port, 5, loopbackAddr)
}
data class ClientLoopbackSocketFactory : RMIClientSocketFactory, Serializable {
throws(IOException::class)
override fun createSocket(host: String, port: Int): Socket {
return Socket(loopbackAddr, port)
// just call the default client socket factory
// return RMISocketFactory.getDefaultSocketFactory().createSocket(host, port)
}
override fun createSocket(host: String, port: Int): Socket = Socket(loopbackAddr, port)
}
@@ -18,8 +18,6 @@ package org.jetbrains.kotlin.rmi.service
import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
import org.jetbrains.kotlin.rmi.*
import org.jetbrains.kotlin.service.CompileServiceImpl
import org.jetbrains.kotlin.service.nowSeconds
import java.io.File
import java.io.IOException
import java.io.OutputStream
@@ -57,151 +55,148 @@ class LogStream(name: String) : OutputStream() {
}
public class CompileDaemon {
public object CompileDaemon {
companion object {
init {
val logPath: String = System.getProperty("kotlin.daemon.log.path")?.trimEnd('/','\\') ?: "%t"
val logTime: String = SimpleDateFormat("yyyy-MM-dd.HH-mm-ss-SSS").format(Date())
val cfg: String =
"handlers = java.util.logging.FileHandler\n" +
"java.util.logging.FileHandler.level = ALL\n" +
"java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter\n" +
"java.util.logging.FileHandler.encoding = UTF-8\n" +
"java.util.logging.FileHandler.limit = 1073741824\n" + // 1Mb
"java.util.logging.FileHandler.count = 3\n" +
"java.util.logging.FileHandler.append = false\n" +
"java.util.logging.FileHandler.pattern = $logPath/$COMPILE_DAEMON_DEFAULT_FILES_PREFIX.$logTime.%u%g.log\n" +
"java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s\\n\n"
init {
val logPath: String = System.getProperty("kotlin.daemon.log.path")?.trimEnd('/','\\') ?: "%t"
val logTime: String = SimpleDateFormat("yyyy-MM-dd.HH-mm-ss-SSS").format(Date())
val cfg: String =
"handlers = java.util.logging.FileHandler\n" +
"java.util.logging.FileHandler.level = ALL\n" +
"java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter\n" +
"java.util.logging.FileHandler.encoding = UTF-8\n" +
"java.util.logging.FileHandler.limit = 1073741824\n" + // 1Mb
"java.util.logging.FileHandler.count = 3\n" +
"java.util.logging.FileHandler.append = false\n" +
"java.util.logging.FileHandler.pattern = $logPath/$COMPILE_DAEMON_DEFAULT_FILES_PREFIX.$logTime.%u%g.log\n" +
"java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s\\n\n"
LogManager.getLogManager().readConfiguration(cfg.byteInputStream())
}
LogManager.getLogManager().readConfiguration(cfg.byteInputStream())
}
val log by lazy { Logger.getLogger("daemon") }
val log by lazy { Logger.getLogger("daemon") }
private fun loadVersionFromResource(): String? {
(javaClass.classLoader as? URLClassLoader)
?.findResource("META-INF/MANIFEST.MF")
?.let {
try {
return Manifest(it.openStream()).mainAttributes.getValue("Implementation-Version") ?: ""
}
catch (e: IOException) {}
private fun loadVersionFromResource(): String? {
(CompileDaemon::class.java.classLoader as? URLClassLoader)
?.findResource("META-INF/MANIFEST.MF")
?.let {
try {
return Manifest(it.openStream()).mainAttributes.getValue("Implementation-Version") ?: null
}
return null
catch (e: IOException) {}
}
return null
}
jvmStatic public fun main(args: Array<String>) {
log.info("Kotlin compiler daemon version " + (loadVersionFromResource() ?: "<unknown>"))
log.info("daemon JVM args: " + ManagementFactory.getRuntimeMXBean().inputArguments.joinToString(" "))
log.info("daemon args: " + args.joinToString(" "))
val compilerId = CompilerId()
val daemonOptions = DaemonOptions()
try {
val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX)
if (filteredArgs.any()) {
val helpLine = "usage: <daemon> <compilerId options> <daemon options>"
log.info(helpLine)
println(helpLine)
throw IllegalArgumentException("Unknown arguments: " + filteredArgs.joinToString(" "))
}
log.info("starting daemon")
// TODO: find minimal set of permissions and restore security management
// note: may be not needed anymore since (hopefully) server is now loopback-only
// if (System.getSecurityManager() == null)
// System.setSecurityManager (RMISecurityManager())
//
// setDaemonPpermissions(daemonOptions.port)
val (registry, port) = createRegistry(COMPILE_DAEMON_FIND_PORT_ATTEMPTS)
val runFileDir = File(if (daemonOptions.runFilesPath.isBlank()) COMPILE_DAEMON_DEFAULT_RUN_DIR_PATH else daemonOptions.runFilesPath)
runFileDir.mkdirs()
val runFile = File(runFileDir,
makeRunFilenameString(ts = "%tFT%<tT.%<tLZ".format(Calendar.getInstance(TimeZone.getTimeZone("Z"))),
digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest(),
port = port.toString()))
if (!runFile.createNewFile()) {
throw IllegalStateException("Unable to create run file '${runFile.absolutePath}'")
}
runFile.deleteOnExit()
val compiler = K2JVMCompiler()
val compilerService = CompileServiceImpl(registry, compiler, compilerId, port)
if (daemonOptions.runFilesPath.isNotEmpty())
println(daemonOptions.runFilesPath)
daemonOptions.clientAliveFlagPath?.let {
if (!File(it).exists()) {
log.info("Client alive flag $it do not exist, disable watching it")
daemonOptions.clientAliveFlagPath = null
}
}
// this supposed to stop redirected streams reader(s) on the client side and prevent some situations with hanging threads, but doesn't work reliably
// TODO: implement more reliable scheme
System.out.close()
System.err.close()
System.setErr(PrintStream(LogStream("stderr")))
System.setOut(PrintStream(LogStream("stdout")))
fun shutdownCondition(check: () -> Boolean, message: String): Boolean {
val res = check()
if (res) {
log.info(message)
}
return res
}
// stopping daemon if any shutdown condition is met
timer(initialDelay = DAEMON_PERIODIC_CHECK_INTERVAL_MS, period = DAEMON_PERIODIC_CHECK_INTERVAL_MS) {
val idleSeconds = nowSeconds() - compilerService.lastUsedSeconds
if (shutdownCondition({ !runFile.exists() }, "Run file removed, shutting down") ||
shutdownCondition({ daemonOptions.autoshutdownIdleSeconds != COMPILE_DAEMON_TIMEOUT_INFINITE_S && idleSeconds > daemonOptions.autoshutdownIdleSeconds},
"Idle timeout exceeded ${daemonOptions.autoshutdownIdleSeconds}s, shutting down") ||
shutdownCondition({ daemonOptions.clientAliveFlagPath?.let { !File(it).exists() } ?: false },
"Client alive flag ${daemonOptions.clientAliveFlagPath} removed, shutting down"))
{
cancel()
compilerService.shutdown()
}
}
}
catch (e: Exception) {
System.err.println("Exception: " + e.getMessage())
log.info("Exception" + e.getMessage())
log.info(e.toString())
// TODO consider exiting without throwing
throw e
}
jvmStatic public fun main(args: Array<String>) {
}
log.info("Kotlin compiler daemon version " + (loadVersionFromResource() ?: "<unknown>"))
log.info("daemon JVM args: " + ManagementFactory.getRuntimeMXBean().inputArguments.joinToString(" "))
log.info("daemon args: " + args.joinToString(" "))
val random = Random()
val compilerId = CompilerId()
val daemonOptions = DaemonOptions()
private fun createRegistry(attempts: Int) : Pair<Registry, Int> {
var i = 0
var lastException: RemoteException? = null
while (i++ < attempts) {
val port = random.nextInt(COMPILE_DAEMON_PORTS_RANGE_END - COMPILE_DAEMON_PORTS_RANGE_START) + COMPILE_DAEMON_PORTS_RANGE_START
try {
val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX)
if (filteredArgs.any()) {
val helpLine = "usage: <daemon> <compilerId options> <daemon options>"
log.info(helpLine)
println(helpLine)
throw IllegalArgumentException("Unknown arguments: " + filteredArgs.joinToString(" "))
}
log.info("starting daemon")
// TODO: find minimal set of permissions and restore security management
// note: may be not needed anymore since (hopefully) server is now loopback-only
// if (System.getSecurityManager() == null)
// System.setSecurityManager (RMISecurityManager())
//
// setDaemonPpermissions(daemonOptions.port)
val (registry, port) = createRegistry(COMPILE_DAEMON_FIND_PORT_ATTEMPTS)
val runFileDir = File(if (daemonOptions.runFilesPath.isBlank()) COMPILE_DAEMON_DEFAULT_RUN_DIR_PATH else daemonOptions.runFilesPath)
runFileDir.mkdirs()
val runFile = File(runFileDir,
makeRunFilenameString(ts = "%tFT%<tT.%<tLZ".format(Calendar.getInstance(TimeZone.getTimeZone("Z"))),
digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest(),
port = port.toString()))
if (!runFile.createNewFile()) {
throw IllegalStateException("Unable to create run file '${runFile.absolutePath}'")
}
runFile.deleteOnExit()
val compiler = K2JVMCompiler()
val compilerService = CompileServiceImpl(registry, compiler, compilerId, daemonOptions, port)
if (daemonOptions.runFilesPath.isNotEmpty())
println(daemonOptions.runFilesPath)
daemonOptions.clientAliveFlagPath?.let {
if (!File(it).exists()) {
log.info("Client alive flag $it do not exist, disable watching it")
daemonOptions.clientAliveFlagPath = null
}
}
// this supposed to stop redirected streams reader(s) on the client side and prevent some situations with hanging threads, but doesn't work reliably
// TODO: implement more reliable scheme
System.out.close()
System.err.close()
System.setErr(PrintStream(LogStream("stderr")))
System.setOut(PrintStream(LogStream("stdout")))
fun shutdownCondition(check: () -> Boolean, message: String): Boolean {
val res = check()
if (res) {
log.info(message)
}
return res
}
// stopping daemon if any shutdown condition is met
timer(initialDelay = DAEMON_PERIODIC_CHECK_INTERVAL_MS, period = DAEMON_PERIODIC_CHECK_INTERVAL_MS) {
val idleSeconds = nowSeconds() - compilerService.lastUsedSeconds
if (shutdownCondition({ !runFile.exists() }, "Run file removed, shutting down") ||
shutdownCondition({ daemonOptions.autoshutdownIdleSeconds != COMPILE_DAEMON_TIMEOUT_INFINITE_S && idleSeconds > daemonOptions.autoshutdownIdleSeconds},
"Idle timeout exceeded ${daemonOptions.autoshutdownIdleSeconds}s, shutting down") ||
shutdownCondition({ daemonOptions.clientAliveFlagPath?.let { !File(it).exists() } ?: false },
"Client alive flag ${daemonOptions.clientAliveFlagPath} removed, shutting down"))
{
cancel()
compilerService.shutdown()
}
}
return Pair(LocateRegistry.createRegistry(port, clientLoopbackSocketFactory, serverLoopbackSocketFactory), port)
}
catch (e: Exception) {
System.err.println("Exception: " + e.getMessage())
log.info("Exception" + e.getMessage())
log.info(e.toString())
// TODO consider exiting without throwing
throw e
catch (e: RemoteException) {
// assuming that the port is already taken
lastException = e
}
}
val random = Random()
private fun createRegistry(attempts: Int) : Pair<Registry, Int> {
var i = 0
var lastException: RemoteException? = null
while (i++ < attempts) {
val port = random.nextInt(COMPILE_DAEMON_PORTS_RANGE_END - COMPILE_DAEMON_PORTS_RANGE_START) + COMPILE_DAEMON_PORTS_RANGE_START
try {
return Pair(LocateRegistry.createRegistry(port, clientLoopbackSocketFactory, serverLoopbackSocketFactory), port)
}
catch (e: RemoteException) {
// assuming that the port is already taken
lastException = e
}
}
throw IllegalStateException("Cannot find free port in $attempts attempts", lastException)
}
throw IllegalStateException("Cannot find free port in $attempts attempts", lastException)
}
}
@@ -14,7 +14,7 @@
* limitations under the License.
*/
package org.jetbrains.kotlin.service
package org.jetbrains.kotlin.rmi.service
import org.jetbrains.kotlin.cli.common.CLICompiler
import org.jetbrains.kotlin.cli.common.ExitCode
@@ -24,13 +24,12 @@ import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
import org.jetbrains.kotlin.modules.TargetId
import org.jetbrains.kotlin.rmi.*
import org.jetbrains.kotlin.rmi.service.RemoteIncrementalCacheClient
import org.jetbrains.kotlin.rmi.service.RemoteOutputStreamClient
import java.io.IOException
import java.io.PrintStream
import java.lang.management.ManagementFactory
import java.lang.management.ThreadMXBean
import java.net.URLClassLoader
import java.rmi.NoSuchObjectException
import java.rmi.registry.Registry
import java.rmi.server.UnicastRemoteObject
import java.util.concurrent.TimeUnit
@@ -47,7 +46,6 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
val registry: Registry,
val compiler: Compiler,
val selfCompilerId: CompilerId,
val daemonOptions: DaemonOptions,
port: Int
) : CompileService, UnicastRemoteObject() {
@@ -120,7 +118,7 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
// cleanup for the case of incorrect restart and many other situations
UnicastRemoteObject.unexportObject(this, false)
}
catch (e: java.rmi.NoSuchObjectException) {
catch (e: NoSuchObjectException) {
// ignoring if object already exported
}
@@ -138,8 +136,8 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
private fun doCompile(args: Array<out String>, compilerMessagesStreamProxy: RemoteOutputStream, serviceOutputStreamProxy: RemoteOutputStream, body: (PrintStream) -> ExitCode): Int =
ifAlive {
val compilerMessagesStream = PrintStream( RemoteOutputStreamClient(compilerMessagesStreamProxy))
val serviceOutputStream = PrintStream( RemoteOutputStreamClient(serviceOutputStreamProxy))
val compilerMessagesStream = PrintStream(RemoteOutputStreamClient(compilerMessagesStreamProxy))
val serviceOutputStream = PrintStream(RemoteOutputStreamClient(serviceOutputStreamProxy))
checkedCompile(args, serviceOutputStream) {
val res = body( compilerMessagesStream).code
_lastUsedSeconds = nowSeconds()
@@ -176,8 +174,8 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
return ""
}
fun ThreadMXBean.threadCputime() = if (isCurrentThreadCpuTimeSupported()) currentThreadCpuTime else 0L
fun ThreadMXBean.threadUsertime() = if (isCurrentThreadCpuTimeSupported()) currentThreadUserTime else 0L
fun ThreadMXBean.threadCpuTime() = if (isCurrentThreadCpuTimeSupported) currentThreadCpuTime else 0L
fun ThreadMXBean.threadUserTime() = if (isCurrentThreadCpuTimeSupported) currentThreadUserTime else 0L
fun<R> checkedCompile(args: Array<out String>, serviceOut: PrintStream, body: () -> R): R {
try {
@@ -187,12 +185,12 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
val threadMXBean: ThreadMXBean = ManagementFactory.getThreadMXBean()
val startMem = usedMemory() / 1024
val startTime = System.nanoTime()
val startThreadTime = threadMXBean.threadCputime()
val startThreadUserTime = threadMXBean.threadUsertime()
val startThreadTime = threadMXBean.threadCpuTime()
val startThreadUserTime = threadMXBean.threadUserTime()
val res = body()
val endTime = System.nanoTime()
val endThreadTime = threadMXBean.threadCputime()
val endThreadUserTime = threadMXBean.threadUsertime()
val endThreadTime = threadMXBean.threadCpuTime()
val endThreadUserTime = threadMXBean.threadUserTime()
val endMem = usedMemory() / 1024
log.info("Done with result " + res.toString())
val elapsed = TimeUnit.NANOSECONDS.toMillis(endTime - startTime)
@@ -201,7 +199,7 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
log.info("Elapsed time: $elapsed ms (thread user: $elapsedThreadUser ms sys: ${elapsedThread - elapsedThreadUser} ms)")
log.info("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)")
System.getProperty(COMPILE_DAEMON_REPORT_PERF_PROPERTY)?.let {
serviceOut.println("PERF: TOTAL: $elapsed ms (thread user: $elapsedThreadUser ms sys: ${elapsedThread - elapsedThreadUser} ms); memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)")
serviceOut.println("PERF: Compile on daemon: $elapsed ms (thread user: $elapsedThreadUser ms sys: ${elapsedThread - elapsedThreadUser} ms); memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)")
}
return res
}
@@ -214,12 +212,12 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
fun<R> ifAlive(body: () -> R): R = rwlock.read {
if (!alive) throw IllegalStateException("Kotlin Compiler Service is not in alive state")
else body()
body()
}
fun<R> ifAliveExclusive(body: () -> R): R = rwlock.write {
if (!alive) throw IllegalStateException("Kotlin Compiler Service is not in alive state")
else body()
body()
}
// sometimes used for debugging
@@ -35,8 +35,8 @@ public class CompilerDaemonTest : KotlinIntegrationTestBase() {
data class CompilerResults(val resultCode: Int, val out: String)
val daemonOptions = DaemonOptions(runFilesPath = tmpdir.absolutePath)
val daemonJVMOptions = DaemonJVMOptions()
val daemonOptions by lazy { DaemonOptions(runFilesPath = tmpdir.absolutePath) }
val daemonJVMOptions by lazy { DaemonJVMOptions() }
val compilerId by lazy { CompilerId.makeCompilerId( File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-compiler.jar"),
File("dependencies/bootstrap-compiler/Kotlin/kotlinc/lib/kotlin-runtime.jar"),
File("dependencies/bootstrap-compiler/Kotlin/kotlinc/lib/kotlin-reflect.jar")) }
@@ -165,7 +165,7 @@ public class KotlinCompilerRunner {
ArrayList<DaemonReportMessage> daemonReportMessages = new ArrayList<DaemonReportMessage>();
CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonJVMOptions, daemonOptions, new DaemonReportingTargets(null, daemonReportMessages), true, true);
CompileService daemon = KotlinCompilerClient.connectToCompileService(compilerId, daemonJVMOptions, daemonOptions, new DaemonReportingTargets(null, daemonReportMessages), true, true);
for (DaemonReportMessage msg: daemonReportMessages) {
if (msg.getCategory() == DaemonReportCategory.EXCEPTION && daemon == null) {
@@ -182,7 +182,7 @@ public class KotlinCompilerRunner {
ByteArrayOutputStream compilerOut = new ByteArrayOutputStream();
ByteArrayOutputStream daemonOut = new ByteArrayOutputStream();
Integer res = KotlinCompilerClient.Companion.incrementalCompile(daemon, argsArray, incrementalCaches, compilerOut, daemonOut);
Integer res = KotlinCompilerClient.incrementalCompile(daemon, argsArray, incrementalCaches, compilerOut, daemonOut);
ProcessCompilerOutput(messageCollector, collector, compilerOut, res.toString());
BufferedReader reader = new BufferedReader(new StringReader(daemonOut.toString()));