Refactorings, reformatting code, applying code style and other cleanup

This commit is contained in:
Ilya Chernikov
2015-09-08 17:28:31 +02:00
parent d448602cb2
commit 96558c52ff
7 changed files with 570 additions and 576 deletions
@@ -26,198 +26,15 @@ import java.util.concurrent.Semaphore
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import kotlin.concurrent.thread import kotlin.concurrent.thread
fun Process.isAlive() =
try {
this.exitValue()
false
}
catch (e: IllegalThreadStateException) {
true
}
public object KotlinCompilerClient {
public enum class DaemonReportCategory {
DEBUG, INFO, EXCEPTION;
}
public data class DaemonReportMessage(public val category: DaemonReportCategory, public val message: String)
public class DaemonReportingTargets(public val out: PrintStream? = null, public val messages: MutableCollection<DaemonReportMessage>? = null)
public class KotlinCompilerClient {
companion object {
val DAEMON_DEFAULT_STARTUP_TIMEOUT_MS = 10000L val DAEMON_DEFAULT_STARTUP_TIMEOUT_MS = 10000L
val DAEMON_CONNECT_CYCLE_ATTEMPTS = 3 val DAEMON_CONNECT_CYCLE_ATTEMPTS = 3
val verboseReporting = System.getProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) != null
fun DaemonReportingTargets.report(category: DaemonReportCategory, message: String, source: String = "daemon client") {
if (category == DaemonReportCategory.DEBUG && !verboseReporting) return
out?.println("[$source] ${category.name()}: $message")
messages?.add(DaemonReportMessage(category, "[$source] $message"))
}
private fun tryFindDaemon(registryDir: File, compilerId: CompilerId, reportingTargets: DaemonReportingTargets): CompileService? {
val classPathDigest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest()
val daemons = registryDir.walk()
.map { Pair(it, makeRunFilenameRegex(digest = classPathDigest, port = "(\\d+)").match(it.name)?.groups?.get(1)?.value?.toInt() ?: 0) }
.filter { it.second != 0 }
.map {
reportingTargets.report(DaemonReportCategory.DEBUG, "found suitable daemon on port ${it.second}, trying to connect")
val daemon = tryConnectToDaemon(it.second, reportingTargets)
// cleaning orphaned file; note: daemon should shut itself down if it detects that the run file is deleted
if (daemon == null && !it.first.delete()) {
reportingTargets.report(DaemonReportCategory.INFO, "WARNING: unable to delete seemingly orphaned file '${it.first.absolutePath}', cleanup recommended")
}
daemon
}
.filterNotNull()
.toList()
return when (daemons.size()) {
0 -> null
1 -> daemons.first()
else -> throw IllegalStateException("Multiple daemons serving the same compiler, reset with the cleanup required")
// TODO: consider implementing automatic recovery instead, e.g. getting the youngest or least used daemon and shut down others
}
}
private fun tryConnectToDaemon(port: Int, reportingTargets: DaemonReportingTargets): CompileService? {
try {
val daemon = LocateRegistry.getRegistry(loopbackAddrName, port)
?.lookup(COMPILER_SERVICE_RMI_NAME)
if (daemon != null)
return daemon as? CompileService ?:
throw ClassCastException("Unable to cast compiler service, actual class received: ${daemon.javaClass}")
reportingTargets.report(DaemonReportCategory.EXCEPTION, "daemon not found")
}
catch (e: ConnectException) {
reportingTargets.report(DaemonReportCategory.EXCEPTION, "cannot connect to registry: " + (e.getCause()?.getMessage() ?: e.getMessage() ?: "unknown exception"))
// ignoring it - processing below
}
return null
}
private fun startDaemon(compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, daemonOptions: DaemonOptions, reportingTargets: DaemonReportingTargets) {
val javaExecutable = File(System.getProperty("java.home"), "bin").let {
val javaw = File(it, "javaw.exe")
if (javaw.exists()) javaw
else File(it, "java")
}
// TODO: add some specific environment variables to the cp and may be command line, to allow some specific startup configs
val args = listOf(javaExecutable.absolutePath,
"-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator)) +
daemonJVMOptions.mappers.flatMap { it.toArgs("-") } +
COMPILER_DAEMON_CLASS_FQN +
daemonOptions.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } +
compilerId.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) }
reportingTargets.report(DaemonReportCategory.DEBUG, "starting the daemon as: " + args.joinToString(" "))
val processBuilder = ProcessBuilder(args).redirectErrorStream(true)
// assuming daemon process is deaf and (mostly) silent, so do not handle streams
val daemon = processBuilder.start()
var isEchoRead = Semaphore(1)
isEchoRead.acquire()
val stdoutThread =
thread {
daemon.getInputStream()
.reader()
.forEachLine {
if (daemonOptions.runFilesPath.isNotEmpty() && it.contains(daemonOptions.runFilesPath)) {
isEchoRead.release()
return@forEachLine
}
reportingTargets.report(DaemonReportCategory.DEBUG, it, "daemon")
}
}
try {
// trying to wait for process
val daemonStartupTimeout = System.getProperty(COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY)?.let {
try {
it.toLong()
}
catch (e: Exception) {
reportingTargets.report(DaemonReportCategory.INFO, "unable to interpret $COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY property ('$it'); using default timeout $DAEMON_DEFAULT_STARTUP_TIMEOUT_MS ms")
null
}
} ?: DAEMON_DEFAULT_STARTUP_TIMEOUT_MS
if (daemonOptions.runFilesPath.isNotEmpty()) {
val succeeded = isEchoRead.tryAcquire(daemonStartupTimeout, TimeUnit.MILLISECONDS)
if (!daemon.isAlive())
throw Exception("Daemon terminated unexpectedly")
if (!succeeded)
throw Exception("Unable to get response from daemon in $daemonStartupTimeout ms")
}
else
// without startEcho defined waiting for max timeout
Thread.sleep(daemonStartupTimeout)
}
finally {
// assuming that all important output is already done, the rest should be routed to the log by the daemon itself
if (stdoutThread.isAlive)
// TODO: find better method to stop the thread, but seems it will require asynchronous consuming of the stream
stdoutThread.stop()
}
}
public fun checkCompilerId(compiler: CompileService, localId: CompilerId): Boolean {
val remoteId = compiler.getCompilerId()
return (localId.compilerVersion.isEmpty() || localId.compilerVersion == remoteId.compilerVersion) &&
(localId.compilerClasspath.all { remoteId.compilerClasspath.contains(it) }) &&
(localId.compilerDigest.isEmpty() || remoteId.compilerDigest.isEmpty() || localId.compilerDigest == remoteId.compilerDigest)
}
class FileBasedLock(compilerId: CompilerId, daemonOptions: DaemonOptions) {
private val lockFile: File =
File(daemonOptions.runFilesPath,
makeRunFilenameString(ts = "lock",
digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest(),
port = "0"))
private var locked: Boolean = acquireLockFile(lockFile)
public fun isLocked(): Boolean = locked
synchronized public fun release(): Unit {
if (locked) {
lock?.release()
channel?.close()
lockFile.delete()
locked = false
}
}
private val channel = if (locked) RandomAccessFile(lockFile, "rw").channel else null
private val lock = channel?.lock()
synchronized private fun acquireLockFile(lockFile: File): Boolean {
if (lockFile.createNewFile()) return true
try {
// attempt to delete if file is orphaned
if (lockFile.delete() && lockFile.createNewFile())
return true // if orphaned file deleted assuming that the probability of
}
catch (e: IOException) {
// Ignoring it - assuming it is another client process owning it
}
var attempts = 0L
while (lockFile.exists() && attempts++ * COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_CHECK_MS < COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_MS) {
Thread.sleep(COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_CHECK_MS)
}
if (lockFile.exists())
throw IOException("Timeout waiting the release of the lock file '${lockFile.absolutePath}")
return lockFile.createNewFile()
}
}
// TODO: remove jvmStatic after all use sites will switch to kotlin
jvmStatic
public fun connectToCompileService(compilerId: CompilerId, public fun connectToCompileService(compilerId: CompilerId,
daemonJVMOptions: DaemonJVMOptions, daemonJVMOptions: DaemonJVMOptions,
daemonOptions: DaemonOptions, daemonOptions: DaemonOptions,
@@ -271,15 +88,18 @@ public class KotlinCompilerClient {
return null return null
} }
public fun shutdownCompileService(daemonOptions: DaemonOptions): Unit { public fun shutdownCompileService(daemonOptions: DaemonOptions): Unit {
KotlinCompilerClient.connectToCompileService(CompilerId(), DaemonJVMOptions(), daemonOptions, DaemonReportingTargets(out = System.out), autostart = false, checkId = false) KotlinCompilerClient.connectToCompileService(CompilerId(), DaemonJVMOptions(), daemonOptions, DaemonReportingTargets(out = System.out), autostart = false, checkId = false)
?.shutdown() ?.shutdown()
} }
public fun shutdownCompileService(): Unit { public fun shutdownCompileService(): Unit {
shutdownCompileService(DaemonOptions()) shutdownCompileService(DaemonOptions())
} }
public fun compile(compiler: CompileService, args: Array<out String>, out: OutputStream): Int { public fun compile(compiler: CompileService, args: Array<out String>, out: OutputStream): Int {
val outStrm = RemoteOutputStreamServer(out) val outStrm = RemoteOutputStreamServer(out)
@@ -291,6 +111,9 @@ public class KotlinCompilerClient {
} }
} }
// TODO: remove jvmStatic after all use sites will switch to kotlin
jvmStatic
public fun incrementalCompile(compiler: CompileService, args: Array<out String>, caches: Map<TargetId, IncrementalCache>, compilerOut: OutputStream, daemonOut: OutputStream): Int { public fun incrementalCompile(compiler: CompileService, args: Array<out String>, caches: Map<TargetId, IncrementalCache>, compilerOut: OutputStream, daemonOut: OutputStream): Int {
val compilerOutStreamServer = RemoteOutputStreamServer(compilerOut) val compilerOutStreamServer = RemoteOutputStreamServer(compilerOut)
@@ -307,6 +130,7 @@ public class KotlinCompilerClient {
} }
} }
data class ClientOptions( data class ClientOptions(
public var stop: Boolean = false public var stop: Boolean = false
) : OptionsGroup { ) : OptionsGroup {
@@ -314,6 +138,7 @@ public class KotlinCompilerClient {
get() = listOf(BoolPropMapper(this, ::stop)) get() = listOf(BoolPropMapper(this, ::stop))
} }
jvmStatic public fun main(vararg args: String) { jvmStatic public fun main(vararg args: String) {
val compilerId = CompilerId() val compilerId = CompilerId()
val daemonOptions = DaemonOptions() val daemonOptions = DaemonOptions()
@@ -331,7 +156,7 @@ public class KotlinCompilerClient {
?.distinct() ?.distinct()
?.map { ?.map {
it?.walk() it?.walk()
?.firstOrNull { it.getName().equals(COMPILER_JAR_NAME, ignoreCase = true) } ?.firstOrNull { it.name.equals(COMPILER_JAR_NAME, ignoreCase = true) }
} }
?.filterNotNull() ?.filterNotNull()
?.firstOrNull() ?.firstOrNull()
@@ -348,7 +173,9 @@ public class KotlinCompilerClient {
val daemon = connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, DaemonReportingTargets(out = System.out), autostart = !clientOptions.stop, checkId = !clientOptions.stop) val daemon = connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, DaemonReportingTargets(out = System.out), autostart = !clientOptions.stop, checkId = !clientOptions.stop)
if (daemon == null) { if (daemon == null) {
if (clientOptions.stop) System.err.println("No daemon found to shut down") if (clientOptions.stop) {
System.err.println("No daemon found to shut down")
}
else throw Exception("Unable to connect to daemon") else throw Exception("Unable to connect to daemon")
} }
else when { else when {
@@ -363,7 +190,9 @@ public class KotlinCompilerClient {
try { try {
val memBefore = daemon.getUsedMemory() / 1024 val memBefore = daemon.getUsedMemory() / 1024
val startTime = System.nanoTime() val startTime = System.nanoTime()
val res = daemon.remoteCompile(filteredArgs.toArrayList().toTypedArray(), outStrm, CompileService.OutputFormat.PLAIN, outStrm) val res = daemon.remoteCompile(filteredArgs.toArrayList().toTypedArray(), outStrm, CompileService.OutputFormat.PLAIN, outStrm)
val endTime = System.nanoTime() val endTime = System.nanoTime()
println("Compilation result code: $res") println("Compilation result code: $res")
val memAfter = daemon.getUsedMemory() / 1024 val memAfter = daemon.getUsedMemory() / 1024
@@ -376,6 +205,190 @@ public class KotlinCompilerClient {
} }
} }
} }
// --- Implementation ---------------------------------------
val verboseReporting = System.getProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) != null
fun DaemonReportingTargets.report(category: DaemonReportCategory, message: String, source: String = "daemon client") {
if (category == DaemonReportCategory.DEBUG && !verboseReporting) return
out?.println("[$source] ${category.name()}: $message")
messages?.add(DaemonReportMessage(category, "[$source] $message"))
}
private fun tryFindDaemon(registryDir: File, compilerId: CompilerId, reportingTargets: DaemonReportingTargets): CompileService? {
val classPathDigest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest()
val daemons = registryDir.walk()
.map { Pair(it, makeRunFilenameRegex(digest = classPathDigest, port = "(\\d+)").match(it.name)?.groups?.get(1)?.value?.toInt() ?: 0) }
.filter { it.second != 0 }
.map {
reportingTargets.report(DaemonReportCategory.DEBUG, "found suitable daemon on port ${it.second}, trying to connect")
val daemon = tryConnectToDaemon(it.second, reportingTargets)
// cleaning orphaned file; note: daemon should shut itself down if it detects that the run file is deleted
if (daemon == null && !it.first.delete()) {
reportingTargets.report(DaemonReportCategory.INFO, "WARNING: unable to delete seemingly orphaned file '${it.first.absolutePath}', cleanup recommended")
}
daemon
}
.filterNotNull()
.toList()
return when (daemons.size()) {
0 -> null
1 -> daemons.first()
else -> throw IllegalStateException("Multiple daemons serving the same compiler, reset with the cleanup required")
// TODO: consider implementing automatic recovery instead, e.g. getting the youngest or least used daemon and shut down others
} }
} }
private fun tryConnectToDaemon(port: Int, reportingTargets: DaemonReportingTargets): CompileService? {
try {
val daemon = LocateRegistry.getRegistry(loopbackAddrName, port)
?.lookup(COMPILER_SERVICE_RMI_NAME)
if (daemon != null)
return daemon as? CompileService ?:
throw ClassCastException("Unable to cast compiler service, actual class received: ${daemon.javaClass}")
reportingTargets.report(DaemonReportCategory.EXCEPTION, "daemon not found")
}
catch (e: ConnectException) {
reportingTargets.report(DaemonReportCategory.EXCEPTION, "cannot connect to registry: " + (e.getCause()?.getMessage() ?: e.getMessage() ?: "unknown exception"))
// ignoring it - processing below
}
return null
}
private fun startDaemon(compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, daemonOptions: DaemonOptions, reportingTargets: DaemonReportingTargets) {
val javaExecutable = File(System.getProperty("java.home"), "bin").let {
val javaw = File(it, "javaw.exe")
// TODO: doesn't seem reliable enough, consider more checks if OS is of windows flavor, etc.
if (javaw.exists() && javaw.isFile && javaw.canExecute()) javaw else File(it, "java")
}
val args = listOf(javaExecutable.absolutePath,
"-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator)) +
daemonJVMOptions.mappers.flatMap { it.toArgs("-") } +
COMPILER_DAEMON_CLASS_FQN +
daemonOptions.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } +
compilerId.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) }
reportingTargets.report(DaemonReportCategory.DEBUG, "starting the daemon as: " + args.joinToString(" "))
val processBuilder = ProcessBuilder(args).redirectErrorStream(true)
// assuming daemon process is deaf and (mostly) silent, so do not handle streams
val daemon = processBuilder.start()
var isEchoRead = Semaphore(1)
isEchoRead.acquire()
val stdoutThread =
thread {
daemon.inputStream
.reader()
.forEachLine {
if (daemonOptions.runFilesPath.isNotEmpty() && it.contains(daemonOptions.runFilesPath)) {
isEchoRead.release()
return@forEachLine
}
reportingTargets.report(DaemonReportCategory.DEBUG, it, "daemon")
}
}
try {
// trying to wait for process
val daemonStartupTimeout = System.getProperty(COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY)?.let {
try {
it.toLong()
}
catch (e: Exception) {
reportingTargets.report(DaemonReportCategory.INFO, "unable to interpret $COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY property ('$it'); using default timeout $DAEMON_DEFAULT_STARTUP_TIMEOUT_MS ms")
null
}
} ?: DAEMON_DEFAULT_STARTUP_TIMEOUT_MS
if (daemonOptions.runFilesPath.isNotEmpty()) {
val succeeded = isEchoRead.tryAcquire(daemonStartupTimeout, TimeUnit.MILLISECONDS)
if (!daemon.isAlive())
throw Exception("Daemon terminated unexpectedly")
if (!succeeded)
throw Exception("Unable to get response from daemon in $daemonStartupTimeout ms")
}
else
// without startEcho defined waiting for max timeout
Thread.sleep(daemonStartupTimeout)
}
finally {
// assuming that all important output is already done, the rest should be routed to the log by the daemon itself
if (stdoutThread.isAlive) {
// TODO: find better method to stop the thread, but seems it will require asynchronous consuming of the stream
stdoutThread.stop()
}
}
}
private fun checkCompilerId(compiler: CompileService, localId: CompilerId): Boolean {
val remoteId = compiler.getCompilerId()
return (localId.compilerVersion.isEmpty() || localId.compilerVersion == remoteId.compilerVersion) &&
(localId.compilerClasspath.all { remoteId.compilerClasspath.contains(it) }) &&
(localId.compilerDigest.isEmpty() || remoteId.compilerDigest.isEmpty() || localId.compilerDigest == remoteId.compilerDigest)
}
class FileBasedLock(compilerId: CompilerId, daemonOptions: DaemonOptions) {
private val lockFile: File =
File(daemonOptions.runFilesPath,
makeRunFilenameString(ts = "lock",
digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest(),
port = "0"))
private var locked: Boolean = acquireLockFile(lockFile)
public fun isLocked(): Boolean = locked
synchronized public fun release(): Unit {
if (locked) {
lock?.release()
channel?.close()
lockFile.delete()
locked = false
}
}
private val channel = if (locked) RandomAccessFile(lockFile, "rw").channel else null
private val lock = channel?.lock()
synchronized private fun acquireLockFile(lockFile: File): Boolean {
if (lockFile.createNewFile()) return true
try {
// attempt to delete if file is orphaned
if (lockFile.delete() && lockFile.createNewFile())
return true // if orphaned file deleted assuming that the probability of
}
catch (e: IOException) {
// Ignoring it - assuming it is another client process owning it
}
var attempts = 0L
while (lockFile.exists() && attempts++ * COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_CHECK_MS < COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_MS) {
Thread.sleep(COMPILE_DAEMON_STARTUP_LOCK_TIMEOUT_CHECK_MS)
}
if (lockFile.exists())
throw IOException("Timeout waiting the release of the lock file '${lockFile.absolutePath}")
return lockFile.createNewFile()
}
}
}
fun Process.isAlive() =
try {
this.exitValue()
false
}
catch (e: IllegalThreadStateException) {
true
}
public enum class DaemonReportCategory {
DEBUG, INFO, EXCEPTION;
}
public data class DaemonReportMessage(public val category: DaemonReportCategory, public val message: String)
public class DaemonReportingTargets(public val out: PrintStream? = null, public val messages: MutableCollection<DaemonReportMessage>? = null)
@@ -51,10 +51,11 @@ public val COMPILE_DAEMON_DEFAULT_RUN_DIR_PATH: String get() =
// TODO consider special case for windows - local appdata // TODO consider special case for windows - local appdata
File(System.getProperty("user.home"), COMPILE_DAEMON_DATA_DIRECTORY_NAME).absolutePath File(System.getProperty("user.home"), COMPILE_DAEMON_DATA_DIRECTORY_NAME).absolutePath
val COMPILER_ID_DIGEST = "MD5" val COMPILER_ID_DIGEST = "MD5"
public fun makeRunFilenameString(ts: String, digest: String, port: String, esc: String = ""): String = "$COMPILE_DAEMON_DEFAULT_FILES_PREFIX$esc.$ts$esc.$digest$esc.$port$esc.run" public fun makeRunFilenameString(ts: String, digest: String, port: String, esc: String = ""): String = "$COMPILE_DAEMON_DEFAULT_FILES_PREFIX$esc.$ts$esc.$digest$esc.$port$esc.run"
public fun makeRunFilenameRegex(ts: String = "[0-9TZ:\\.\\+-]+", digest: String = "[0-9a-f]+", port: String = "\\d+"): Regex = makeRunFilenameString(ts, digest, port, esc = "\\").toRegex() public fun makeRunFilenameRegex(ts: String = "[0-9TZ:\\.\\+-]+", digest: String = "[0-9a-f]+", port: String = "\\d+"): Regex = makeRunFilenameString(ts, digest, port, esc = "\\").toRegex()
@@ -64,14 +65,14 @@ open class PropMapper<C, V, P: KMutableProperty1<C, V>>(val dest: C,
val fromString: (String) -> V, val fromString: (String) -> V,
val toString: ((V) -> String?) = { it.toString() }, val toString: ((V) -> String?) = { it.toString() },
val skipIf: ((V) -> Boolean) = { false }, val skipIf: ((V) -> Boolean) = { false },
val mergeDelimiter: String? = null) val mergeDelimiter: String? = null) {
{
open fun toArgs(prefix: String = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX): List<String> = open fun toArgs(prefix: String = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX): List<String> =
when { when {
skipIf(prop.get(dest)) -> listOf<String>() skipIf(prop.get(dest)) -> listOf<String>()
mergeDelimiter != null -> listOf(listOf(prefix + names.first(), toString(prop.get(dest))).filterNotNull().joinToString(mergeDelimiter)) mergeDelimiter != null -> listOf(listOf(prefix + names.first(), toString(prop.get(dest))).filterNotNull().joinToString(mergeDelimiter))
else -> listOf(prefix + names.first(), toString(prop.get(dest))).filterNotNull() else -> listOf(prefix + names.first(), toString(prop.get(dest))).filterNotNull()
} }
open fun apply(s: String) = prop.set(dest, fromString(s)) open fun apply(s: String) = prop.set(dest, fromString(s))
} }
@@ -86,6 +87,7 @@ class NullablePropMapper<C, V: Any?, P: KMutableProperty1<C, V>>(dest: C,
: PropMapper<C, V, P>(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name), : PropMapper<C, V, P>(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name),
fromString = fromString, toString = toString, skipIf = skipIf, mergeDelimiter = mergeDelimiter) fromString = fromString, toString = toString, skipIf = skipIf, mergeDelimiter = mergeDelimiter)
class StringPropMapper<C, P : KMutableProperty1<C, String>>(dest: C, class StringPropMapper<C, P : KMutableProperty1<C, String>>(dest: C,
prop: P, prop: P,
names: List<String> = listOf(), names: List<String> = listOf(),
@@ -103,11 +105,12 @@ class BoolPropMapper<C, P: KMutableProperty1<C, Boolean>>(dest: C, prop: P, name
class RestPropMapper<C, P : KMutableProperty1<C, MutableCollection<String>>>(dest: C, prop: P) class RestPropMapper<C, P : KMutableProperty1<C, MutableCollection<String>>>(dest: C, prop: P)
: PropMapper<C, MutableCollection<String>, P>(dest = dest, prop = prop, toString = { null }, fromString = { arrayListOf() }) : PropMapper<C, MutableCollection<String>, P>(dest = dest, prop = prop, toString = { null }, fromString = { arrayListOf() }) {
{
override fun toArgs(prefix: String): List<String> = prop.get(dest).map { prefix + it } override fun toArgs(prefix: String): List<String> = prop.get(dest).map { prefix + it }
override fun apply(s: String) = add(s) override fun apply(s: String) = add(s)
fun add(s: String) { prop.get(dest).add(s) } fun add(s: String) {
prop.get(dest).add(s)
}
} }
@@ -166,14 +169,6 @@ fun Iterable<String>.filterExtractProps(propMappers: List<PropMapper<*,*,*>>, pr
} }
// TODO: find out how to create more generic variant using first constructor
//fun<C> C.propsToParams() {
// val kc = C::class
// kc.constructors.first().
//}
public interface OptionsGroup : Serializable { public interface OptionsGroup : Serializable {
public val mappers: List<PropMapper<*, *, *>> public val mappers: List<PropMapper<*, *, *>>
} }
@@ -232,8 +227,8 @@ fun updateEntryDigest(entry: File, md: MessageDigest) {
entry.isDirectory entry.isDirectory
-> updateForAllClasses(entry, md) -> updateForAllClasses(entry, md)
entry.isFile && entry.isFile &&
(entry.getName().endsWith(".class", ignoreCase = true) || (entry.extension.equals("class", ignoreCase = true) ||
entry.getName().endsWith(".jar", ignoreCase = true)) entry.extension.equals("jar", ignoreCase = true))
-> updateSingleFileDigest(entry, md) -> updateSingleFileDigest(entry, md)
// else skip // else skip
} }
@@ -251,7 +246,7 @@ fun Iterable<String>.getFilesClasspathDigest(): String = map { File(it) }.getFil
fun Iterable<String>.distinctStringsDigest(): String = fun Iterable<String>.distinctStringsDigest(): String =
MessageDigest.getInstance(COMPILER_ID_DIGEST) MessageDigest.getInstance(COMPILER_ID_DIGEST)
.digest(this.distinct().sort().joinToString("").toByteArray()) .digest(this.distinct().sorted().joinToString("").toByteArray())
.joinToString("", transform = { "%02x".format(it) }) .joinToString("", transform = { "%02x".format(it) })
@@ -259,7 +254,6 @@ public data class CompilerId(
public var compilerClasspath: List<String> = listOf(), public var compilerClasspath: List<String> = listOf(),
public var compilerDigest: String = "", public var compilerDigest: String = "",
public var compilerVersion: String = "" public var compilerVersion: String = ""
// TODO: checksum
) : OptionsGroup { ) : OptionsGroup {
override val mappers: List<PropMapper<*, *, *>> override val mappers: List<PropMapper<*, *, *>>
@@ -275,7 +269,6 @@ public data class CompilerId(
public jvmStatic fun makeCompilerId(vararg paths: File): CompilerId = makeCompilerId(paths.asIterable()) public jvmStatic fun makeCompilerId(vararg paths: File): CompilerId = makeCompilerId(paths.asIterable())
public jvmStatic fun makeCompilerId(paths: Iterable<File>): CompilerId = public jvmStatic fun makeCompilerId(paths: Iterable<File>): CompilerId =
// TODO consider reading version here
CompilerId(compilerClasspath = paths.map { it.absolutePath }, compilerDigest = paths.getFilesClasspathDigest()) CompilerId(compilerClasspath = paths.map { it.absolutePath }, compilerDigest = paths.getFilesClasspathDigest())
} }
} }
@@ -286,9 +279,9 @@ public fun isDaemonEnabled(): Boolean = System.getProperty(COMPILE_DAEMON_ENABLE
public fun configureDaemonJVMOptions(opts: DaemonJVMOptions, inheritMemoryLimits: Boolean): DaemonJVMOptions { public fun configureDaemonJVMOptions(opts: DaemonJVMOptions, inheritMemoryLimits: Boolean): DaemonJVMOptions {
// note: sequence matters, explicit override in COMPILE_DAEMON_JVM_OPTIONS_PROPERTY should be done after inputArguments processing // note: sequence matters, explicit override in COMPILE_DAEMON_JVM_OPTIONS_PROPERTY should be done after inputArguments processing
if (inheritMemoryLimits) if (inheritMemoryLimits) {
ManagementFactory.getRuntimeMXBean().inputArguments.filterExtractProps(opts.mappers, "-") ManagementFactory.getRuntimeMXBean().inputArguments.filterExtractProps(opts.mappers, "-")
}
System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)?.let { System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)?.let {
opts.jvmParams.addAll(it.trim('"', '\'') opts.jvmParams.addAll(it.trim('"', '\'')
.split("(?<!\\\\),".toRegex()) .split("(?<!\\\\),".toRegex())
@@ -26,28 +26,23 @@ import java.rmi.server.RMIServerSocketFactory
// TODO switch to InetAddress.getLoopbackAddress on java 7+ // TODO switch to InetAddress.getLoopbackAddress on java 7+
val loopbackAddrName = if (java.net.InetAddress.getLocalHost() is java.net.Inet6Address) "::1" else "127.0.0.1" val loopbackAddrName by lazy { if (java.net.InetAddress.getLocalHost() is java.net.Inet6Address) "::1" else "127.0.0.1" }
val loopbackAddr by lazy { InetAddress.getByName(loopbackAddrName) } val loopbackAddr by lazy { InetAddress.getByName(loopbackAddrName) }
val serverLoopbackSocketFactory by lazy { ServerLoopbackSocketFactory() } val serverLoopbackSocketFactory by lazy { ServerLoopbackSocketFactory() }
val clientLoopbackSocketFactory by lazy { ClientLoopbackSocketFactory() } val clientLoopbackSocketFactory by lazy { ClientLoopbackSocketFactory() }
data class ServerLoopbackSocketFactory : RMIServerSocketFactory, Serializable { data class ServerLoopbackSocketFactory : RMIServerSocketFactory, Serializable {
throws(IOException::class) throws(IOException::class)
override fun createServerSocket(port: Int): ServerSocket { override fun createServerSocket(port: Int): ServerSocket = ServerSocket(port, 5, loopbackAddr)
return ServerSocket(port, 5, loopbackAddr)
}
} }
data class ClientLoopbackSocketFactory : RMIClientSocketFactory, Serializable { data class ClientLoopbackSocketFactory : RMIClientSocketFactory, Serializable {
throws(IOException::class) throws(IOException::class)
override fun createSocket(host: String, port: Int): Socket { override fun createSocket(host: String, port: Int): Socket = Socket(loopbackAddr, port)
return Socket(loopbackAddr, port)
// just call the default client socket factory
// return RMISocketFactory.getDefaultSocketFactory().createSocket(host, port)
}
} }
@@ -18,8 +18,6 @@ package org.jetbrains.kotlin.rmi.service
import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
import org.jetbrains.kotlin.rmi.* import org.jetbrains.kotlin.rmi.*
import org.jetbrains.kotlin.service.CompileServiceImpl
import org.jetbrains.kotlin.service.nowSeconds
import java.io.File import java.io.File
import java.io.IOException import java.io.IOException
import java.io.OutputStream import java.io.OutputStream
@@ -57,9 +55,7 @@ class LogStream(name: String) : OutputStream() {
} }
public class CompileDaemon { public object CompileDaemon {
companion object {
init { init {
val logPath: String = System.getProperty("kotlin.daemon.log.path")?.trimEnd('/','\\') ?: "%t" val logPath: String = System.getProperty("kotlin.daemon.log.path")?.trimEnd('/','\\') ?: "%t"
@@ -81,11 +77,11 @@ public class CompileDaemon {
val log by lazy { Logger.getLogger("daemon") } val log by lazy { Logger.getLogger("daemon") }
private fun loadVersionFromResource(): String? { private fun loadVersionFromResource(): String? {
(javaClass.classLoader as? URLClassLoader) (CompileDaemon::class.java.classLoader as? URLClassLoader)
?.findResource("META-INF/MANIFEST.MF") ?.findResource("META-INF/MANIFEST.MF")
?.let { ?.let {
try { try {
return Manifest(it.openStream()).mainAttributes.getValue("Implementation-Version") ?: "" return Manifest(it.openStream()).mainAttributes.getValue("Implementation-Version") ?: null
} }
catch (e: IOException) {} catch (e: IOException) {}
} }
@@ -133,7 +129,7 @@ public class CompileDaemon {
runFile.deleteOnExit() runFile.deleteOnExit()
val compiler = K2JVMCompiler() val compiler = K2JVMCompiler()
val compilerService = CompileServiceImpl(registry, compiler, compilerId, daemonOptions, port) val compilerService = CompileServiceImpl(registry, compiler, compilerId, port)
if (daemonOptions.runFilesPath.isNotEmpty()) if (daemonOptions.runFilesPath.isNotEmpty())
println(daemonOptions.runFilesPath) println(daemonOptions.runFilesPath)
@@ -204,4 +200,3 @@ public class CompileDaemon {
throw IllegalStateException("Cannot find free port in $attempts attempts", lastException) throw IllegalStateException("Cannot find free port in $attempts attempts", lastException)
} }
} }
}
@@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.jetbrains.kotlin.service package org.jetbrains.kotlin.rmi.service
import org.jetbrains.kotlin.cli.common.CLICompiler import org.jetbrains.kotlin.cli.common.CLICompiler
import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.cli.common.ExitCode
@@ -24,13 +24,12 @@ import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
import org.jetbrains.kotlin.modules.TargetId import org.jetbrains.kotlin.modules.TargetId
import org.jetbrains.kotlin.rmi.* import org.jetbrains.kotlin.rmi.*
import org.jetbrains.kotlin.rmi.service.RemoteIncrementalCacheClient
import org.jetbrains.kotlin.rmi.service.RemoteOutputStreamClient
import java.io.IOException import java.io.IOException
import java.io.PrintStream import java.io.PrintStream
import java.lang.management.ManagementFactory import java.lang.management.ManagementFactory
import java.lang.management.ThreadMXBean import java.lang.management.ThreadMXBean
import java.net.URLClassLoader import java.net.URLClassLoader
import java.rmi.NoSuchObjectException
import java.rmi.registry.Registry import java.rmi.registry.Registry
import java.rmi.server.UnicastRemoteObject import java.rmi.server.UnicastRemoteObject
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
@@ -47,7 +46,6 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
val registry: Registry, val registry: Registry,
val compiler: Compiler, val compiler: Compiler,
val selfCompilerId: CompilerId, val selfCompilerId: CompilerId,
val daemonOptions: DaemonOptions,
port: Int port: Int
) : CompileService, UnicastRemoteObject() { ) : CompileService, UnicastRemoteObject() {
@@ -120,7 +118,7 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
// cleanup for the case of incorrect restart and many other situations // cleanup for the case of incorrect restart and many other situations
UnicastRemoteObject.unexportObject(this, false) UnicastRemoteObject.unexportObject(this, false)
} }
catch (e: java.rmi.NoSuchObjectException) { catch (e: NoSuchObjectException) {
// ignoring if object already exported // ignoring if object already exported
} }
@@ -176,8 +174,8 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
return "" return ""
} }
fun ThreadMXBean.threadCputime() = if (isCurrentThreadCpuTimeSupported()) currentThreadCpuTime else 0L fun ThreadMXBean.threadCpuTime() = if (isCurrentThreadCpuTimeSupported) currentThreadCpuTime else 0L
fun ThreadMXBean.threadUsertime() = if (isCurrentThreadCpuTimeSupported()) currentThreadUserTime else 0L fun ThreadMXBean.threadUserTime() = if (isCurrentThreadCpuTimeSupported) currentThreadUserTime else 0L
fun<R> checkedCompile(args: Array<out String>, serviceOut: PrintStream, body: () -> R): R { fun<R> checkedCompile(args: Array<out String>, serviceOut: PrintStream, body: () -> R): R {
try { try {
@@ -187,12 +185,12 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
val threadMXBean: ThreadMXBean = ManagementFactory.getThreadMXBean() val threadMXBean: ThreadMXBean = ManagementFactory.getThreadMXBean()
val startMem = usedMemory() / 1024 val startMem = usedMemory() / 1024
val startTime = System.nanoTime() val startTime = System.nanoTime()
val startThreadTime = threadMXBean.threadCputime() val startThreadTime = threadMXBean.threadCpuTime()
val startThreadUserTime = threadMXBean.threadUsertime() val startThreadUserTime = threadMXBean.threadUserTime()
val res = body() val res = body()
val endTime = System.nanoTime() val endTime = System.nanoTime()
val endThreadTime = threadMXBean.threadCputime() val endThreadTime = threadMXBean.threadCpuTime()
val endThreadUserTime = threadMXBean.threadUsertime() val endThreadUserTime = threadMXBean.threadUserTime()
val endMem = usedMemory() / 1024 val endMem = usedMemory() / 1024
log.info("Done with result " + res.toString()) log.info("Done with result " + res.toString())
val elapsed = TimeUnit.NANOSECONDS.toMillis(endTime - startTime) val elapsed = TimeUnit.NANOSECONDS.toMillis(endTime - startTime)
@@ -201,7 +199,7 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
log.info("Elapsed time: $elapsed ms (thread user: $elapsedThreadUser ms sys: ${elapsedThread - elapsedThreadUser} ms)") log.info("Elapsed time: $elapsed ms (thread user: $elapsedThreadUser ms sys: ${elapsedThread - elapsedThreadUser} ms)")
log.info("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)") log.info("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)")
System.getProperty(COMPILE_DAEMON_REPORT_PERF_PROPERTY)?.let { System.getProperty(COMPILE_DAEMON_REPORT_PERF_PROPERTY)?.let {
serviceOut.println("PERF: TOTAL: $elapsed ms (thread user: $elapsedThreadUser ms sys: ${elapsedThread - elapsedThreadUser} ms); memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)") serviceOut.println("PERF: Compile on daemon: $elapsed ms (thread user: $elapsedThreadUser ms sys: ${elapsedThread - elapsedThreadUser} ms); memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)")
} }
return res return res
} }
@@ -214,12 +212,12 @@ class CompileServiceImpl<Compiler: CLICompiler<*>>(
fun<R> ifAlive(body: () -> R): R = rwlock.read { fun<R> ifAlive(body: () -> R): R = rwlock.read {
if (!alive) throw IllegalStateException("Kotlin Compiler Service is not in alive state") if (!alive) throw IllegalStateException("Kotlin Compiler Service is not in alive state")
else body() body()
} }
fun<R> ifAliveExclusive(body: () -> R): R = rwlock.write { fun<R> ifAliveExclusive(body: () -> R): R = rwlock.write {
if (!alive) throw IllegalStateException("Kotlin Compiler Service is not in alive state") if (!alive) throw IllegalStateException("Kotlin Compiler Service is not in alive state")
else body() body()
} }
// sometimes used for debugging // sometimes used for debugging
@@ -35,8 +35,8 @@ public class CompilerDaemonTest : KotlinIntegrationTestBase() {
data class CompilerResults(val resultCode: Int, val out: String) data class CompilerResults(val resultCode: Int, val out: String)
val daemonOptions = DaemonOptions(runFilesPath = tmpdir.absolutePath) val daemonOptions by lazy { DaemonOptions(runFilesPath = tmpdir.absolutePath) }
val daemonJVMOptions = DaemonJVMOptions() val daemonJVMOptions by lazy { DaemonJVMOptions() }
val compilerId by lazy { CompilerId.makeCompilerId( File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-compiler.jar"), val compilerId by lazy { CompilerId.makeCompilerId( File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-compiler.jar"),
File("dependencies/bootstrap-compiler/Kotlin/kotlinc/lib/kotlin-runtime.jar"), File("dependencies/bootstrap-compiler/Kotlin/kotlinc/lib/kotlin-runtime.jar"),
File("dependencies/bootstrap-compiler/Kotlin/kotlinc/lib/kotlin-reflect.jar")) } File("dependencies/bootstrap-compiler/Kotlin/kotlinc/lib/kotlin-reflect.jar")) }
@@ -165,7 +165,7 @@ public class KotlinCompilerRunner {
ArrayList<DaemonReportMessage> daemonReportMessages = new ArrayList<DaemonReportMessage>(); ArrayList<DaemonReportMessage> daemonReportMessages = new ArrayList<DaemonReportMessage>();
CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonJVMOptions, daemonOptions, new DaemonReportingTargets(null, daemonReportMessages), true, true); CompileService daemon = KotlinCompilerClient.connectToCompileService(compilerId, daemonJVMOptions, daemonOptions, new DaemonReportingTargets(null, daemonReportMessages), true, true);
for (DaemonReportMessage msg: daemonReportMessages) { for (DaemonReportMessage msg: daemonReportMessages) {
if (msg.getCategory() == DaemonReportCategory.EXCEPTION && daemon == null) { if (msg.getCategory() == DaemonReportCategory.EXCEPTION && daemon == null) {
@@ -182,7 +182,7 @@ public class KotlinCompilerRunner {
ByteArrayOutputStream compilerOut = new ByteArrayOutputStream(); ByteArrayOutputStream compilerOut = new ByteArrayOutputStream();
ByteArrayOutputStream daemonOut = new ByteArrayOutputStream(); ByteArrayOutputStream daemonOut = new ByteArrayOutputStream();
Integer res = KotlinCompilerClient.Companion.incrementalCompile(daemon, argsArray, incrementalCaches, compilerOut, daemonOut); Integer res = KotlinCompilerClient.incrementalCompile(daemon, argsArray, incrementalCaches, compilerOut, daemonOut);
ProcessCompilerOutput(messageCollector, collector, compilerOut, res.toString()); ProcessCompilerOutput(messageCollector, collector, compilerOut, res.toString());
BufferedReader reader = new BufferedReader(new StringReader(daemonOut.toString())); BufferedReader reader = new BufferedReader(new StringReader(daemonOut.toString()));