diff --git a/.idea/artifacts/KotlinJpsPlugin.xml b/.idea/artifacts/KotlinJpsPlugin.xml index c48f2a07a6f..7e9ed15d83a 100644 --- a/.idea/artifacts/KotlinJpsPlugin.xml +++ b/.idea/artifacts/KotlinJpsPlugin.xml @@ -19,6 +19,8 @@ + + \ No newline at end of file diff --git a/.idea/artifacts/KotlinPlugin.xml b/.idea/artifacts/KotlinPlugin.xml index 3476cd93123..1ff79edf65e 100644 --- a/.idea/artifacts/KotlinPlugin.xml +++ b/.idea/artifacts/KotlinPlugin.xml @@ -44,6 +44,8 @@ + + diff --git a/.idea/modules.xml b/.idea/modules.xml index 21e53538f25..8bd75e0c6c8 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -50,10 +50,13 @@ + + + diff --git a/build.xml b/build.xml index 53607d88ee4..f1084a2b0b9 100644 --- a/build.xml +++ b/build.xml @@ -86,6 +86,8 @@ + + @@ -115,6 +117,8 @@ + + @@ -198,6 +202,8 @@ + + @@ -535,6 +541,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -839,7 +865,7 @@ depends="builtins,stdlib,core,reflection,pack-runtime,pack-runtime-sources"/> + + + + + + + + + + + + + \ No newline at end of file diff --git a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt new file mode 100644 index 00000000000..1850cafdaa6 --- /dev/null +++ b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt @@ -0,0 +1,263 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.kotlinr + +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache +import org.jetbrains.kotlin.rmi.* +import java.io.File +import java.io.OutputStream +import java.io.PrintStream +import java.rmi.ConnectException +import java.rmi.Remote +import java.rmi.registry.LocateRegistry +import java.util.concurrent.Semaphore +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread + +fun Process.isAlive() = + try { + this.exitValue() + false + } + catch (e: IllegalThreadStateException) { + true + } + +public class KotlinCompilerClient { + + companion object { + + val DAEMON_STARTUP_TIMEOUT_MS = 10000L + + private fun connectToService(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream): CompileService? { + + val compilerObj = connectToDaemon(compilerId, daemonOptions, errStream) ?: return null // no registry - no daemon running + return compilerObj as? CompileService ?: + throw ClassCastException("Unable to cast compiler service, actual class received: ${compilerObj.javaClass}") + } + + private fun connectToDaemon(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream): Remote? { + try { + val daemon = LocateRegistry.getRegistry("localhost", daemonOptions.port) + ?.lookup(COMPILER_SERVICE_RMI_NAME) + if (daemon != null) + return daemon + errStream.println("[daemon client] daemon not found") + } + catch (e: ConnectException) { + errStream.println("[daemon client] cannot connect to registry: " + (e.getCause()?.getMessage() ?: e.getMessage() ?: "unknown exception")) + // ignoring it - processing below + } + return null + } + + + private fun startDaemon(compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, daemonOptions: DaemonOptions, errStream: PrintStream) { + val javaExecutable = File(System.getProperty("java.home"), "bin").let { + val javaw = File(it, "javaw.exe") + if (javaw.exists()) javaw + else File(it, "java") + } + // TODO: add some specific environment variables to the cp and may be command line, to allow some specific startup configs + val args = listOf(javaExecutable.absolutePath, + "-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator)) + + daemonJVMOptions.mappers.flatMap { it.toArgs("-") } + + COMPILER_DAEMON_CLASS_FQN + + daemonOptions.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } + + compilerId.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } + errStream.println("[daemon client] starting the daemon as: " + args.joinToString(" ")) + val processBuilder = ProcessBuilder(args).redirectErrorStream(true) + // assuming daemon process is deaf and (mostly) silent, so do not handle streams + val daemon = processBuilder.start() + + var isEchoRead = Semaphore(1) + isEchoRead.acquire() + + val stdoutThread = + thread { + daemon.getInputStream() + .reader() + .forEachLine { + if (daemonOptions.startEcho.isNotEmpty() && it.contains(daemonOptions.startEcho)) { + isEchoRead.release() + return@forEachLine + } + errStream.println("[daemon] " + it) + } + } + try { + // trying to wait for process + if (daemonOptions.startEcho.isNotEmpty()) { + errStream.println("[daemon client] waiting for daemon to respond") + val succeeded = isEchoRead.tryAcquire(DAEMON_STARTUP_TIMEOUT_MS, TimeUnit.MILLISECONDS) + if (!daemon.isAlive()) + throw Exception("Daemon terminated unexpectedly") + if (!succeeded) + throw Exception("Unable to get response from daemon in $DAEMON_STARTUP_TIMEOUT_MS ms") + } + else + // without startEcho defined waiting for max timeout + Thread.sleep(DAEMON_STARTUP_TIMEOUT_MS) + } + finally { + // assuming that all important output is already done, the rest should be routed to the log by the daemon itself + if (stdoutThread.isAlive) + // TODO: find better method to stop the thread, but seems it will require asynchronous consuming of the stream + stdoutThread.stop() + } + } + + public fun checkCompilerId(compiler: CompileService, localId: CompilerId, errStream: PrintStream): Boolean { + val remoteId = compiler.getCompilerId() + errStream.println("[daemon client] remoteId = " + remoteId.toString()) + errStream.println("[daemon client] localId = " + localId.toString()) + return (localId.compilerVersion.isEmpty() || localId.compilerVersion == remoteId.compilerVersion) && + (localId.compilerClasspath.all { remoteId.compilerClasspath.contains(it) }) && + (localId.compilerDigest.isEmpty() || remoteId.compilerDigest.isEmpty() || localId.compilerDigest == remoteId.compilerDigest) + } + + public fun connectToCompileService(compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, daemonOptions: DaemonOptions, errStream: PrintStream, autostart: Boolean = true, checkId: Boolean = true): CompileService? { + val service = connectToService(compilerId, daemonOptions, errStream) + if (service != null) { + if (!checkId || checkCompilerId(service, compilerId, errStream)) { + errStream.println("[daemon client] found the suitable daemon") + return service + } + errStream.println("[daemon client] compiler identity don't match: " + compilerId.mappers.flatMap { it.toArgs("") }.joinToString(" ")) + if (!autostart) return null; + errStream.println("[daemon client] shutdown the daemon") + service.shutdown() + // TODO: find more reliable way + Thread.sleep(1000) + errStream.println("[daemon client] daemon shut down correctly, restarting") + } + else { + if (!autostart) return null; + else errStream.println("[daemon client] cannot connect to Compile Daemon, trying to start") + } + + startDaemon(compilerId, daemonJVMOptions, daemonOptions, errStream) + errStream.println("[daemon client] daemon started, trying to connect") + return connectToService(compilerId, daemonOptions, errStream) + } + + public fun shutdownCompileService(daemonOptions: DaemonOptions): Unit { + KotlinCompilerClient.connectToCompileService(CompilerId(), DaemonJVMOptions(), daemonOptions, System.out, autostart = false, checkId = false) + ?.shutdown() + } + + public fun shutdownCompileService(): Unit { + shutdownCompileService(DaemonOptions()) + } + + public fun compile(compiler: CompileService, args: Array, out: OutputStream): Int { + + val outStrm = RemoteOutputStreamServer(out) + try { + return compiler.remoteCompile(args, outStrm, CompileService.OutputFormat.PLAIN) + } + finally { + outStrm.disconnect() + } + } + + public fun incrementalCompile(compiler: CompileService, args: Array, caches: Map, out: OutputStream): Int { + + val outStrm = RemoteOutputStreamServer(out) + val cacheServers = hashMapOf() + try { + caches.mapValuesTo(cacheServers, { RemoteIncrementalCacheServer( it.getValue()) }) + return compiler.remoteIncrementalCompile(args, cacheServers, outStrm, CompileService.OutputFormat.XML) + } + finally { + cacheServers.forEach { it.getValue().disconnect() } + outStrm.disconnect() + } + } + + data class ClientOptions( + public var stop: Boolean = false + ) : OptionsGroup { + override val mappers: List> + get() = listOf( BoolPropMapper(this, ::stop)) + } + + jvmStatic public fun main(vararg args: String) { + val compilerId = CompilerId() + val daemonOptions = DaemonOptions() + val daemonLaunchingOptions = DaemonJVMOptions() + val clientOptions = ClientOptions() + val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, daemonLaunchingOptions, clientOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) + + if (!clientOptions.stop) { + if (compilerId.compilerClasspath.none()) { + // attempt to find compiler to use + println("compiler wasn't explicitly specified, attempt to find appropriate jar") + System.getProperty("java.class.path") + ?.split(File.pathSeparator) + ?.map { File(it).parent } + ?.distinct() + ?.map { + it?.walk() + ?.firstOrNull { it.getName().equals(COMPILER_JAR_NAME, ignoreCase = true) } + } + ?.filterNotNull() + ?.firstOrNull() + ?.let { compilerId.compilerClasspath = listOf(it.absolutePath) } + } + if (compilerId.compilerClasspath.none()) + throw IllegalArgumentException("Cannot find compiler jar") + else + println("desired compiler classpath: " + compilerId.compilerClasspath.joinToString(File.pathSeparator)) + + compilerId.updateDigest() + } + + val daemon = connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, System.out, autostart = !clientOptions.stop, checkId = !clientOptions.stop) + + if (daemon == null) { + if (clientOptions.stop) println("No daemon found to shut down") + else throw Exception("Unable to connect to daemon") + } + else when { + clientOptions.stop -> { + println("Shutdown the daemon") + daemon.shutdown() + println("Daemon shut down successfully") + } + else -> { + println("Executing daemon compilation with args: " + filteredArgs.joinToString(" ")) + val outStrm = RemoteOutputStreamServer(System.out) + try { + val memBefore = daemon.getUsedMemory() / 1024 + val startTime = System.nanoTime() + val res = daemon.remoteCompile(filteredArgs.toArrayList().toTypedArray(), outStrm, CompileService.OutputFormat.PLAIN) + val endTime = System.nanoTime() + println("Compilation result code: $res") + val memAfter = daemon.getUsedMemory() / 1024 + println("Compilation time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") + println("Used memory $memAfter (${"%+d".format(memAfter - memBefore)} kb)") + } + finally { + outStrm.disconnect() + } + } + } + } + } +} + diff --git a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteIncrementalCacheServer.kt b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteIncrementalCacheServer.kt new file mode 100644 index 00000000000..c2baf48baf0 --- /dev/null +++ b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteIncrementalCacheServer.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.kotlinr + +import org.jetbrains.kotlin.rmi.CompileService +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache; +import java.rmi.server.UnicastRemoteObject + + +public class RemoteIncrementalCacheServer(val cache: IncrementalCache) : CompileService.RemoteIncrementalCache { + + init { + UnicastRemoteObject.exportObject(this, 0) + } + + override fun getObsoletePackageParts(): Collection = cache.getObsoletePackageParts() + + override fun getPackageData(fqName: String): ByteArray? = cache.getPackageData(fqName) + + override fun close() { + cache.close() + } + + public fun disconnect() { + UnicastRemoteObject.unexportObject(this, true) + } +} \ No newline at end of file diff --git a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteOutputStreamServer.kt b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteOutputStreamServer.kt new file mode 100644 index 00000000000..4b67c389486 --- /dev/null +++ b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteOutputStreamServer.kt @@ -0,0 +1,45 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.kotlinr + +import org.jetbrains.kotlin.rmi.RemoteOutputStream +import java.io.OutputStream +import java.rmi.server.UnicastRemoteObject + + +class RemoteOutputStreamServer(val out: OutputStream) : RemoteOutputStream { + + init { + UnicastRemoteObject.exportObject(this, 0) + } + + public fun disconnect() { + UnicastRemoteObject.unexportObject(this, true) + } + + override fun close() { + out.close() + } + + override fun write(data: ByteArray, offset: Int, length: Int) { + out.write(data, offset, length) + } + + override fun write(dataByte: Int) { + out.write(dataByte) + } +} diff --git a/compiler/rmi/rmi-interface/rmi-interface.iml b/compiler/rmi/rmi-interface/rmi-interface.iml new file mode 100644 index 00000000000..50c2d073f1a --- /dev/null +++ b/compiler/rmi/rmi-interface/rmi-interface.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt new file mode 100644 index 00000000000..5d5cd9c7530 --- /dev/null +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi + +import java.rmi.Remote +import java.rmi.RemoteException + +public interface CompileService : Remote { + + public enum class OutputFormat : java.io.Serializable { + PLAIN, + XML + } + + public interface RemoteIncrementalCache : Remote { + throws(RemoteException::class) + public fun getObsoletePackageParts(): Collection + + throws(RemoteException::class) + public fun getPackageData(fqName: String): ByteArray? + + throws(RemoteException::class) + public fun close() + } + + throws(RemoteException::class) + public fun getCompilerId(): CompilerId + + throws(RemoteException::class) + public fun getUsedMemory(): Long + + throws(RemoteException::class) + public fun shutdown() + + throws(RemoteException::class) + public fun remoteCompile(args: Array, errStream: RemoteOutputStream, outputFormat: OutputFormat): Int + + throws(RemoteException::class) + public fun remoteIncrementalCompile( + args: Array, + caches: Map, + outputStream: RemoteOutputStream, + outputFormat: OutputFormat): Int +} diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt new file mode 100644 index 00000000000..eacf096d24b --- /dev/null +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt @@ -0,0 +1,269 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi + +import java.io.File +import java.io.Serializable +import java.lang.management.ManagementFactory +import java.security.DigestInputStream +import java.security.MessageDigest +import kotlin.reflect.KMutableProperty1 + + +public val COMPILER_JAR_NAME: String = "kotlin-compiler.jar" +public val COMPILER_SERVICE_RMI_NAME: String = "KotlinJvmCompilerService" +public val COMPILER_DAEMON_CLASS_FQN: String = "org.jetbrains.kotlin.rmi.service.CompileDaemon" +public val COMPILE_DAEMON_DEFAULT_PORT: Int = 17031 +public val COMPILE_DAEMON_ENABLED_PROPERTY: String ="kotlin.daemon.enabled" +public val COMPILE_DAEMON_JVM_OPTIONS_PROPERTY: String ="kotlin.daemon.jvm.options" +public val COMPILE_DAEMON_OPTIONS_PROPERTY: String ="kotlin.daemon.options" +public val COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX: String ="--daemon-" +public val COMPILE_DAEMON_TIMEOUT_INFINITE_S: Int = 0 +public val COMPILE_DAEMON_MEMORY_THRESHOLD_INFINITE: Long = 0L + +val COMPILER_ID_DIGEST = "MD5" + + +open class PropMapper>(val dest: C, + val prop: P, + val names: List = listOf(prop.name), + val fromString: (s: String) -> V, + val toString: ((v: V) -> String?) = { it.toString() }, + val skipIf: ((v: V) -> Boolean) = { false }, + val mergeDelimiter: String? = null) +{ + open fun toArgs(prefix: String = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX): List = + when { + skipIf(prop.get(dest)) -> listOf() + mergeDelimiter != null -> listOf( listOf(prefix + names.first(), toString(prop.get(dest))).filterNotNull().joinToString(mergeDelimiter)) + else -> listOf(prefix + names.first(), toString(prop.get(dest))).filterNotNull() + } + open fun apply(s: String) = prop.set(dest, fromString(s)) +} + +class StringPropMapper>(dest: C, + prop: P, + names: List = listOf(), + fromString: ((String) -> String) = { it }, + toString: ((String) -> String?) = { it.toString() }, + skipIf: ((String) -> Boolean) = { it.isEmpty() }, + mergeDelimiter: String? = null) +: PropMapper(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name), + fromString = fromString, toString = toString, skipIf = skipIf, mergeDelimiter = mergeDelimiter) + +class BoolPropMapper>(dest: C, prop: P, names: List = listOf()) + : PropMapper(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name), + fromString = { true }, toString = { null }, skipIf = { !prop.get(dest) }) + +class RestPropMapper>>(dest: C, prop: P) + : PropMapper, P>(dest = dest, prop = prop, toString = { null }, fromString = { arrayListOf() }) +{ + override fun toArgs(prefix: String): List = prop.get(dest).map { prefix + it } + override fun apply(s: String) = add(s) + fun add(s: String) { prop.get(dest).add(s) } +} + + +inline fun Iterable.firstMapOrNull(mappingPredicate: (T) -> Pair): R? { + for (element in this) { + val (found, mapped) = mappingPredicate(element) + if (found) return mapped + } + return null +} + + +fun Iterable.filterExtractProps(propMappers: List>, prefix: String, restParser: RestPropMapper<*,*>? = null) : Iterable { + + val iter = iterator() + val rest = arrayListOf() + + while (iter.hasNext()) { + val param = iter.next() + val (propMapper, matchingOption) = propMappers.firstMapOrNull { + val name = if (it !is RestPropMapper<*,*>) it.names.firstOrNull { param.startsWith(prefix + it) } else null + Pair(name != null, Pair(it, name)) + } ?: Pair(null, null) + + when { + propMapper != null -> { + val optionLength = prefix.length() + matchingOption!!.length() + when { + propMapper is BoolPropMapper<*,*> -> { + if (param.length() > optionLength) + throw IllegalArgumentException("Invalid switch option '$param', expecting $prefix$matchingOption without arguments") + propMapper.apply("") + } + param.length() > optionLength -> + if (param[optionLength] != '=') { + if (propMapper.mergeDelimiter == null) + throw IllegalArgumentException("Invalid option syntax '$param', expecting $prefix$matchingOption[= ]") + propMapper.apply(param.substring(optionLength)) + } + else { + propMapper.apply(param.substring(optionLength + 1)) + } + else -> { + if (!iter.hasNext()) throw IllegalArgumentException("Expecting argument for the option $prefix$matchingOption") + propMapper.apply(iter.next()) + } + } + } + restParser != null && param.startsWith(prefix) -> + restParser.add(param.removePrefix(prefix)) + else -> rest.add(param) + } + } + return rest +} + + +// TODO: find out how to create more generic variant using first constructor +//fun C.propsToParams() { +// val kc = C::class +// kc.constructors.first(). +//} + + + +public interface OptionsGroup : Serializable { + public val mappers: List> +} + +public fun Iterable.filterExtractProps(vararg groups: OptionsGroup, prefix: String) : Iterable = + filterExtractProps(groups.flatMap { it.mappers }, prefix) + + +public data class DaemonJVMOptions( + public var maxMemory: String = "", + public var maxPermSize: String = "", + public var reservedCodeCacheSize: String = "", + public var jvmParams: MutableCollection = arrayListOf() +) : OptionsGroup { + + override val mappers: List> + get() = listOf( StringPropMapper(this, ::maxMemory, listOf("Xmx"), mergeDelimiter = ""), + StringPropMapper(this, ::maxPermSize, listOf("XX:MaxPermSize"), mergeDelimiter = "="), + StringPropMapper(this, ::reservedCodeCacheSize, listOf("XX:ReservedCodeCacheSize"), mergeDelimiter = "="), + restMapper) + + val restMapper: RestPropMapper<*,*> + get() = RestPropMapper(this, ::jvmParams) +} + +public data class DaemonOptions( + public var port: Int = COMPILE_DAEMON_DEFAULT_PORT, + public var autoshutdownMemoryThreshold: Long = COMPILE_DAEMON_MEMORY_THRESHOLD_INFINITE, + public var autoshutdownIdleSeconds: Int = COMPILE_DAEMON_TIMEOUT_INFINITE_S, + public var startEcho: String = COMPILER_SERVICE_RMI_NAME +) : OptionsGroup { + + override val mappers: List> + get() = listOf( PropMapper(this, ::port, fromString = { it.toInt() }), + PropMapper(this, ::autoshutdownMemoryThreshold, fromString = { it.toLong() }, skipIf = { it == 0L }), + PropMapper(this, ::autoshutdownIdleSeconds, fromString = { it.toInt() }, skipIf = { it == 0 }), + PropMapper(this, ::startEcho, fromString = { it.trim('"') })) +} + + +fun updateSingleFileDigest(file: File, md: MessageDigest) { + DigestInputStream(file.inputStream(), md).use { + val buf = ByteArray(1024) + while (it.read(buf) != -1) { } + it.close() + } +} + +fun updateForAllClasses(dir: File, md: MessageDigest) { + dir.walk().forEach { updateEntryDigest(it, md) } +} + +fun updateEntryDigest(entry: File, md: MessageDigest) { + when { + entry.isDirectory + -> updateForAllClasses(entry, md) + entry.isFile && + (entry.getName().endsWith(".class", ignoreCase = true) || + entry.getName().endsWith(".jar", ignoreCase = true)) + -> updateSingleFileDigest(entry, md) + // else skip + } +} + +fun Iterable.getFilesClasspathDigest(): String { + val md = MessageDigest.getInstance(COMPILER_ID_DIGEST) + this.forEach { updateEntryDigest(it, md) } + return md.digest().joinToString("", transform = { "%02x".format(it) }) +} + +fun Iterable.getClasspathDigest(): String = map { File(it) }.getFilesClasspathDigest() + + +public data class CompilerId( + public var compilerClasspath: List = listOf(), + public var compilerDigest: String = "", + public var compilerVersion: String = "" + // TODO: checksum +) : OptionsGroup { + + override val mappers: List> + get() = listOf( PropMapper(this, ::compilerClasspath, toString = { it.joinToString(File.pathSeparator) }, fromString = { it.trim('"').split(File.pathSeparator)}), + StringPropMapper(this, ::compilerDigest), + StringPropMapper(this, ::compilerVersion)) + + public fun updateDigest() { + compilerDigest = compilerClasspath.getClasspathDigest() + } + + companion object { + public jvmStatic fun makeCompilerId(vararg paths: File): CompilerId = makeCompilerId(paths.asIterable()) + + public jvmStatic fun makeCompilerId(paths: Iterable): CompilerId = + // TODO consider reading version here + CompilerId(compilerClasspath = paths.map { it.absolutePath }, compilerDigest = paths.getFilesClasspathDigest()) + } +} + + +public fun isDaemonEnabled(): Boolean = System.getProperty(COMPILE_DAEMON_ENABLED_PROPERTY) != null + + +public fun configureDaemonLaunchingOptions(opts: DaemonJVMOptions, inheritMemoryLimits: Boolean): DaemonJVMOptions { + // note: sequence matters, explicit override in COMPILE_DAEMON_JVM_OPTIONS_PROPERTY should be done after inputArguments processing + if (inheritMemoryLimits) + ManagementFactory.getRuntimeMXBean().inputArguments.filterExtractProps(opts.mappers, "-") + + System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)?.let { + opts.jvmParams.addAll(it.trim('"', '\'').split(",").filterExtractProps(opts.mappers, "-", opts.restMapper)) + } + return opts +} + +public fun configureDaemonLaunchingOptions(inheritMemoryLimits: Boolean): DaemonJVMOptions = + configureDaemonLaunchingOptions(DaemonJVMOptions(), inheritMemoryLimits = inheritMemoryLimits) + +jvmOverloads public fun configureDaemonOptions(opts: DaemonOptions = DaemonOptions()): DaemonOptions { + System.getProperty(COMPILE_DAEMON_OPTIONS_PROPERTY)?.let { + val unrecognized = it.trim('"', '\'').split(",").filterExtractProps(opts.mappers, "") + if (unrecognized.any()) + throw IllegalArgumentException( + "Unrecognized daemon options passed via property $COMPILE_DAEMON_OPTIONS_PROPERTY: " + unrecognized.joinToString(" ") + + "\nSupported options: " + opts.mappers.joinToString(", ", transform = { it.names.first() })) + } + return opts +} + diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.kt new file mode 100644 index 00000000000..a295bf34ba1 --- /dev/null +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi + +import java.io.IOException +import java.rmi.Remote +import java.rmi.RemoteException + +public interface RemoteOutputStream : Remote { + + throws(IOException::class, RemoteException::class) + public fun close() + + throws(IOException::class, RemoteException::class) + public fun write(data: ByteArray, offset: Int, length: Int) + + throws(IOException::class, RemoteException::class) + public fun write(dataByte: Int) +} diff --git a/compiler/rmi/rmi-server/rmi-server.iml b/compiler/rmi/rmi-server/rmi-server.iml new file mode 100644 index 00000000000..1083739c346 --- /dev/null +++ b/compiler/rmi/rmi-server/rmi-server.iml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt new file mode 100644 index 00000000000..84c70919de2 --- /dev/null +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt @@ -0,0 +1,129 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.service + +import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler +import org.jetbrains.kotlin.rmi.* +import org.jetbrains.kotlin.service.CompileServiceImpl +import java.io.OutputStream +import java.io.PrintStream +import java.rmi.RMISecurityManager +import java.rmi.registry.LocateRegistry +import java.text.SimpleDateFormat +import java.util.* +import java.util.logging.LogManager +import java.util.logging.Logger +import kotlin.platform.platformStatic + +class LogStream(name: String) : OutputStream() { + + val log by lazy { Logger.getLogger(name) } + + val lineBuf = StringBuilder() + + override fun write(byte: Int) { + if (byte.toChar() == '\n') flush() + else lineBuf.append(byte.toChar()) + } + + override fun write(data: ByteArray, offset: Int, length: Int) { + var ofs = offset + var lineStart = ofs + while (ofs < length) { + if (data[ofs].toChar() == '\n') { + flush(data, lineStart, ofs - lineStart) + lineStart = ofs + 1 + } + ofs++ + } + if (lineStart < length) + lineBuf.append(data, lineStart, length - lineStart) + } + + fun flush(data: ByteArray, offset: Int, length: Int) { + log.info(lineBuf.toString() + data.toString().substring(offset, length)) + lineBuf.setLength(0) + } + + override fun flush() { + log.info(lineBuf.toString()) + lineBuf.setLength(0) + } +} + + +public class CompileDaemon { + + companion object { + + init { + val logPath: String = System.getProperty("kotlin.daemon.log.path")?.trimEnd('/','\\') ?: "%t" + val logTime: String = SimpleDateFormat("yyyy-MM-dd.HH-mm-ss-SSS").format(Date()) + val cfg: String = + "handlers = java.util.logging.FileHandler\n" + + "java.util.logging.FileHandler.level = ALL\n" + + "java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter\n" + + "java.util.logging.FileHandler.encoding = UTF-8\n" + + "java.util.logging.FileHandler.limit = 1073741824\n" + // 1Mb + "java.util.logging.FileHandler.count = 3\n" + + "java.util.logging.FileHandler.append = false\n" + + "java.util.logging.FileHandler.pattern = $logPath/kotlin-daemon.$logTime.%g.log\n" + + "java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s\\n\n" + + LogManager.getLogManager().readConfiguration(cfg.byteInputStream()) + } + + val log by lazy { Logger.getLogger("daemon") } + + platformStatic public fun main(args: Array) { + + val compilerId = CompilerId() + val daemonOptions = DaemonOptions() + val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) + + if (filteredArgs.any()) { + val helpLine = "usage: " + log.info(helpLine) + println(helpLine) + throw IllegalArgumentException("Unknown arguments") + } + + log.info("starting daemon") + + // TODO: find minimal set of permissions and restore security management +// if (System.getSecurityManager() == null) +// System.setSecurityManager (RMISecurityManager()) +// +// setDaemonPpermissions(daemonOptions.port) + + val registry = LocateRegistry.createRegistry(daemonOptions.port); + val compiler = K2JVMCompiler() + + CompileServiceImpl(registry, compiler, compilerId, daemonOptions) + + if (daemonOptions.startEcho.isNotEmpty()) + println(daemonOptions.startEcho) + + // this stops redirected streams reader(s) on the client side and prevent some situations with hanging threads + System.out.close() + System.err.close() + + System.setErr(PrintStream(LogStream("stderr"))) + System.setOut(PrintStream(LogStream("stdout"))) + } + } +} \ No newline at end of file diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt new file mode 100644 index 00000000000..797a6e7cb5c --- /dev/null +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt @@ -0,0 +1,203 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.service + +import org.jetbrains.kotlin.cli.common.CLICompiler +import org.jetbrains.kotlin.cli.common.ExitCode +import org.jetbrains.kotlin.config.Services +import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents +import org.jetbrains.kotlin.rmi.* +import org.jetbrains.kotlin.rmi.service.RemoteIncrementalCacheClient +import org.jetbrains.kotlin.rmi.service.RemoteOutputStreamClient +import java.io.IOException +import java.io.PrintStream +import java.lang.management.ManagementFactory +import java.net.URLClassLoader +import java.rmi.registry.Registry +import java.rmi.server.UnicastRemoteObject +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.ReentrantReadWriteLock +import java.util.jar.Manifest +import java.util.logging.Logger +import kotlin.concurrent.read +import kotlin.concurrent.write + + +class CompileServiceImpl>( + val registry: Registry, + val compiler: Compiler, + val selfCompilerId: CompilerId, + val daemonOptions: DaemonOptions +) : CompileService, UnicastRemoteObject() { + + val log by lazy { Logger.getLogger("compiler") } + + private val rwlock = ReentrantReadWriteLock() + private var alive = false + + // TODO: consider matching compilerId coming from outside with actual one +// private val selfCompilerId by lazy { +// CompilerId( +// compilerClasspath = System.getProperty("java.class.path") +// ?.split(File.pathSeparator) +// ?.map { File(it) } +// ?.filter { it.exists() } +// ?.map { it.absolutePath } +// ?: listOf(), +// compilerVersion = loadKotlinVersionFromResource() +// ) +// } + + init { + // assuming logically synchronized + try { + // cleanup for the case of incorrect restart + UnicastRemoteObject.unexportObject(this, false) + } + catch (e: java.rmi.NoSuchObjectException) { + // ignoring if object already exported + } + + val stub = UnicastRemoteObject.exportObject(this, 0) as CompileService + // TODO: use version-specific name + registry.rebind (COMPILER_SERVICE_RMI_NAME, stub); + alive = true + } + + public class IncrementalCompilationComponentsImpl(val idToCache: Map): IncrementalCompilationComponents { + // perf: cheap object, but still the pattern may be costly if there are too many calls to cache with the same id (which seems not to be the case now) + override fun getIncrementalCache(moduleId: String): IncrementalCache = RemoteIncrementalCacheClient(idToCache[moduleId]!!) + // TODO: add appropriate proxy into interaction when lookup tracker is needed + override fun getLookupTracker(): LookupTracker = LookupTracker.DO_NOTHING + } + + private fun createCompileServices(incrementalCaches: Map): Services = + Services.Builder() + .register(IncrementalCompilationComponents::class.java, IncrementalCompilationComponentsImpl(incrementalCaches)) + // TODO: add remote proxy for cancellation status tracking +// .register(javaClass(), object: CompilationCanceledStatus { +// override fun checkCanceled(): Unit = if (context.getCancelStatus().isCanceled()) throw CompilationCanceledException() +// }) + .build() + + + fun usedMemory(): Long { + System.gc() + val rt = Runtime.getRuntime() + return (rt.totalMemory() - rt.freeMemory()) + } + + fun usedMemoryMX(): Long { + System.gc() + val memoryMXBean= ManagementFactory.getMemoryMXBean() + val memHeap=memoryMXBean.getHeapMemoryUsage() + return memHeap.used + } + + // TODO: consider using version as a part of compiler ID or drop this function + private fun loadKotlinVersionFromResource(): String { + (javaClass.classLoader as? URLClassLoader) + ?.findResource("META-INF/MANIFEST.MF") + ?.let { + try { + return Manifest(it.openStream()).mainAttributes.getValue("Implementation-Version") ?: "" + } + catch (e: IOException) {} + } + return "" + } + + + fun checkedCompile(args: Array, body: () -> R): R { + try { + if (args.none()) + throw IllegalArgumentException("Error: empty arguments list.") + log.info("Starting compilation with args: " + args.joinToString(" ")) + val startMemMX = usedMemoryMX() / 1024 + val startMem = usedMemory() / 1024 + val startTime = System.nanoTime() + val res = body() + val endTime = System.nanoTime() + val endMem = usedMemory() / 1024 + val endMemMX = usedMemoryMX() / 1024 + log.info("Done with result " + res.toString()) + log.info("Elapsed time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") + log.info("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)") + log.info("Used memory (from MemoryMXBean): $endMemMX kb (${"%+d".format(endMemMX - startMemMX)} kb)") + return res + } + catch (e: Exception) { + log.info("Error: $e") + throw e + } + } + + fun ifAlive(body: () -> R): R = rwlock.read { + if (!alive) throw IllegalStateException("Kotlin Compiler Service is not in alive state") + else body() + } + + fun ifAliveExclusive(body: () -> R): R = rwlock.write { + if (!alive) throw IllegalStateException("Kotlin Compiler Service is not in alive state") + else body() + } + + // sometimes used for debugging + fun spy(msg: String, body: () -> R): R { + val res = body() + log.info(msg + " = " + res.toString()) + return res + } + + override fun getCompilerId(): CompilerId = ifAlive { selfCompilerId } + + override fun getUsedMemory(): Long = ifAlive { usedMemory() } + + override fun shutdown() { + ifAliveExclusive { + log.info("Shutdown started") + alive = false + UnicastRemoteObject.unexportObject(this, true) + log.info("Shutdown complete") + } + } + + override fun remoteCompile(args: Array, errStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int = + doCompile(args, errStream) { printStream -> + when (outputFormat) { + CompileService.OutputFormat.PLAIN -> compiler.exec(printStream, *args) + CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStream, Services.EMPTY, *args) + } + } + + override fun remoteIncrementalCompile(args: Array, caches: Map, outputStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int = + doCompile(args, outputStream) { printStream -> + when (outputFormat) { + CompileService.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation") + CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStream, createCompileServices(caches), *args) + } + } + + fun doCompile(args: Array, errStream: RemoteOutputStream, body: (PrintStream) -> ExitCode): Int = + ifAlive { + checkedCompile(args) { + body( PrintStream( RemoteOutputStreamClient(errStream))).code + } + } +} diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/DaemonPermissions.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/DaemonPermissions.kt new file mode 100644 index 00000000000..909e1a6dd1f --- /dev/null +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/DaemonPermissions.kt @@ -0,0 +1,79 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.service + +import java.io.FilePermission +import java.net.SocketPermission +import java.security.CodeSource +import java.security.Permission +import java.security.PermissionCollection +import java.security.Policy +import java.util.ArrayList +import java.util.Collections +import java.util.Enumeration +import java.util.PropertyPermission + + +public class DaemonPolicy(val port: Int) : Policy() { + + private fun createPermissions(): PermissionCollection { + val perms = DaemonPermissionCollection() + + val socketPermission = SocketPermission("localhost:$port-", "connect, accept, resolve") + val propertyPermission = PropertyPermission("localhost:$port", "read") + //val filePermission = FilePermission("<>", "read") + + perms.add(socketPermission) + perms.add(propertyPermission) + //perms.add(filePermission) + + return perms + } + + private val perms: PermissionCollection by lazy { createPermissions() } + + override fun getPermissions(codesource: CodeSource?): PermissionCollection { + return perms + } +} + + +class DaemonPermissionCollection : PermissionCollection() { + val perms = ArrayList() + + override fun add(p: Permission) { + perms.add(p) + } + + override fun implies(p: Permission): Boolean = perms.any { implies(p) } + + override fun elements(): Enumeration = Collections.enumeration(perms) + + override fun isReadOnly(): Boolean = false +// +// companion object { +// +// private val serialVersionUID = 614300921365729272L +// } + +} + + +public fun setDaemonPpermissions(port: Int) { + Policy.setPolicy(DaemonPolicy(port)) +} + diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt new file mode 100644 index 00000000000..900c1e088d2 --- /dev/null +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.service + +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache +import org.jetbrains.kotlin.rmi.CompileService + +public class RemoteIncrementalCacheClient(val cache: CompileService.RemoteIncrementalCache): IncrementalCache { + override fun getObsoletePackageParts(): Collection = cache.getObsoletePackageParts() + + override fun getPackageData(fqName: String): ByteArray? = cache.getPackageData(fqName) + + override fun close(): Unit = cache.close() +} diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteOutputStreamClient.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteOutputStreamClient.kt new file mode 100644 index 00000000000..33bd30b3006 --- /dev/null +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteOutputStreamClient.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.service + +import org.jetbrains.kotlin.rmi.RemoteOutputStream +import java.io.OutputStream + +class RemoteOutputStreamClient(val remote: RemoteOutputStream): OutputStream() { + override fun write(data: ByteArray) { + remote.write(data, 0, data.size()) + } + + override fun write(data: ByteArray, offset: Int, length: Int) { + remote.write(data, offset, length) + } + + override fun write(byte: Int) { + remote.write(byte) + } +} diff --git a/compiler/tests/compiler-tests.iml b/compiler/tests/compiler-tests.iml index fb072c4387d..5448f557960 100644 --- a/compiler/tests/compiler-tests.iml +++ b/compiler/tests/compiler-tests.iml @@ -26,5 +26,7 @@ + + \ No newline at end of file diff --git a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt new file mode 100644 index 00000000000..804afae51dc --- /dev/null +++ b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt @@ -0,0 +1,76 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.daemon + +import junit.framework.TestCase +import org.jetbrains.kotlin.cli.CliBaseTest +import org.jetbrains.kotlin.integration.KotlinIntegrationTestBase +import org.jetbrains.kotlin.rmi.CompilerId +import org.jetbrains.kotlin.rmi.DaemonJVMOptions +import org.jetbrains.kotlin.rmi.DaemonOptions +import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient +import org.jetbrains.kotlin.test.JetTestUtils +import java.io.ByteArrayOutputStream +import java.io.File + +val KOTLIN_DAEMON_TEST_PORT = 19753 + +public class CompilerDaemonTest : KotlinIntegrationTestBase() { + + data class CompilerResults(val resultCode: Int, val out: String) + + val daemonOptions = DaemonOptions(port = KOTLIN_DAEMON_TEST_PORT) + val daemonLaunchingOptions = DaemonJVMOptions() + val compilerId by lazy { CompilerId.makeCompilerId( File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-compiler.jar"), + File("dependencies/bootstrap-compiler/Kotlin/kotlinc/lib/kotlin-runtime.jar"), + File("dependencies/bootstrap-compiler/Kotlin/kotlinc/lib/kotlin-reflect.jar")) } + + private fun compileOnDaemon(args: Array): CompilerResults { + val daemon = KotlinCompilerClient.connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, System.err, autostart = true, checkId = true) + TestCase.assertNotNull("failed to connect daemon", daemon) + val strm = ByteArrayOutputStream() + val code = KotlinCompilerClient.compile(daemon!!, args, strm) + return CompilerResults(code, strm.toString()) + } + + private fun runDaemonCompilerTwice(logName: String, vararg arguments: String): Unit { + KotlinCompilerClient.shutdownCompileService(daemonOptions) + + try { + val res1 = compileOnDaemon(arguments) + TestCase.assertEquals("first compilation failed:\n${res1.out}", 0, res1.resultCode) + val res2 = compileOnDaemon(arguments) + TestCase.assertEquals("second compilation failed:\n${res2.out}", 0, res2.resultCode) + TestCase.assertEquals("build results differ", CliBaseTest.removePerfOutput(res1.out), CliBaseTest.removePerfOutput(res2.out)) + // TODO: add performance comparison assert + } + finally { + KotlinCompilerClient.shutdownCompileService(daemonOptions) + } + } + + private fun getTestBaseDir(): String = JetTestUtils.getTestDataPathBase() + "/integration/smoke/" + getTestName(true) + + private fun run(logName: String, vararg args: String): Int = runJava(getTestBaseDir(), logName, *args) + + public fun testHelloApp() { + val jar = tmpdir.absolutePath + File.separator + "hello.jar" + + runDaemonCompilerTwice("hello.compile", "-include-runtime", File(getTestBaseDir(), "hello.kt").absolutePath, "-d", jar) + run("hello.run", "-cp", jar, "Hello.HelloPackage") + } +} diff --git a/compiler/tests/org/jetbrains/kotlin/integration/KotlinIntegrationTestBase.java b/compiler/tests/org/jetbrains/kotlin/integration/KotlinIntegrationTestBase.java index 4766c779fa4..2b81e49fc34 100644 --- a/compiler/tests/org/jetbrains/kotlin/integration/KotlinIntegrationTestBase.java +++ b/compiler/tests/org/jetbrains/kotlin/integration/KotlinIntegrationTestBase.java @@ -133,7 +133,7 @@ public abstract class KotlinIntegrationTestBase extends TestCaseWithTmpdir { return runtime; } - protected static File getCompilerLib() { + public static File getCompilerLib() { File file = PathUtil.getKotlinPathsForDistDirectory().getLibPath().getAbsoluteFile(); assertTrue("Lib directory doesn't exist. Run 'ant dist'", file.isDirectory()); return file; diff --git a/idea/idea.iml b/idea/idea.iml index f0de3f1fc81..725e973dd63 100644 --- a/idea/idea.iml +++ b/idea/idea.iml @@ -50,5 +50,6 @@ + \ No newline at end of file diff --git a/idea/src/org/jetbrains/kotlin/idea/PluginStartupComponent.java b/idea/src/org/jetbrains/kotlin/idea/PluginStartupComponent.java index d9fbc93f52f..cfea2bc5d71 100644 --- a/idea/src/org/jetbrains/kotlin/idea/PluginStartupComponent.java +++ b/idea/src/org/jetbrains/kotlin/idea/PluginStartupComponent.java @@ -23,6 +23,7 @@ import org.jetbrains.kotlin.idea.caches.JarUserDataManager; import org.jetbrains.kotlin.idea.debugger.filter.FilterPackage; import org.jetbrains.kotlin.idea.decompiler.HasCompiledKotlinInJar; import org.jetbrains.kotlin.idea.framework.KotlinJavaScriptLibraryDetectionUtil; +import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient; import org.jetbrains.kotlin.utils.PathUtil; public class PluginStartupComponent implements ApplicationComponent { @@ -51,5 +52,6 @@ public class PluginStartupComponent implements ApplicationComponent { @Override public void disposeComponent() { + KotlinCompilerClient.Companion.shutdownCompileService(); } } diff --git a/jps-plugin/jps-plugin.iml b/jps-plugin/jps-plugin.iml index bae91cce456..e092109dfa7 100644 --- a/jps-plugin/jps-plugin.iml +++ b/jps-plugin/jps-plugin.iml @@ -21,5 +21,7 @@ + + \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/CompilerRunnerUtil.java b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/CompilerRunnerUtil.java index e8701e0c366..cc97d0376f3 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/CompilerRunnerUtil.java +++ b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/CompilerRunnerUtil.java @@ -56,7 +56,7 @@ public class CompilerRunnerUtil { } @Nullable - private static File getLibPath(@NotNull KotlinPaths paths, @NotNull MessageCollector messageCollector) { + public static File getLibPath(@NotNull KotlinPaths paths, @NotNull MessageCollector messageCollector) { File libs = paths.getLibPath(); if (libs.exists() && !libs.isFile()) return libs; diff --git a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java index 887f515087a..3f62f14e35b 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java +++ b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java @@ -30,6 +30,9 @@ import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments; import org.jetbrains.kotlin.cli.common.messages.MessageCollector; import org.jetbrains.kotlin.cli.common.messages.MessageCollectorUtil; import org.jetbrains.kotlin.config.CompilerSettings; +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache; +import org.jetbrains.kotlin.rmi.*; +import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient; import org.jetbrains.kotlin.utils.UtilsPackage; import java.io.*; @@ -38,6 +41,7 @@ import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; import static org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation.NO_LOCATION; import static org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR; @@ -54,13 +58,15 @@ public class KotlinCompilerRunner { CompilerSettings compilerSettings, MessageCollector messageCollector, CompilerEnvironment environment, + Map incrementalCaches, File moduleFile, OutputItemsCollector collector ) { K2JVMCompilerArguments arguments = mergeBeans(commonArguments, k2jvmArguments); setupK2JvmArguments(moduleFile, arguments); - runCompiler(K2JVM_COMPILER, arguments, compilerSettings.getAdditionalArguments(), messageCollector, collector, environment); + runCompiler(K2JVM_COMPILER, arguments, compilerSettings.getAdditionalArguments(), messageCollector, collector, environment, + incrementalCaches); } public static void runK2JsCompiler( @@ -69,6 +75,7 @@ public class KotlinCompilerRunner { @NotNull CompilerSettings compilerSettings, @NotNull MessageCollector messageCollector, @NotNull CompilerEnvironment environment, + Map incrementalCaches, @NotNull OutputItemsCollector collector, @NotNull Collection sourceFiles, @NotNull List libraryFiles, @@ -77,7 +84,8 @@ public class KotlinCompilerRunner { K2JSCompilerArguments arguments = mergeBeans(commonArguments, k2jsArguments); setupK2JsArguments(outputFile, sourceFiles, libraryFiles, arguments); - runCompiler(K2JS_COMPILER, arguments, compilerSettings.getAdditionalArguments(), messageCollector, collector, environment); + runCompiler(K2JS_COMPILER, arguments, compilerSettings.getAdditionalArguments(), messageCollector, collector, environment, + incrementalCaches); } private static void runCompiler( @@ -86,12 +94,13 @@ public class KotlinCompilerRunner { String additionalArguments, MessageCollector messageCollector, OutputItemsCollector collector, - CompilerEnvironment environment + CompilerEnvironment environment, + Map incrementalCaches ) { ByteArrayOutputStream stream = new ByteArrayOutputStream(); PrintStream out = new PrintStream(stream); - String exitCode = execCompiler(compilerClassName, arguments, additionalArguments, environment, out, messageCollector); + String exitCode = execCompiler(compilerClassName, arguments, additionalArguments, environment, incrementalCaches, out, messageCollector); BufferedReader reader = new BufferedReader(new StringReader(stream.toString())); CompilerOutputParser.parseCompilerMessagesFromReader(messageCollector, reader, collector); @@ -107,6 +116,7 @@ public class KotlinCompilerRunner { CommonCompilerArguments arguments, String additionalArguments, CompilerEnvironment environment, + Map incrementalCaches, PrintStream out, MessageCollector messageCollector ) { @@ -116,8 +126,28 @@ public class KotlinCompilerRunner { List argumentsList = ArgumentUtils.convertArgumentsToStringList(arguments); argumentsList.addAll(StringUtil.split(additionalArguments, " ")); + String[] argsArray = ArrayUtil.toStringArray(argumentsList); + + // trying the daemon first + if (incrementalCaches != null && RmiPackage.isDaemonEnabled()) { + File libPath = CompilerRunnerUtil.getLibPath(environment.getKotlinPaths(), messageCollector); + // TODO: it may be a good idea to cache the compilerId, since making it means calculating digest over jar(s) and if \\ + // the lifetime of JPS process is small anyway, we can neglect the probability of changed compiler + CompilerId compilerId = CompilerId.makeCompilerId(new File(libPath, "kotlin-compiler.jar")); + DaemonOptions daemonOptions = RmiPackage.configureDaemonOptions(); + DaemonJVMOptions daemonJVMOptions = RmiPackage.configureDaemonLaunchingOptions(true); + // TODO: find proper stream to report daemon connection progress + CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonJVMOptions, daemonOptions, System.out, true, true); + if (daemon != null) { + Integer res = KotlinCompilerClient.Companion.incrementalCompile(daemon, argsArray, incrementalCaches, out); + return res.toString(); + } + } + + // otherwise fallback to in-process + Object rc = CompilerRunnerUtil.invokeExecMethod( - compilerClassName, ArrayUtil.toStringArray(argumentsList), environment, messageCollector, out + compilerClassName, argsArray, environment, messageCollector, out ); // exec() returns an ExitCode object, class of which is loaded with a different class loader, diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt index 4606b72da80..e207b2e7200 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt @@ -247,9 +247,10 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR filesToCompile: MultiMap, incrementalCaches: Map, messageCollector: MessageCollectorAdapter, project: JpsProject ): OutputItemsCollectorImpl? { + if (JpsUtils.isJsKotlinModule(chunk.representativeTarget())) { LOG.debug("Compiling to JS ${filesToCompile.values().size()} files in " + filesToCompile.keySet().map { it.getPresentableName() }.join()) - return compileToJs(chunk, commonArguments, environment, messageCollector, project) + return compileToJs(chunk, commonArguments, environment, null, messageCollector, project) } if (IncrementalCompilation.ENABLED) { @@ -279,7 +280,9 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR ) } - return compileToJvm(allCompiledFiles, chunk, commonArguments, context, dirtyFilesHolder, environment, filesToCompile, messageCollector) + return compileToJvm(allCompiledFiles, chunk, commonArguments, context, dirtyFilesHolder, environment, + incrementalCaches.mapKeysTo(HashMap(incrementalCaches.size()), { it.getKey().id }), + filesToCompile, messageCollector) } private fun createCompileEnvironment( @@ -436,8 +439,8 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR private fun compileToJs(chunk: ModuleChunk, commonArguments: CommonCompilerArguments, environment: CompilerEnvironment, - messageCollector: KotlinBuilder.MessageCollectorAdapter, - project: JpsProject + incrementalCaches: MutableMap?, + messageCollector: MessageCollectorAdapter, project: JpsProject ): OutputItemsCollectorImpl? { val outputItemCollector = OutputItemsCollectorImpl() @@ -468,7 +471,7 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR val compilerSettings = JpsKotlinCompilerSettings.getCompilerSettings(project) val k2JsArguments = JpsKotlinCompilerSettings.getK2JsCompilerArguments(project) - runK2JsCompiler(commonArguments, k2JsArguments, compilerSettings, messageCollector, environment, outputItemCollector, sourceFiles, libraryFiles, outputFile) + runK2JsCompiler(commonArguments, k2JsArguments, compilerSettings, messageCollector, environment, incrementalCaches, outputItemCollector, sourceFiles, libraryFiles, outputFile) return outputItemCollector } @@ -491,8 +494,8 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR context: CompileContext, dirtyFilesHolder: DirtyFilesHolder, environment: CompilerEnvironment, - filesToCompile: MultiMap, - messageCollector: KotlinBuilder.MessageCollectorAdapter + incrementalCaches: MutableMap?, + filesToCompile: MultiMap, messageCollector: MessageCollectorAdapter ): OutputItemsCollectorImpl? { val outputItemCollector = OutputItemsCollectorImpl() @@ -535,7 +538,7 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR + (if (totalRemovedFiles == 0) "" else " ($totalRemovedFiles removed files)") + " in " + filesToCompile.keySet().map { it.getPresentableName() }.join()) - runK2JvmCompiler(commonArguments, k2JvmArguments, compilerSettings, messageCollector, environment, moduleFile, outputItemCollector) + runK2JvmCompiler(commonArguments, k2JvmArguments, compilerSettings, messageCollector, environment, incrementalCaches, moduleFile, outputItemCollector) moduleFile.delete() return outputItemCollector