From 7ab2208fc5a671c10ab0fda88ae5a141b461da06 Mon Sep 17 00:00:00 2001 From: ligee Date: Sun, 16 Aug 2015 22:44:06 +0200 Subject: [PATCH] implementing daemon startup, basic check and restart machanisms, switching to kotlin everythere, additional logging and checks, in JPS plugin daemon compilation is off by default, turned on by "kotlin.daemon.enabled" property --- .../rmi/kotlinr/src/KotlinCompilerClient.kt | 177 +++++++++++++++-- .../src/RemoteIncrementalCacheServer.kt | 4 +- .../jetbrains/kotlin/rmi/CompileService.kt | 58 ++++++ .../jetbrains/kotlin/rmi/CompilerFacade.java | 49 ----- .../org/jetbrains/kotlin/rmi/DaemonParams.kt | 116 +++++++++++ ...utputStream.java => RemoteOutputStream.kt} | 19 +- .../kotlin/rmi/service/CompileDaemon.kt | 58 ++++++ .../kotlin/rmi/service/CompileServer.kt | 48 ----- .../kotlin/rmi/service/CompileServiceImpl.kt | 186 ++++++++++++++++++ .../kotlin/rmi/service/CompilerFacadeImpl.kt | 95 --------- .../kotlin/rmi/service/DaemonPermissions.kt | 91 +++++++++ .../service/RemoteIncrementalCacheClient.kt | 6 +- .../compilerRunner/CompilerRunnerUtil.java | 2 +- .../compilerRunner/KotlinCompilerRunner.java | 14 +- 14 files changed, 697 insertions(+), 226 deletions(-) create mode 100644 compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt delete mode 100644 compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerFacade.java create mode 100644 compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt rename compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/{RemoteOutputStream.java => RemoteOutputStream.kt} (57%) create mode 100644 compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt delete mode 100644 compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServer.kt create mode 100644 compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt delete mode 100644 compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompilerFacadeImpl.kt create mode 100644 compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/DaemonPermissions.kt diff --git a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt index e8be605c362..3dba28fc892 100644 --- a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt +++ b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt @@ -17,35 +17,146 @@ package org.jetbrains.kotlin.rmi.kotlinr import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache -import org.jetbrains.kotlin.rmi.CompilerFacade +import org.jetbrains.kotlin.rmi.* +import java.io.File import java.io.OutputStream +import java.rmi.ConnectException +import java.rmi.Remote import java.rmi.registry.LocateRegistry +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.ReentrantReadWriteLock +import java.util.logging.Logger +import kotlin.concurrent.read +import kotlin.concurrent.thread +import kotlin.concurrent.write import kotlin.platform.platformStatic +val DAEMON_STARTUP_TIMEOUT_MILIS = 10000L +val DAEMON_STARTUP_CHECH_INTERVAL_MILIS = 100L + public class KotlinCompilerClient { companion object { - public fun connectToCompilerServer(): CompilerFacade? { - val compilerObj = LocateRegistry.getRegistry("localhost", 17031).lookup("KotlinJvmCompilerService") - if (compilerObj == null) - println("Unable to find compiler service") - else { - val compiler = compilerObj as? CompilerFacade - if (compiler == null) - println("Unable to cast compiler service: ${compilerObj.javaClass}") - return compiler + val log: Logger by lazy { Logger.getLogger("KotlinDaemonClient") } + + private fun connectToService(compilerId: CompilerId, daemonOptions: DaemonOptions, log: Logger): CompileService? { + + val compilerObj = connectToDaemon(compilerId, daemonOptions, log) ?: return null // no registry - no daemon running + return compilerObj as? CompileService ?: + throw ClassCastException("Unable to cast compiler service, actual class received: ${compilerObj.javaClass}") + } + + private fun connectToDaemon(compilerId: CompilerId, daemonOptions: DaemonOptions, log: Logger): Remote? { + try { + val daemon = LocateRegistry.getRegistry("localhost", daemonOptions.port) + ?.lookup(COMPILER_SERVICE_RMI_NAME) + if (daemon != null) + return daemon + log.info("Failed to find compile daemon") + } + catch (e: ConnectException) { + log.info("Error connecting registry: " + e.toString()) + // ignoring it - processing below } return null } - public fun incrementalCompile(compiler: CompilerFacade, args: Array, caches: Map, out: OutputStream): Int { + + fun Process.isAlive() = + try { + this.exitValue() + false + } + catch (e: IllegalThreadStateException) { + true + } + + + private fun startDaemon(compilerId: CompilerId, daemonOptions: DaemonOptions, log: Logger) { + val javaExecutable = listOf(System.getProperty("java.home"), "bin", "java").joinToString(File.separator) + // TODO: add some specific environment variables to the cp and may be command line, to allow some specific startup configs + val args = listOf(javaExecutable, + "-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator), + COMPILER_DAEMON_CLASS_FQN) + + daemonOptions.asParams + + compilerId.asParams + log.info("Starting the daemon as: " + args.joinToString(" ")) + val processBuilder = ProcessBuilder(args).redirectErrorStream(true) + // assuming daemon process is deaf and (mostly) silent, so do not handle streams + val daemon = processBuilder.start() + + val lock = ReentrantReadWriteLock() + var isEchoRead = false + + val stdouThread = + thread { + daemon.getInputStream() + .reader() + .forEachLine { + if (daemonOptions.startEcho.isNotEmpty() && it.contains(daemonOptions.startEcho)) + lock.write { isEchoRead = true; return@forEachLine } + log.info("daemon: " + it) + } + } + // trying to wait for process + if (daemonOptions.startEcho.isNotEmpty()) { + log.info("waiting for daemon to respond") + var waitMillis: Long = DAEMON_STARTUP_TIMEOUT_MILIS / DAEMON_STARTUP_CHECH_INTERVAL_MILIS + while (waitMillis-- > 0) { + Thread.sleep(DAEMON_STARTUP_CHECH_INTERVAL_MILIS) + if (!daemon.isAlive() || lock.read { isEchoRead } == true) break; + } + if (!daemon.isAlive()) + throw Exception("Daemon terminated unexpectedly") + if (lock.read { isEchoRead } == false) + throw Exception("Unable to get response from daemon in $DAEMON_STARTUP_TIMEOUT_MILIS ms") + } + else + // without startEcho defined waiting for max timeout + Thread.sleep(DAEMON_STARTUP_TIMEOUT_MILIS) + // assuming that all important output is already done, the rest should be routed to the log by the daemon itself + if (stdouThread.isAlive) + // TODO: find better method to stop the thread, but seems it will require asynchronous consuming of the stream + lock.write { stdouThread.stop() } + } + + public fun checkCompilerId(compiler: CompileService, localId: CompilerId, log: Logger): Boolean { + val remoteId = compiler.getCompilerId() + log.info("remoteId = " + remoteId.toString()) + log.info("localId = " + localId.toString()) + return (localId.compilerVersion.isEmpty() || localId.compilerVersion == remoteId.compilerVersion) && + (localId.compilerClasspath.all { remoteId.compilerClasspath.contains(it) }) + } + + public fun connectToCompileService(compilerId: CompilerId, daemonOptions: DaemonOptions, log: Logger): CompileService? { + val service = connectToService(compilerId, daemonOptions, log) + if (service != null) { + if (checkCompilerId(service, compilerId, log)) { + log.info("Found the suitable daemon") + return service + } + log.info("Compiler identity don't match: " + compilerId.asParams.joinToString(" ")) + log.info("Shutdown the daemon") + service.shutdown() + // TODO: find more reliable way + Thread.sleep(1000) + log.info("Daemon shut down correctly, restarting") + } + else + log.info("cannot connect to Compile Daemon, trying to start") + startDaemon(compilerId, daemonOptions, log) + log.info("Daemon started, trying to connect") + return connectToService(compilerId, daemonOptions, log) + } + + public fun incrementalCompile(compiler: CompileService, args: Array, caches: Map, out: OutputStream): Int { val outStrm = RemoteOutputStreamServer(out) val cacheServers = hashMapOf() try { caches.forEach { cacheServers.put( it.getKey(), RemoteIncrementalCacheServer( it.getValue())) } - val res = compiler.remoteIncrementalCompile(args, cacheServers, outStrm, CompilerFacade.OutputFormat.XML) + val res = compiler.remoteIncrementalCompile(args, cacheServers, outStrm, CompileService.OutputFormat.XML) return res } finally { @@ -54,19 +165,51 @@ public class KotlinCompilerClient { } } + public fun isDaemonEnabled(): Boolean = System.getProperty(COMPILE_DAEMON_ENABLED_PROPERTY) != null + + platformStatic public fun main(vararg args: String) { - connectToCompilerServer()?.let { - println("Executing daemon compilation with args: " + args.joinToString(" ")) + val compilerId = CompilerId() + val daemonOptions = DaemonOptions() + val filteredArgs = args.asIterable().propParseFilter(compilerId, daemonOptions) + + if (compilerId.compilerClasspath.none()) { + // attempt to find compiler to use + log.info("compiler wasn't explicitly specified, attempt to find appropriate jar") + System.getProperty("java.class.path") + ?.split(File.pathSeparator) + ?.map { File(it).parent } + ?.distinct() + ?.map { it?.walk() + ?.firstOrNull { it.getName().equals(COMPILER_JAR_NAME, ignoreCase = true) } } + ?.filterNotNull() + ?.firstOrNull() + ?.let { compilerId.compilerClasspath = listOf(it.absolutePath)} + } + if (compilerId.compilerClasspath.none()) + throw IllegalArgumentException("Cannot find compiler jar") + else + log.info("desired compiler classpath: " + compilerId.compilerClasspath.joinToString(File.pathSeparator)) + + connectToCompileService(compilerId, daemonOptions, log)?.let { + log.info("Executing daemon compilation with args: " + args.joinToString(" ")) val outStrm = RemoteOutputStreamServer(System.out) try { - val res = it.remoteCompile(args, outStrm, CompilerFacade.OutputFormat.PLAIN) - println("Compilation result code: $res") + val memBefore = it.getUsedMemory() / 1024 + val startTime = System.nanoTime() + val res = it.remoteCompile(args, outStrm, CompileService.OutputFormat.PLAIN) + val endTime = System.nanoTime() + log.info("Compilation result code: $res") + val memAfter = it.getUsedMemory() / 1024 + log.info("Compilation time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") + log.info("Used memory $memAfter (${"%+d".format(memAfter - memBefore)} kb)") } finally { outStrm.disconnect() } } - } + ?: throw Exception("Unable to connect to daemon") + } } } diff --git a/compiler/rmi/kotlinr/src/RemoteIncrementalCacheServer.kt b/compiler/rmi/kotlinr/src/RemoteIncrementalCacheServer.kt index a924ad64f40..c2baf48baf0 100644 --- a/compiler/rmi/kotlinr/src/RemoteIncrementalCacheServer.kt +++ b/compiler/rmi/kotlinr/src/RemoteIncrementalCacheServer.kt @@ -16,12 +16,12 @@ package org.jetbrains.kotlin.rmi.kotlinr -import org.jetbrains.kotlin.rmi.CompilerFacade +import org.jetbrains.kotlin.rmi.CompileService import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache; import java.rmi.server.UnicastRemoteObject -public class RemoteIncrementalCacheServer(val cache: IncrementalCache) : CompilerFacade.RemoteIncrementalCache { +public class RemoteIncrementalCacheServer(val cache: IncrementalCache) : CompileService.RemoteIncrementalCache { init { UnicastRemoteObject.exportObject(this, 0) diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt new file mode 100644 index 00000000000..890c4eae466 --- /dev/null +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi + +import java.rmi.Remote +import java.rmi.RemoteException + +public interface CompileService : Remote { + + public enum class OutputFormat : java.io.Serializable { + PLAIN, + XML + } + + public interface RemoteIncrementalCache : Remote { + throws(RemoteException::class) + public fun getObsoletePackageParts(): Collection + + throws(RemoteException::class) + public fun getPackageData(fqName: String): ByteArray? + + throws(RemoteException::class) + public fun close() + } + + throws(RemoteException::class) + public fun getCompilerId(): CompilerId + + throws(RemoteException::class) + public fun getUsedMemory(): Long + + throws(RemoteException::class) + public fun shutdown() + + throws(RemoteException::class) + public fun remoteCompile(args: Array, errStream: RemoteOutputStream, outputFormat: OutputFormat): Int + + throws(RemoteException::class) + public fun remoteIncrementalCompile( + args: Array, + caches: Map, + outputStream: RemoteOutputStream, + outputFormat: OutputFormat): Int +} diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerFacade.java b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerFacade.java deleted file mode 100644 index adc151039fa..00000000000 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerFacade.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2010-2015 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.kotlin.rmi; - -import java.rmi.Remote; -import java.rmi.RemoteException; -import java.util.Collection; -import java.util.Map; - - -public interface CompilerFacade extends Remote { - - enum OutputFormat implements java.io.Serializable { - PLAIN, - XML - } - - interface RemoteIncrementalCache extends Remote { - Collection getObsoletePackageParts() throws RemoteException;; - - byte[] getPackageData(String fqName) throws RemoteException;; - - void close() throws RemoteException; - } - - - int remoteCompile(String[] args, RemoteOutputStream errStream, OutputFormat outputFormat) throws RemoteException; - - int remoteIncrementalCompile( - String[] args, - Map caches, - RemoteOutputStream outputStream, - OutputFormat outputFormat - ) throws RemoteException; -} diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt new file mode 100644 index 00000000000..a1dfb866393 --- /dev/null +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt @@ -0,0 +1,116 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi + +import java.io.File +import java.io.Serializable +import kotlin.platform.platformStatic +import kotlin.reflect.KMutableProperty1 +import kotlin.reflect.KProperty1 + + +public val COMPILER_JAR_NAME: String = "kotlin-compiler.jar" +public val COMPILER_SERVICE_RMI_NAME: String = "KotlinJvmCompilerService" +public val COMPILER_DAEMON_CLASS_FQN: String = "org.jetbrains.kotlin.rmi.service.CompileDaemon" +public val COMPILE_DAEMON_DEFAULT_PORT: Int = 17031 +public val COMPILE_DAEMON_ENABLED_PROPERTY: String ="kotlin.daemon.enabled" + + +fun> C.propToParams(p: P, conv: ((v: V) -> String) = { it.toString() } ) = + listOf("--daemon-" + p.name, conv(p.get(this))) + +class PropParser>(val dest: C, val prop: P, val parse: (s: String) -> V) { + fun apply(s: String) = prop.set(dest, parse(s)) +} + +fun Iterable.propParseFilter(parsers: List>) : Iterable { + var currentParser: PropParser<*,*,*>? = null + return filter { param -> + if (currentParser == null) { + currentParser = parsers.find { param.equals("--daemon-" + it.prop.name) } + if (currentParser != null) false + else true + } + else { + currentParser!!.apply(param) + currentParser = null + false + } + } +} + +// TODO: find out how to create more generic variant using first constructor +//fun C.propsToParams() { +// val kc = C::class +// kc.constructors.first(). +//} + +public interface CmdlineParams : Serializable { + public val asParams: Iterable + public val parsers: List> +} + +public fun Iterable.propParseFilter(vararg cs: CmdlineParams) : Iterable = + propParseFilter(cs.flatMap { it.parsers }) + + +public data class DaemonOptions( + public var port: Int = COMPILE_DAEMON_DEFAULT_PORT, + public var autoshutdownMemoryThreshold: Long = 0 /* 0 means unchecked */, + public var autoshutdownIdleSeconds: Int = 0 /* 0 means unchecked */, + public var startEcho: String = COMPILER_SERVICE_RMI_NAME +) : CmdlineParams { + + override val asParams: Iterable + get() = + propToParams(::port) + + propToParams(::autoshutdownMemoryThreshold) + + propToParams(::autoshutdownIdleSeconds) + + propToParams(::startEcho) + + override val parsers: List> + get() = listOf( PropParser(this, ::port, { it.toInt()}), + PropParser(this, ::autoshutdownMemoryThreshold, { it.toLong()}), + PropParser(this, ::autoshutdownIdleSeconds, { it.toInt()}), + PropParser(this, ::startEcho, { it.trim('"') })) +} + + +public data class CompilerId( + public var compilerClasspath: List = listOf(), + public var compilerVersion: String = "" + // TODO: checksum +) : CmdlineParams { + + override val asParams: Iterable + get() = + propToParams(::compilerClasspath, { it.joinToString(File.pathSeparator) }) + + propToParams(::compilerVersion) + + override val parsers: List> + get() = + listOf( PropParser(this, ::compilerClasspath, { it.trim('"').split(File.pathSeparator)}), + PropParser(this, ::compilerVersion, { it.trim('"') })) + + companion object { + public platformStatic fun makeCompilerId(libPath: File): CompilerId = + // TODO consider reading version and calculating checksum here + CompilerId(compilerClasspath = listOf(libPath.absolutePath)) + } +} + + diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.java b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.kt similarity index 57% rename from compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.java rename to compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.kt index 09694f0fdfb..a295bf34ba1 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.java +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.kt @@ -14,17 +14,20 @@ * limitations under the License. */ -package org.jetbrains.kotlin.rmi; +package org.jetbrains.kotlin.rmi -import java.io.IOException; -import java.rmi.Remote; -import java.rmi.RemoteException; +import java.io.IOException +import java.rmi.Remote +import java.rmi.RemoteException -public interface RemoteOutputStream extends Remote { +public interface RemoteOutputStream : Remote { - void close() throws IOException, RemoteException; + throws(IOException::class, RemoteException::class) + public fun close() - void write(byte[] data, int offset, int length) throws IOException, RemoteException; + throws(IOException::class, RemoteException::class) + public fun write(data: ByteArray, offset: Int, length: Int) - void write(int dataByte) throws IOException, RemoteException; + throws(IOException::class, RemoteException::class) + public fun write(dataByte: Int) } diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt new file mode 100644 index 00000000000..2d6b560dce0 --- /dev/null +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.service + +import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler +import org.jetbrains.kotlin.rmi.COMPILE_DAEMON_DEFAULT_PORT +import org.jetbrains.kotlin.rmi.CompilerId +import org.jetbrains.kotlin.rmi.DaemonOptions +import org.jetbrains.kotlin.rmi.propParseFilter +import org.jetbrains.kotlin.service.CompileServiceImpl +import java.rmi.RMISecurityManager +import java.rmi.registry.LocateRegistry +import kotlin.platform.platformStatic + +public class CompileDaemon { + + companion object { + + platformStatic public fun main(args: Array) { + + val compilerId = CompilerId() + val daemonOptions = DaemonOptions() + val filteredArgs = args.asIterable().propParseFilter(compilerId, daemonOptions) + + if (filteredArgs.any()) { + println("usage: ") + throw IllegalArgumentException("Unknown arguments") + } + + // TODO: find minimal set of permissions and restore security management +// if (System.getSecurityManager() == null) +// System.setSecurityManager (RMISecurityManager()) +// +// setDaemonPpermissions(daemonOptions.port) + + val registry = LocateRegistry.createRegistry(daemonOptions.port); + + val server = CompileServiceImpl(registry, K2JVMCompiler()) + + if (daemonOptions.startEcho.isNotEmpty()) + println(daemonOptions.startEcho) + } + } +} \ No newline at end of file diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServer.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServer.kt deleted file mode 100644 index f90f76485a7..00000000000 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServer.kt +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2010-2015 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.kotlin.rmi.service - -import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler -import org.jetbrains.kotlin.rmi.CompilerFacade -import org.jetbrains.kotlin.service.CompilerFacadeImpl -import java.rmi.RMISecurityManager -import java.rmi.registry.LocateRegistry -import java.rmi.server.UnicastRemoteObject -import kotlin.platform.platformStatic - -public class CompileServer { - - companion object { - platformStatic public fun main(args: Array) { - if (System.getSecurityManager() == null) - System.setSecurityManager (RMISecurityManager()) - - val registry = LocateRegistry.createRegistry(17031); - - val server = CompilerFacadeImpl(K2JVMCompiler()) - try { - UnicastRemoteObject.unexportObject(server, false) - } - catch (e: java.rmi.NoSuchObjectException) { - // ignoring if object already exported - } - - val stub = UnicastRemoteObject.exportObject(server, 0) as CompilerFacade - registry.rebind ("KotlinJvmCompilerService", stub); - } - } -} \ No newline at end of file diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt new file mode 100644 index 00000000000..244f957ebec --- /dev/null +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt @@ -0,0 +1,186 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.service + +import org.jetbrains.kotlin.cli.common.CLICompiler +import org.jetbrains.kotlin.config.Services +import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents +import org.jetbrains.kotlin.rmi.COMPILER_SERVICE_RMI_NAME +import org.jetbrains.kotlin.rmi.CompileService +import org.jetbrains.kotlin.rmi.CompilerId +import org.jetbrains.kotlin.rmi.RemoteOutputStream +import org.jetbrains.kotlin.rmi.service.RemoteIncrementalCacheClient +import org.jetbrains.kotlin.rmi.service.RemoteOutputStreamClient +import java.io.File +import java.io.FileNotFoundException +import java.io.IOException +import java.io.PrintStream +import java.net.URLClassLoader +import java.rmi.registry.Registry +import java.rmi.server.UnicastRemoteObject +import java.util.* +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.ReentrantReadWriteLock +import java.util.jar.Manifest +import java.util.logging.Logger +import kotlin.concurrent.read +import kotlin.concurrent.write + + +class CompileServiceImpl>(val registry: Registry, val compiler: Compiler) : CompileService, UnicastRemoteObject() { + + private val rwlock = ReentrantReadWriteLock() + private var alive = false + private val selfCompilerId by lazy { + // TODO: add classpath checksum calculated on init, add it to the interface and use to decide whether it was changed during the lifetime of the daemon + CompilerId( + compilerClasspath = System.getProperty("java.class.path") + ?.split(File.pathSeparator) + ?.map { File(it) } + ?.filter { it.exists() } + ?.map { it.absolutePath } + ?: listOf(), + compilerVersion = loadKotlinVersionFromResource() + ) + } + + init { + // assuming logically synchronized + try { + // cleanup for the case of incorrect restart + UnicastRemoteObject.unexportObject(this, false) + } + catch (e: java.rmi.NoSuchObjectException) { + // ignoring if object already exported + } + + val stub = UnicastRemoteObject.exportObject(this, 0) as CompileService + // TODO: use version-specific name + registry.rebind (COMPILER_SERVICE_RMI_NAME, stub); + alive = true + } + + public class IncrementalCompilationComponentsImpl(val idToCache: Map): IncrementalCompilationComponents { + // perf: cheap object, but still the pattern may be costly if there are too many calls to cache with the same id (which seems not to be the case now) + override fun getIncrementalCache(moduleId: String): IncrementalCache = RemoteIncrementalCacheClient(idToCache[moduleId]!!) + override fun getLookupTracker(): LookupTracker = LookupTracker.DO_NOTHING + } + + private fun createCompileServices(incrementalCaches: Map): Services = + Services.Builder() + .register(javaClass(), IncrementalCompilationComponentsImpl(incrementalCaches)) +// .register(javaClass(), object: CompilationCanceledStatus { +// override fun checkCanceled(): Unit = if (context.getCancelStatus().isCanceled()) throw CompilationCanceledException() +// }) + .build() + + + fun usedMemory(): Long { + System.gc() + val rt = Runtime.getRuntime() + return (rt.totalMemory() - rt.freeMemory()) + } + + private fun loadKotlinVersionFromResource(): String { + (javaClass.classLoader as? URLClassLoader) + ?.findResource("META-INF/MANIFEST.MF") + ?.let { + try { + return Manifest(it.openStream()).mainAttributes.getValue("Implementation-Version") ?: "" + } + catch (e: IOException) {} + } + return "" + } + + + fun checkedCompile(args: Array, body: () -> R): R { + try { + if (args.none()) + throw IllegalArgumentException("Error: empty arguments list.") + println("Starting compilation with args: " + args.joinToString(" ")) + val startMem = usedMemory() / 1024 + val startTime = System.nanoTime() + val res = body() + val endTime = System.nanoTime() + val endMem = usedMemory() / 1024 + println("Done") + println("Elapsed time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") + println("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)") + return res + } + catch (e: Exception) { + println("Error: $e") + throw e + } + } + + fun ifAlive(body: () -> R): R = rwlock.read { + if (!alive) throw IllegalStateException("Kotlin Compiler Service is not in alive state") + else body() + } + + fun ifAliveExclusive(body: () -> R): R = rwlock.write { + if (!alive) throw IllegalStateException("Kotlin Compiler Service is not in alive state") + else body() + } + + fun spy(msg: String, body: () -> R): R { + val res = body() + println(msg + " = " + res.toString()) + return res + } + + override fun getCompilerId(): CompilerId = ifAlive { selfCompilerId } + + override fun getUsedMemory(): Long = ifAlive { usedMemory() } + + override fun shutdown() { + ifAliveExclusive { + println("Shutdown started") + alive = false + UnicastRemoteObject.unexportObject(this, true) + println("Shutdown complete") + } + } + + override fun remoteCompile(args: Array, errStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int = + ifAlive { + checkedCompile(args) { + val strm = RemoteOutputStreamClient(errStream) + val printStrm = PrintStream(strm) + when (outputFormat) { + CompileService.OutputFormat.PLAIN -> compiler.exec(printStrm, *args) + CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStrm, Services.EMPTY, *args) + }.code + } + } + + override fun remoteIncrementalCompile(args: Array, caches: Map, errStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int = + ifAlive { + checkedCompile(args) { + val strm = RemoteOutputStreamClient(errStream) + val printStrm = PrintStream(strm) + when (outputFormat) { + CompileService.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation") + CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStrm, createCompileServices(caches), *args) + }.code + } + } +} diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompilerFacadeImpl.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompilerFacadeImpl.kt deleted file mode 100644 index 156c409242b..00000000000 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompilerFacadeImpl.kt +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2010-2015 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.kotlin.service - -import org.jetbrains.kotlin.cli.common.CLICompiler -import org.jetbrains.kotlin.config.Services -import org.jetbrains.kotlin.incremental.components.LookupTracker -import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache -import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents -import org.jetbrains.kotlin.rmi.CompilerFacade -import org.jetbrains.kotlin.rmi.RemoteOutputStream -import org.jetbrains.kotlin.rmi.service.RemoteIncrementalCacheClient -import org.jetbrains.kotlin.rmi.service.RemoteOutputStreamClient -import java.io.PrintStream -import java.rmi.server.UnicastRemoteObject -import java.util.concurrent.TimeUnit - - -class CompilerFacadeImpl>(val compiler: Compiler) : CompilerFacade, UnicastRemoteObject() { - - public class IncrementalCompilationComponentsImpl(val idToCache: Map): IncrementalCompilationComponents { - // perf: cheap object, but still the pattern may be costy if there are too many calls to cache with the same id (which seems not to be the case now) - override fun getIncrementalCache(moduleId: String): IncrementalCache = RemoteIncrementalCacheClient(idToCache[moduleId]!!) - override fun getLookupTracker(): LookupTracker = LookupTracker.DO_NOTHING - } - - private fun createCompileServices(incrementalCaches: Map): Services = - Services.Builder() - .register(javaClass(), IncrementalCompilationComponentsImpl(incrementalCaches)) -// .register(javaClass(), object: CompilationCanceledStatus { -// override fun checkCanceled(): Unit = if (context.getCancelStatus().isCanceled()) throw CompilationCanceledException() -// }) - .build() - - fun usedMemoryKb(): Long { - System.gc() - val rt = Runtime.getRuntime() - return (rt.totalMemory() - rt.freeMemory()) / 1024 - } - - fun checked(args: Array, body: () -> R): R { - try { - if (args.none()) - throw IllegalArgumentException("Error: empty arguments list.") - println("Starting compilation with args: " + args.joinToString(" ")) - val startMem = usedMemoryKb() - val startTime = System.nanoTime() - val res = body() - val endTime = System.nanoTime() - val endMem = usedMemoryKb() - println("Done") - println("Elapsed time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") - println("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)") - return res - } - catch (e: Exception) { - println("Error: $e") - throw e - } - } - - override fun remoteCompile(args: Array, errStream: RemoteOutputStream, outputFormat: CompilerFacade.OutputFormat): Int = - checked(args) { - val strm = RemoteOutputStreamClient(errStream) - val printStrm = PrintStream(strm) - when (outputFormat) { - CompilerFacade.OutputFormat.PLAIN -> compiler.exec(printStrm, *args) - CompilerFacade.OutputFormat.XML -> compiler.execAndOutputXml(printStrm, Services.EMPTY, *args) - }.code - } - - override fun remoteIncrementalCompile(args: Array, caches: Map, errStream: RemoteOutputStream, outputFormat: CompilerFacade.OutputFormat): Int = - checked(args) { - val strm = RemoteOutputStreamClient(errStream) - val printStrm = PrintStream(strm) - when (outputFormat) { - CompilerFacade.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation") - CompilerFacade.OutputFormat.XML -> compiler.execAndOutputXml(printStrm, createCompileServices(caches), *args) - }.code - } -} diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/DaemonPermissions.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/DaemonPermissions.kt new file mode 100644 index 00000000000..51d88d97837 --- /dev/null +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/DaemonPermissions.kt @@ -0,0 +1,91 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.service + +import java.io.FilePermission +import java.net.SocketPermission +import java.security.CodeSource +import java.security.Permission +import java.security.PermissionCollection +import java.security.Policy +import java.util.ArrayList +import java.util.Collections +import java.util.Enumeration +import java.util.PropertyPermission + + +public class DaemonPolicy(val port: Int) : Policy() { + + private fun createPermissions(): PermissionCollection { + val perms = DaemonPermissionCollection() + + val socketPermission = SocketPermission("localhost:$port-", "connect, accept, resolve") + val propertyPermission = PropertyPermission("localhost:$port", "read") + //val filePermission = FilePermission("<>", "read") + + perms.add(socketPermission) + perms.add(propertyPermission) + //perms.add(filePermission) + + return perms + } + + private val perms: PermissionCollection by lazy { createPermissions() } + + override fun getPermissions(codesource: CodeSource?): PermissionCollection { + return perms + } +} + + +class DaemonPermissionCollection : PermissionCollection() { + var perms = ArrayList() + + override fun add(p: Permission) { + perms.add(p) + } + + override fun implies(p: Permission): Boolean { + val i = perms.iterator() + while (i.hasNext()) { + if (i.next().implies(p)) { + return true + } + } + return false + } + + override fun elements(): Enumeration { + return Collections.enumeration(perms) + } + + override fun isReadOnly(): Boolean { + return false + } +// +// companion object { +// +// private val serialVersionUID = 614300921365729272L +// } + +} + + +public fun setDaemonPpermissions(port: Int) { + Policy.setPolicy(DaemonPolicy(port)) +} + diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt index 990a1c0885a..900c1e088d2 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt @@ -17,12 +17,12 @@ package org.jetbrains.kotlin.rmi.service import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache -import org.jetbrains.kotlin.rmi.CompilerFacade +import org.jetbrains.kotlin.rmi.CompileService -public class RemoteIncrementalCacheClient(val cache: CompilerFacade.RemoteIncrementalCache): IncrementalCache { +public class RemoteIncrementalCacheClient(val cache: CompileService.RemoteIncrementalCache): IncrementalCache { override fun getObsoletePackageParts(): Collection = cache.getObsoletePackageParts() - override fun getPackageData(fqName: String): ByteArray = cache.getPackageData(fqName) + override fun getPackageData(fqName: String): ByteArray? = cache.getPackageData(fqName) override fun close(): Unit = cache.close() } diff --git a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/CompilerRunnerUtil.java b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/CompilerRunnerUtil.java index e8701e0c366..cc97d0376f3 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/CompilerRunnerUtil.java +++ b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/CompilerRunnerUtil.java @@ -56,7 +56,7 @@ public class CompilerRunnerUtil { } @Nullable - private static File getLibPath(@NotNull KotlinPaths paths, @NotNull MessageCollector messageCollector) { + public static File getLibPath(@NotNull KotlinPaths paths, @NotNull MessageCollector messageCollector) { File libs = paths.getLibPath(); if (libs.exists() && !libs.isFile()) return libs; diff --git a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java index 5da5119d20a..e313bd74d6a 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java +++ b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java @@ -31,7 +31,9 @@ import org.jetbrains.kotlin.cli.common.messages.MessageCollector; import org.jetbrains.kotlin.cli.common.messages.MessageCollectorUtil; import org.jetbrains.kotlin.config.CompilerSettings; import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache; -import org.jetbrains.kotlin.rmi.CompilerFacade; +import org.jetbrains.kotlin.rmi.CompileService; +import org.jetbrains.kotlin.rmi.CompilerId; +import org.jetbrains.kotlin.rmi.DaemonOptions; import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient; import org.jetbrains.kotlin.utils.UtilsPackage; @@ -42,6 +44,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.logging.Logger; import static org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation.NO_LOCATION; import static org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR; @@ -127,8 +130,13 @@ public class KotlinCompilerRunner { String[] argsArray = ArrayUtil.toStringArray(argumentsList); // trying the daemon first - if (incrementalCaches != null) { - CompilerFacade daemon = KotlinCompilerClient.Companion.connectToCompilerServer(); + if (incrementalCaches != null && KotlinCompilerClient.Companion.isDaemonEnabled()) { + File libPath = CompilerRunnerUtil.getLibPath(environment.getKotlinPaths(), messageCollector); + CompilerId compilerId = CompilerId.makeCompilerId(libPath); + DaemonOptions daemonOptions = new DaemonOptions(); + // TODO: find a proper logger + Logger log = Logger.getAnonymousLogger(); + CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonOptions, log); if (daemon != null) { Integer res = KotlinCompilerClient.Companion.incrementalCompile(daemon, argsArray, incrementalCaches, out); return res.toString();