From 06b3ca8343654c842186bb7e95f0e3de3a41f25c Mon Sep 17 00:00:00 2001 From: ligee Date: Mon, 10 Aug 2015 18:15:29 +0200 Subject: [PATCH 01/14] Initial version of rmi-interface, compile server and kotlinr - commandline compile server client app --- .idea/modules.xml | 3 + build.xml | 24 ++++- compiler/rmi/kotlinr/kotlinr.iml | 13 +++ .../rmi/kotlinr/src/KotlinCompilerClient.kt | 49 ++++++++++ .../kotlinr/src/RemoteOutputStreamServer.kt | 42 +++++++++ compiler/rmi/rmi-interface/rmi-interface.iml | 13 +++ .../jetbrains/kotlin/rmi/CompilerFacade.java | 43 +++++++++ .../kotlin/rmi/RemoteOutputStream.java | 30 ++++++ compiler/rmi/rmi-server/rmi-server.iml | 16 ++++ .../kotlin/rmi/service/CompileServer.kt | 47 ++++++++++ .../kotlin/rmi/service/CompilerFacadeImpl.kt | 92 +++++++++++++++++++ .../rmi/service/RemoteOutputStreamClient.kt | 34 +++++++ 12 files changed, 405 insertions(+), 1 deletion(-) create mode 100644 compiler/rmi/kotlinr/kotlinr.iml create mode 100644 compiler/rmi/kotlinr/src/KotlinCompilerClient.kt create mode 100644 compiler/rmi/kotlinr/src/RemoteOutputStreamServer.kt create mode 100644 compiler/rmi/rmi-interface/rmi-interface.iml create mode 100644 compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerFacade.java create mode 100644 compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.java create mode 100644 compiler/rmi/rmi-server/rmi-server.iml create mode 100644 compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServer.kt create mode 100644 compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompilerFacadeImpl.kt create mode 100644 compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteOutputStreamClient.kt diff --git a/.idea/modules.xml b/.idea/modules.xml index 21e53538f25..8bd75e0c6c8 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -50,10 +50,13 @@ + + + diff --git a/build.xml b/build.xml index 53607d88ee4..6ef2272dc24 100644 --- a/build.xml +++ b/build.xml @@ -86,6 +86,8 @@ + + @@ -535,6 +537,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -839,7 +861,7 @@ depends="builtins,stdlib,core,reflection,pack-runtime,pack-runtime-sources"/> + + + + + + + + + + + + \ No newline at end of file diff --git a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt new file mode 100644 index 00000000000..e37a030ec69 --- /dev/null +++ b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt @@ -0,0 +1,49 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.kotlinr + +import org.jetbrains.kotlin.rmi.CompilerFacade +import java.rmi.registry.LocateRegistry +import kotlin.platform.platformStatic + +public class KotlinCompilerClient { + + companion object { + platformStatic public fun main(vararg args: String) { + val compilerObj = LocateRegistry.getRegistry("localhost", 17031).lookup("KotlinJvmCompilerService") + if (compilerObj == null) + println("Unable to find compiler service") + else { + val compiler = compilerObj as? CompilerFacade + if (compiler == null) + println("Unable to cast compiler service: ${compilerObj.javaClass}") + else { + println("Executing daemon compilation with args: " + args.joinToString(" ")) + val outStrm = RemoteOutputStreamServer(System.out) + try { + val res = compiler.remoteCompile(args, outStrm, CompilerFacade.OutputFormat.PLAIN) + println("Compilation result code: $res") + } + finally { + outStrm.close() + } + } + } + } + } +} + diff --git a/compiler/rmi/kotlinr/src/RemoteOutputStreamServer.kt b/compiler/rmi/kotlinr/src/RemoteOutputStreamServer.kt new file mode 100644 index 00000000000..9e3ed540291 --- /dev/null +++ b/compiler/rmi/kotlinr/src/RemoteOutputStreamServer.kt @@ -0,0 +1,42 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.kotlinr + +import org.jetbrains.kotlin.rmi.RemoteOutputStream +import java.io.OutputStream +import java.rmi.server.UnicastRemoteObject + + +class RemoteOutputStreamServer(val out: OutputStream) : RemoteOutputStream { + + init { + UnicastRemoteObject.exportObject(this, 0) + } + + override fun close() { + out.close() + UnicastRemoteObject.unexportObject(this, true) + } + + override fun write(data: ByteArray, start: Int, length: Int) { + out.write(data, start, length) + } + + override fun write(dataByte: Int) { + out.write(dataByte) + } +} diff --git a/compiler/rmi/rmi-interface/rmi-interface.iml b/compiler/rmi/rmi-interface/rmi-interface.iml new file mode 100644 index 00000000000..0bd974ce23a --- /dev/null +++ b/compiler/rmi/rmi-interface/rmi-interface.iml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerFacade.java b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerFacade.java new file mode 100644 index 00000000000..9401cd62316 --- /dev/null +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerFacade.java @@ -0,0 +1,43 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi; + +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache; +import java.rmi.Remote; +import java.rmi.RemoteException; +import java.util.Map; + + +public interface CompilerFacade extends Remote { + + enum OutputFormat implements java.io.Serializable { + PLAIN, + XML + } + + interface RemoteIncrementalCache extends Remote, IncrementalCache {} + + + int remoteCompile(String[] args, RemoteOutputStream errStream, OutputFormat outputFormat) throws RemoteException; + + int remoteIncrementalCompile( + String[] args, + Map caches, + RemoteOutputStream outputStream, + OutputFormat outputFormat + ) throws RemoteException; +} diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.java b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.java new file mode 100644 index 00000000000..09694f0fdfb --- /dev/null +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.java @@ -0,0 +1,30 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi; + +import java.io.IOException; +import java.rmi.Remote; +import java.rmi.RemoteException; + +public interface RemoteOutputStream extends Remote { + + void close() throws IOException, RemoteException; + + void write(byte[] data, int offset, int length) throws IOException, RemoteException; + + void write(int dataByte) throws IOException, RemoteException; +} diff --git a/compiler/rmi/rmi-server/rmi-server.iml b/compiler/rmi/rmi-server/rmi-server.iml new file mode 100644 index 00000000000..1083739c346 --- /dev/null +++ b/compiler/rmi/rmi-server/rmi-server.iml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServer.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServer.kt new file mode 100644 index 00000000000..1b2f1c6bf58 --- /dev/null +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServer.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.service + +import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler +import org.jetbrains.kotlin.rmi.CompilerFacade +import java.rmi.RMISecurityManager +import java.rmi.registry.LocateRegistry +import java.rmi.server.UnicastRemoteObject +import kotlin.platform.platformStatic + +public class CompileServer { + + companion object { + platformStatic public fun main(args: Array) { + if (System.getSecurityManager() == null) + System.setSecurityManager (RMISecurityManager()) + + val registry = LocateRegistry.createRegistry(17031); + + val server = CompilerFacadeImpl(K2JVMCompiler()) + try { + UnicastRemoteObject.unexportObject(server, false) + } + catch (e: java.rmi.NoSuchObjectException) { + // ignoring if object already exported + } + + val stub = UnicastRemoteObject.exportObject(server, 0) as CompilerFacade + registry.rebind ("KotlinJvmCompilerService", stub); + } + } +} \ No newline at end of file diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompilerFacadeImpl.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompilerFacadeImpl.kt new file mode 100644 index 00000000000..63bab51e845 --- /dev/null +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompilerFacadeImpl.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.service + +import org.jetbrains.kotlin.cli.common.CLICompiler +import org.jetbrains.kotlin.config.Services +import org.jetbrains.kotlin.incremental.components.UsageCollector +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents +import org.jetbrains.kotlin.rmi.CompilerFacade +import org.jetbrains.kotlin.rmi.RemoteOutputStream +import java.io.PrintStream +import java.rmi.server.UnicastRemoteObject +import java.util.concurrent.TimeUnit + + +class CompilerFacadeImpl>(val compiler: Compiler) : CompilerFacade, UnicastRemoteObject() { + + public class IncrementalCompilationComponentsImpl(val idToCache: Map): IncrementalCompilationComponents { + override fun getIncrementalCache(moduleId: String): IncrementalCache = idToCache[moduleId]!! + override fun getUsageCollector(): UsageCollector = UsageCollector.DO_NOTHING + } + + private fun createCompileServices(incrementalCaches: Map): Services = + Services.Builder() + .register(javaClass(), IncrementalCompilationComponentsImpl(incrementalCaches)) +// .register(javaClass(), object: CompilationCanceledStatus { +// override fun checkCanceled(): Unit = if (context.getCancelStatus().isCanceled()) throw CompilationCanceledException() +// }) + .build() + + fun usedMemoryKb(): Long { + System.gc() + val rt = Runtime.getRuntime() + return (rt.totalMemory() - rt.freeMemory()) / 1024 + } + + fun checked(args: Array, body: () -> R): R { + try { + if (args.none()) + throw IllegalArgumentException("Error: empty arguments list.") + println("Starting compilation with args: " + args.joinToString(" ")) + val startMem = usedMemoryKb() + val startTime = System.nanoTime() + val res = body() + val endTime = System.nanoTime() + val endMem = usedMemoryKb() + println("Done") + println("Elapsed time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") + println("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)") + return res + } + catch (e: Exception) { + println("Error: $e") + throw e + } + } + + override fun remoteCompile(args: Array, errStream: RemoteOutputStream, outputFormat: CompilerFacade.OutputFormat): Int = + checked(args) { + val strm = RemoteOutputStreamClient(errStream) + val printStrm = PrintStream(strm) + when (outputFormat) { + CompilerFacade.OutputFormat.PLAIN -> compiler.exec(printStrm, *args) + CompilerFacade.OutputFormat.XML -> compiler.execAndOutputXml(printStrm, Services.EMPTY, *args) + }.code + } + + override fun remoteIncrementalCompile(args: Array, caches: Map, errStream: RemoteOutputStream, outputFormat: CompilerFacade.OutputFormat): Int = + checked(args) { + val strm = RemoteOutputStreamClient(errStream) + val printStrm = PrintStream(strm) + when (outputFormat) { + CompilerFacade.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation") + CompilerFacade.OutputFormat.XML -> compiler.execAndOutputXml(printStrm, createCompileServices(caches), *args) + }.code + } +} diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteOutputStreamClient.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteOutputStreamClient.kt new file mode 100644 index 00000000000..33bd30b3006 --- /dev/null +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteOutputStreamClient.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.service + +import org.jetbrains.kotlin.rmi.RemoteOutputStream +import java.io.OutputStream + +class RemoteOutputStreamClient(val remote: RemoteOutputStream): OutputStream() { + override fun write(data: ByteArray) { + remote.write(data, 0, data.size()) + } + + override fun write(data: ByteArray, offset: Int, length: Int) { + remote.write(data, offset, length) + } + + override fun write(byte: Int) { + remote.write(byte) + } +} From b55894499fdaad17089998e07f9063205897a29c Mon Sep 17 00:00:00 2001 From: ligee Date: Tue, 11 Aug 2015 15:44:30 +0200 Subject: [PATCH 02/14] incremental compilation via daemon --- compiler/rmi/kotlinr/kotlinr.iml | 1 + .../rmi/kotlinr/src/KotlinCompilerClient.kt | 33 ++++++++++++--- .../src/RemoteIncrementalCacheServer.kt | 41 +++++++++++++++++++ .../kotlinr/src/RemoteOutputStreamServer.kt | 5 ++- compiler/rmi/rmi-interface/rmi-interface.iml | 1 + jps-plugin/jps-plugin.iml | 2 + .../compilerRunner/KotlinCompilerRunner.java | 36 ++++++++++++---- .../kotlin/jps/build/KotlinBuilder.kt | 19 +++++---- 8 files changed, 117 insertions(+), 21 deletions(-) create mode 100644 compiler/rmi/kotlinr/src/RemoteIncrementalCacheServer.kt diff --git a/compiler/rmi/kotlinr/kotlinr.iml b/compiler/rmi/kotlinr/kotlinr.iml index 240b6471a57..36eff60cdf2 100644 --- a/compiler/rmi/kotlinr/kotlinr.iml +++ b/compiler/rmi/kotlinr/kotlinr.iml @@ -9,5 +9,6 @@ + \ No newline at end of file diff --git a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt index e37a030ec69..e8be605c362 100644 --- a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt +++ b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt @@ -16,14 +16,17 @@ package org.jetbrains.kotlin.rmi.kotlinr +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache import org.jetbrains.kotlin.rmi.CompilerFacade +import java.io.OutputStream import java.rmi.registry.LocateRegistry import kotlin.platform.platformStatic public class KotlinCompilerClient { companion object { - platformStatic public fun main(vararg args: String) { + + public fun connectToCompilerServer(): CompilerFacade? { val compilerObj = LocateRegistry.getRegistry("localhost", 17031).lookup("KotlinJvmCompilerService") if (compilerObj == null) println("Unable to find compiler service") @@ -31,19 +34,39 @@ public class KotlinCompilerClient { val compiler = compilerObj as? CompilerFacade if (compiler == null) println("Unable to cast compiler service: ${compilerObj.javaClass}") - else { + return compiler + } + return null + } + + public fun incrementalCompile(compiler: CompilerFacade, args: Array, caches: Map, out: OutputStream): Int { + + val outStrm = RemoteOutputStreamServer(out) + val cacheServers = hashMapOf() + try { + caches.forEach { cacheServers.put( it.getKey(), RemoteIncrementalCacheServer( it.getValue())) } + val res = compiler.remoteIncrementalCompile(args, cacheServers, outStrm, CompilerFacade.OutputFormat.XML) + return res + } + finally { + cacheServers.forEach { it.getValue().disconnect() } + outStrm.disconnect() + } + } + + platformStatic public fun main(vararg args: String) { + connectToCompilerServer()?.let { println("Executing daemon compilation with args: " + args.joinToString(" ")) val outStrm = RemoteOutputStreamServer(System.out) try { - val res = compiler.remoteCompile(args, outStrm, CompilerFacade.OutputFormat.PLAIN) + val res = it.remoteCompile(args, outStrm, CompilerFacade.OutputFormat.PLAIN) println("Compilation result code: $res") } finally { - outStrm.close() + outStrm.disconnect() } } } - } } } diff --git a/compiler/rmi/kotlinr/src/RemoteIncrementalCacheServer.kt b/compiler/rmi/kotlinr/src/RemoteIncrementalCacheServer.kt new file mode 100644 index 00000000000..a924ad64f40 --- /dev/null +++ b/compiler/rmi/kotlinr/src/RemoteIncrementalCacheServer.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.kotlinr + +import org.jetbrains.kotlin.rmi.CompilerFacade +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache; +import java.rmi.server.UnicastRemoteObject + + +public class RemoteIncrementalCacheServer(val cache: IncrementalCache) : CompilerFacade.RemoteIncrementalCache { + + init { + UnicastRemoteObject.exportObject(this, 0) + } + + override fun getObsoletePackageParts(): Collection = cache.getObsoletePackageParts() + + override fun getPackageData(fqName: String): ByteArray? = cache.getPackageData(fqName) + + override fun close() { + cache.close() + } + + public fun disconnect() { + UnicastRemoteObject.unexportObject(this, true) + } +} \ No newline at end of file diff --git a/compiler/rmi/kotlinr/src/RemoteOutputStreamServer.kt b/compiler/rmi/kotlinr/src/RemoteOutputStreamServer.kt index 9e3ed540291..7a96d0dc927 100644 --- a/compiler/rmi/kotlinr/src/RemoteOutputStreamServer.kt +++ b/compiler/rmi/kotlinr/src/RemoteOutputStreamServer.kt @@ -27,9 +27,12 @@ class RemoteOutputStreamServer(val out: OutputStream) : RemoteOutputStream { UnicastRemoteObject.exportObject(this, 0) } + public fun disconnect() { + UnicastRemoteObject.unexportObject(this, true) + } + override fun close() { out.close() - UnicastRemoteObject.unexportObject(this, true) } override fun write(data: ByteArray, start: Int, length: Int) { diff --git a/compiler/rmi/rmi-interface/rmi-interface.iml b/compiler/rmi/rmi-interface/rmi-interface.iml index 0bd974ce23a..50c2d073f1a 100644 --- a/compiler/rmi/rmi-interface/rmi-interface.iml +++ b/compiler/rmi/rmi-interface/rmi-interface.iml @@ -7,6 +7,7 @@ + diff --git a/jps-plugin/jps-plugin.iml b/jps-plugin/jps-plugin.iml index bae91cce456..e092109dfa7 100644 --- a/jps-plugin/jps-plugin.iml +++ b/jps-plugin/jps-plugin.iml @@ -21,5 +21,7 @@ + + \ No newline at end of file diff --git a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java index 887f515087a..5da5119d20a 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java +++ b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java @@ -30,6 +30,9 @@ import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments; import org.jetbrains.kotlin.cli.common.messages.MessageCollector; import org.jetbrains.kotlin.cli.common.messages.MessageCollectorUtil; import org.jetbrains.kotlin.config.CompilerSettings; +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache; +import org.jetbrains.kotlin.rmi.CompilerFacade; +import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient; import org.jetbrains.kotlin.utils.UtilsPackage; import java.io.*; @@ -38,6 +41,7 @@ import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; import static org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation.NO_LOCATION; import static org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR; @@ -54,13 +58,14 @@ public class KotlinCompilerRunner { CompilerSettings compilerSettings, MessageCollector messageCollector, CompilerEnvironment environment, - File moduleFile, + Map incrementalCaches, File moduleFile, OutputItemsCollector collector ) { K2JVMCompilerArguments arguments = mergeBeans(commonArguments, k2jvmArguments); setupK2JvmArguments(moduleFile, arguments); - runCompiler(K2JVM_COMPILER, arguments, compilerSettings.getAdditionalArguments(), messageCollector, collector, environment); + runCompiler(K2JVM_COMPILER, arguments, compilerSettings.getAdditionalArguments(), messageCollector, collector, environment, + incrementalCaches); } public static void runK2JsCompiler( @@ -69,7 +74,7 @@ public class KotlinCompilerRunner { @NotNull CompilerSettings compilerSettings, @NotNull MessageCollector messageCollector, @NotNull CompilerEnvironment environment, - @NotNull OutputItemsCollector collector, + Map incrementalCaches, @NotNull OutputItemsCollector collector, @NotNull Collection sourceFiles, @NotNull List libraryFiles, @NotNull File outputFile @@ -77,7 +82,8 @@ public class KotlinCompilerRunner { K2JSCompilerArguments arguments = mergeBeans(commonArguments, k2jsArguments); setupK2JsArguments(outputFile, sourceFiles, libraryFiles, arguments); - runCompiler(K2JS_COMPILER, arguments, compilerSettings.getAdditionalArguments(), messageCollector, collector, environment); + runCompiler(K2JS_COMPILER, arguments, compilerSettings.getAdditionalArguments(), messageCollector, collector, environment, + incrementalCaches); } private static void runCompiler( @@ -86,12 +92,13 @@ public class KotlinCompilerRunner { String additionalArguments, MessageCollector messageCollector, OutputItemsCollector collector, - CompilerEnvironment environment + CompilerEnvironment environment, + Map incrementalCaches ) { ByteArrayOutputStream stream = new ByteArrayOutputStream(); PrintStream out = new PrintStream(stream); - String exitCode = execCompiler(compilerClassName, arguments, additionalArguments, environment, out, messageCollector); + String exitCode = execCompiler(compilerClassName, arguments, additionalArguments, environment, incrementalCaches, out, messageCollector); BufferedReader reader = new BufferedReader(new StringReader(stream.toString())); CompilerOutputParser.parseCompilerMessagesFromReader(messageCollector, reader, collector); @@ -107,6 +114,7 @@ public class KotlinCompilerRunner { CommonCompilerArguments arguments, String additionalArguments, CompilerEnvironment environment, + Map incrementalCaches, PrintStream out, MessageCollector messageCollector ) { @@ -116,13 +124,27 @@ public class KotlinCompilerRunner { List argumentsList = ArgumentUtils.convertArgumentsToStringList(arguments); argumentsList.addAll(StringUtil.split(additionalArguments, " ")); + String[] argsArray = ArrayUtil.toStringArray(argumentsList); + + // trying the daemon first + if (incrementalCaches != null) { + CompilerFacade daemon = KotlinCompilerClient.Companion.connectToCompilerServer(); + if (daemon != null) { + Integer res = KotlinCompilerClient.Companion.incrementalCompile(daemon, argsArray, incrementalCaches, out); + return res.toString(); + } + } + + // otherwise fallback to in-process + Object rc = CompilerRunnerUtil.invokeExecMethod( - compilerClassName, ArrayUtil.toStringArray(argumentsList), environment, messageCollector, out + compilerClassName, argsArray, environment, messageCollector, out ); // exec() returns an ExitCode object, class of which is loaded with a different class loader, // so we take it's contents through reflection return getReturnCodeFromObject(rc); + } catch (Throwable e) { MessageCollectorUtil.reportException(messageCollector, e); diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt index 4606b72da80..e207b2e7200 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt @@ -247,9 +247,10 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR filesToCompile: MultiMap, incrementalCaches: Map, messageCollector: MessageCollectorAdapter, project: JpsProject ): OutputItemsCollectorImpl? { + if (JpsUtils.isJsKotlinModule(chunk.representativeTarget())) { LOG.debug("Compiling to JS ${filesToCompile.values().size()} files in " + filesToCompile.keySet().map { it.getPresentableName() }.join()) - return compileToJs(chunk, commonArguments, environment, messageCollector, project) + return compileToJs(chunk, commonArguments, environment, null, messageCollector, project) } if (IncrementalCompilation.ENABLED) { @@ -279,7 +280,9 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR ) } - return compileToJvm(allCompiledFiles, chunk, commonArguments, context, dirtyFilesHolder, environment, filesToCompile, messageCollector) + return compileToJvm(allCompiledFiles, chunk, commonArguments, context, dirtyFilesHolder, environment, + incrementalCaches.mapKeysTo(HashMap(incrementalCaches.size()), { it.getKey().id }), + filesToCompile, messageCollector) } private fun createCompileEnvironment( @@ -436,8 +439,8 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR private fun compileToJs(chunk: ModuleChunk, commonArguments: CommonCompilerArguments, environment: CompilerEnvironment, - messageCollector: KotlinBuilder.MessageCollectorAdapter, - project: JpsProject + incrementalCaches: MutableMap?, + messageCollector: MessageCollectorAdapter, project: JpsProject ): OutputItemsCollectorImpl? { val outputItemCollector = OutputItemsCollectorImpl() @@ -468,7 +471,7 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR val compilerSettings = JpsKotlinCompilerSettings.getCompilerSettings(project) val k2JsArguments = JpsKotlinCompilerSettings.getK2JsCompilerArguments(project) - runK2JsCompiler(commonArguments, k2JsArguments, compilerSettings, messageCollector, environment, outputItemCollector, sourceFiles, libraryFiles, outputFile) + runK2JsCompiler(commonArguments, k2JsArguments, compilerSettings, messageCollector, environment, incrementalCaches, outputItemCollector, sourceFiles, libraryFiles, outputFile) return outputItemCollector } @@ -491,8 +494,8 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR context: CompileContext, dirtyFilesHolder: DirtyFilesHolder, environment: CompilerEnvironment, - filesToCompile: MultiMap, - messageCollector: KotlinBuilder.MessageCollectorAdapter + incrementalCaches: MutableMap?, + filesToCompile: MultiMap, messageCollector: MessageCollectorAdapter ): OutputItemsCollectorImpl? { val outputItemCollector = OutputItemsCollectorImpl() @@ -535,7 +538,7 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR + (if (totalRemovedFiles == 0) "" else " ($totalRemovedFiles removed files)") + " in " + filesToCompile.keySet().map { it.getPresentableName() }.join()) - runK2JvmCompiler(commonArguments, k2JvmArguments, compilerSettings, messageCollector, environment, moduleFile, outputItemCollector) + runK2JvmCompiler(commonArguments, k2JvmArguments, compilerSettings, messageCollector, environment, incrementalCaches, moduleFile, outputItemCollector) moduleFile.delete() return outputItemCollector From 33cead12e0dc73e0593f9a516f3793d047a3c621 Mon Sep 17 00:00:00 2001 From: ligee Date: Wed, 12 Aug 2015 14:56:31 +0200 Subject: [PATCH 03/14] working version of jps-plugin integration with daemon --- .idea/artifacts/KotlinJpsPlugin.xml | 2 ++ .../jetbrains/kotlin/rmi/CompilerFacade.java | 10 +++++-- .../kotlin/rmi/service/CompileServer.kt | 1 + .../kotlin/rmi/service/CompilerFacadeImpl.kt | 11 +++++--- .../service/RemoteIncrementalCacheClient.kt | 28 +++++++++++++++++++ 5 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt diff --git a/.idea/artifacts/KotlinJpsPlugin.xml b/.idea/artifacts/KotlinJpsPlugin.xml index c48f2a07a6f..7e9ed15d83a 100644 --- a/.idea/artifacts/KotlinJpsPlugin.xml +++ b/.idea/artifacts/KotlinJpsPlugin.xml @@ -19,6 +19,8 @@ + + \ No newline at end of file diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerFacade.java b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerFacade.java index 9401cd62316..adc151039fa 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerFacade.java +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerFacade.java @@ -16,9 +16,9 @@ package org.jetbrains.kotlin.rmi; -import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache; import java.rmi.Remote; import java.rmi.RemoteException; +import java.util.Collection; import java.util.Map; @@ -29,7 +29,13 @@ public interface CompilerFacade extends Remote { XML } - interface RemoteIncrementalCache extends Remote, IncrementalCache {} + interface RemoteIncrementalCache extends Remote { + Collection getObsoletePackageParts() throws RemoteException;; + + byte[] getPackageData(String fqName) throws RemoteException;; + + void close() throws RemoteException; + } int remoteCompile(String[] args, RemoteOutputStream errStream, OutputFormat outputFormat) throws RemoteException; diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServer.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServer.kt index 1b2f1c6bf58..f90f76485a7 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServer.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServer.kt @@ -18,6 +18,7 @@ package org.jetbrains.kotlin.rmi.service import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler import org.jetbrains.kotlin.rmi.CompilerFacade +import org.jetbrains.kotlin.service.CompilerFacadeImpl import java.rmi.RMISecurityManager import java.rmi.registry.LocateRegistry import java.rmi.server.UnicastRemoteObject diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompilerFacadeImpl.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompilerFacadeImpl.kt index 63bab51e845..156c409242b 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompilerFacadeImpl.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompilerFacadeImpl.kt @@ -14,15 +14,17 @@ * limitations under the License. */ -package org.jetbrains.kotlin.rmi.service +package org.jetbrains.kotlin.service import org.jetbrains.kotlin.cli.common.CLICompiler import org.jetbrains.kotlin.config.Services -import org.jetbrains.kotlin.incremental.components.UsageCollector +import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents import org.jetbrains.kotlin.rmi.CompilerFacade import org.jetbrains.kotlin.rmi.RemoteOutputStream +import org.jetbrains.kotlin.rmi.service.RemoteIncrementalCacheClient +import org.jetbrains.kotlin.rmi.service.RemoteOutputStreamClient import java.io.PrintStream import java.rmi.server.UnicastRemoteObject import java.util.concurrent.TimeUnit @@ -31,8 +33,9 @@ import java.util.concurrent.TimeUnit class CompilerFacadeImpl>(val compiler: Compiler) : CompilerFacade, UnicastRemoteObject() { public class IncrementalCompilationComponentsImpl(val idToCache: Map): IncrementalCompilationComponents { - override fun getIncrementalCache(moduleId: String): IncrementalCache = idToCache[moduleId]!! - override fun getUsageCollector(): UsageCollector = UsageCollector.DO_NOTHING + // perf: cheap object, but still the pattern may be costy if there are too many calls to cache with the same id (which seems not to be the case now) + override fun getIncrementalCache(moduleId: String): IncrementalCache = RemoteIncrementalCacheClient(idToCache[moduleId]!!) + override fun getLookupTracker(): LookupTracker = LookupTracker.DO_NOTHING } private fun createCompileServices(incrementalCaches: Map): Services = diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt new file mode 100644 index 00000000000..990a1c0885a --- /dev/null +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.service + +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache +import org.jetbrains.kotlin.rmi.CompilerFacade + +public class RemoteIncrementalCacheClient(val cache: CompilerFacade.RemoteIncrementalCache): IncrementalCache { + override fun getObsoletePackageParts(): Collection = cache.getObsoletePackageParts() + + override fun getPackageData(fqName: String): ByteArray = cache.getPackageData(fqName) + + override fun close(): Unit = cache.close() +} From 7ab2208fc5a671c10ab0fda88ae5a141b461da06 Mon Sep 17 00:00:00 2001 From: ligee Date: Sun, 16 Aug 2015 22:44:06 +0200 Subject: [PATCH 04/14] implementing daemon startup, basic check and restart machanisms, switching to kotlin everythere, additional logging and checks, in JPS plugin daemon compilation is off by default, turned on by "kotlin.daemon.enabled" property --- .../rmi/kotlinr/src/KotlinCompilerClient.kt | 177 +++++++++++++++-- .../src/RemoteIncrementalCacheServer.kt | 4 +- .../jetbrains/kotlin/rmi/CompileService.kt | 58 ++++++ .../jetbrains/kotlin/rmi/CompilerFacade.java | 49 ----- .../org/jetbrains/kotlin/rmi/DaemonParams.kt | 116 +++++++++++ ...utputStream.java => RemoteOutputStream.kt} | 19 +- .../kotlin/rmi/service/CompileDaemon.kt | 58 ++++++ .../kotlin/rmi/service/CompileServer.kt | 48 ----- .../kotlin/rmi/service/CompileServiceImpl.kt | 186 ++++++++++++++++++ .../kotlin/rmi/service/CompilerFacadeImpl.kt | 95 --------- .../kotlin/rmi/service/DaemonPermissions.kt | 91 +++++++++ .../service/RemoteIncrementalCacheClient.kt | 6 +- .../compilerRunner/CompilerRunnerUtil.java | 2 +- .../compilerRunner/KotlinCompilerRunner.java | 14 +- 14 files changed, 697 insertions(+), 226 deletions(-) create mode 100644 compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt delete mode 100644 compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerFacade.java create mode 100644 compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt rename compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/{RemoteOutputStream.java => RemoteOutputStream.kt} (57%) create mode 100644 compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt delete mode 100644 compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServer.kt create mode 100644 compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt delete mode 100644 compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompilerFacadeImpl.kt create mode 100644 compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/DaemonPermissions.kt diff --git a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt index e8be605c362..3dba28fc892 100644 --- a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt +++ b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt @@ -17,35 +17,146 @@ package org.jetbrains.kotlin.rmi.kotlinr import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache -import org.jetbrains.kotlin.rmi.CompilerFacade +import org.jetbrains.kotlin.rmi.* +import java.io.File import java.io.OutputStream +import java.rmi.ConnectException +import java.rmi.Remote import java.rmi.registry.LocateRegistry +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.ReentrantReadWriteLock +import java.util.logging.Logger +import kotlin.concurrent.read +import kotlin.concurrent.thread +import kotlin.concurrent.write import kotlin.platform.platformStatic +val DAEMON_STARTUP_TIMEOUT_MILIS = 10000L +val DAEMON_STARTUP_CHECH_INTERVAL_MILIS = 100L + public class KotlinCompilerClient { companion object { - public fun connectToCompilerServer(): CompilerFacade? { - val compilerObj = LocateRegistry.getRegistry("localhost", 17031).lookup("KotlinJvmCompilerService") - if (compilerObj == null) - println("Unable to find compiler service") - else { - val compiler = compilerObj as? CompilerFacade - if (compiler == null) - println("Unable to cast compiler service: ${compilerObj.javaClass}") - return compiler + val log: Logger by lazy { Logger.getLogger("KotlinDaemonClient") } + + private fun connectToService(compilerId: CompilerId, daemonOptions: DaemonOptions, log: Logger): CompileService? { + + val compilerObj = connectToDaemon(compilerId, daemonOptions, log) ?: return null // no registry - no daemon running + return compilerObj as? CompileService ?: + throw ClassCastException("Unable to cast compiler service, actual class received: ${compilerObj.javaClass}") + } + + private fun connectToDaemon(compilerId: CompilerId, daemonOptions: DaemonOptions, log: Logger): Remote? { + try { + val daemon = LocateRegistry.getRegistry("localhost", daemonOptions.port) + ?.lookup(COMPILER_SERVICE_RMI_NAME) + if (daemon != null) + return daemon + log.info("Failed to find compile daemon") + } + catch (e: ConnectException) { + log.info("Error connecting registry: " + e.toString()) + // ignoring it - processing below } return null } - public fun incrementalCompile(compiler: CompilerFacade, args: Array, caches: Map, out: OutputStream): Int { + + fun Process.isAlive() = + try { + this.exitValue() + false + } + catch (e: IllegalThreadStateException) { + true + } + + + private fun startDaemon(compilerId: CompilerId, daemonOptions: DaemonOptions, log: Logger) { + val javaExecutable = listOf(System.getProperty("java.home"), "bin", "java").joinToString(File.separator) + // TODO: add some specific environment variables to the cp and may be command line, to allow some specific startup configs + val args = listOf(javaExecutable, + "-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator), + COMPILER_DAEMON_CLASS_FQN) + + daemonOptions.asParams + + compilerId.asParams + log.info("Starting the daemon as: " + args.joinToString(" ")) + val processBuilder = ProcessBuilder(args).redirectErrorStream(true) + // assuming daemon process is deaf and (mostly) silent, so do not handle streams + val daemon = processBuilder.start() + + val lock = ReentrantReadWriteLock() + var isEchoRead = false + + val stdouThread = + thread { + daemon.getInputStream() + .reader() + .forEachLine { + if (daemonOptions.startEcho.isNotEmpty() && it.contains(daemonOptions.startEcho)) + lock.write { isEchoRead = true; return@forEachLine } + log.info("daemon: " + it) + } + } + // trying to wait for process + if (daemonOptions.startEcho.isNotEmpty()) { + log.info("waiting for daemon to respond") + var waitMillis: Long = DAEMON_STARTUP_TIMEOUT_MILIS / DAEMON_STARTUP_CHECH_INTERVAL_MILIS + while (waitMillis-- > 0) { + Thread.sleep(DAEMON_STARTUP_CHECH_INTERVAL_MILIS) + if (!daemon.isAlive() || lock.read { isEchoRead } == true) break; + } + if (!daemon.isAlive()) + throw Exception("Daemon terminated unexpectedly") + if (lock.read { isEchoRead } == false) + throw Exception("Unable to get response from daemon in $DAEMON_STARTUP_TIMEOUT_MILIS ms") + } + else + // without startEcho defined waiting for max timeout + Thread.sleep(DAEMON_STARTUP_TIMEOUT_MILIS) + // assuming that all important output is already done, the rest should be routed to the log by the daemon itself + if (stdouThread.isAlive) + // TODO: find better method to stop the thread, but seems it will require asynchronous consuming of the stream + lock.write { stdouThread.stop() } + } + + public fun checkCompilerId(compiler: CompileService, localId: CompilerId, log: Logger): Boolean { + val remoteId = compiler.getCompilerId() + log.info("remoteId = " + remoteId.toString()) + log.info("localId = " + localId.toString()) + return (localId.compilerVersion.isEmpty() || localId.compilerVersion == remoteId.compilerVersion) && + (localId.compilerClasspath.all { remoteId.compilerClasspath.contains(it) }) + } + + public fun connectToCompileService(compilerId: CompilerId, daemonOptions: DaemonOptions, log: Logger): CompileService? { + val service = connectToService(compilerId, daemonOptions, log) + if (service != null) { + if (checkCompilerId(service, compilerId, log)) { + log.info("Found the suitable daemon") + return service + } + log.info("Compiler identity don't match: " + compilerId.asParams.joinToString(" ")) + log.info("Shutdown the daemon") + service.shutdown() + // TODO: find more reliable way + Thread.sleep(1000) + log.info("Daemon shut down correctly, restarting") + } + else + log.info("cannot connect to Compile Daemon, trying to start") + startDaemon(compilerId, daemonOptions, log) + log.info("Daemon started, trying to connect") + return connectToService(compilerId, daemonOptions, log) + } + + public fun incrementalCompile(compiler: CompileService, args: Array, caches: Map, out: OutputStream): Int { val outStrm = RemoteOutputStreamServer(out) val cacheServers = hashMapOf() try { caches.forEach { cacheServers.put( it.getKey(), RemoteIncrementalCacheServer( it.getValue())) } - val res = compiler.remoteIncrementalCompile(args, cacheServers, outStrm, CompilerFacade.OutputFormat.XML) + val res = compiler.remoteIncrementalCompile(args, cacheServers, outStrm, CompileService.OutputFormat.XML) return res } finally { @@ -54,19 +165,51 @@ public class KotlinCompilerClient { } } + public fun isDaemonEnabled(): Boolean = System.getProperty(COMPILE_DAEMON_ENABLED_PROPERTY) != null + + platformStatic public fun main(vararg args: String) { - connectToCompilerServer()?.let { - println("Executing daemon compilation with args: " + args.joinToString(" ")) + val compilerId = CompilerId() + val daemonOptions = DaemonOptions() + val filteredArgs = args.asIterable().propParseFilter(compilerId, daemonOptions) + + if (compilerId.compilerClasspath.none()) { + // attempt to find compiler to use + log.info("compiler wasn't explicitly specified, attempt to find appropriate jar") + System.getProperty("java.class.path") + ?.split(File.pathSeparator) + ?.map { File(it).parent } + ?.distinct() + ?.map { it?.walk() + ?.firstOrNull { it.getName().equals(COMPILER_JAR_NAME, ignoreCase = true) } } + ?.filterNotNull() + ?.firstOrNull() + ?.let { compilerId.compilerClasspath = listOf(it.absolutePath)} + } + if (compilerId.compilerClasspath.none()) + throw IllegalArgumentException("Cannot find compiler jar") + else + log.info("desired compiler classpath: " + compilerId.compilerClasspath.joinToString(File.pathSeparator)) + + connectToCompileService(compilerId, daemonOptions, log)?.let { + log.info("Executing daemon compilation with args: " + args.joinToString(" ")) val outStrm = RemoteOutputStreamServer(System.out) try { - val res = it.remoteCompile(args, outStrm, CompilerFacade.OutputFormat.PLAIN) - println("Compilation result code: $res") + val memBefore = it.getUsedMemory() / 1024 + val startTime = System.nanoTime() + val res = it.remoteCompile(args, outStrm, CompileService.OutputFormat.PLAIN) + val endTime = System.nanoTime() + log.info("Compilation result code: $res") + val memAfter = it.getUsedMemory() / 1024 + log.info("Compilation time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") + log.info("Used memory $memAfter (${"%+d".format(memAfter - memBefore)} kb)") } finally { outStrm.disconnect() } } - } + ?: throw Exception("Unable to connect to daemon") + } } } diff --git a/compiler/rmi/kotlinr/src/RemoteIncrementalCacheServer.kt b/compiler/rmi/kotlinr/src/RemoteIncrementalCacheServer.kt index a924ad64f40..c2baf48baf0 100644 --- a/compiler/rmi/kotlinr/src/RemoteIncrementalCacheServer.kt +++ b/compiler/rmi/kotlinr/src/RemoteIncrementalCacheServer.kt @@ -16,12 +16,12 @@ package org.jetbrains.kotlin.rmi.kotlinr -import org.jetbrains.kotlin.rmi.CompilerFacade +import org.jetbrains.kotlin.rmi.CompileService import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache; import java.rmi.server.UnicastRemoteObject -public class RemoteIncrementalCacheServer(val cache: IncrementalCache) : CompilerFacade.RemoteIncrementalCache { +public class RemoteIncrementalCacheServer(val cache: IncrementalCache) : CompileService.RemoteIncrementalCache { init { UnicastRemoteObject.exportObject(this, 0) diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt new file mode 100644 index 00000000000..890c4eae466 --- /dev/null +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi + +import java.rmi.Remote +import java.rmi.RemoteException + +public interface CompileService : Remote { + + public enum class OutputFormat : java.io.Serializable { + PLAIN, + XML + } + + public interface RemoteIncrementalCache : Remote { + throws(RemoteException::class) + public fun getObsoletePackageParts(): Collection + + throws(RemoteException::class) + public fun getPackageData(fqName: String): ByteArray? + + throws(RemoteException::class) + public fun close() + } + + throws(RemoteException::class) + public fun getCompilerId(): CompilerId + + throws(RemoteException::class) + public fun getUsedMemory(): Long + + throws(RemoteException::class) + public fun shutdown() + + throws(RemoteException::class) + public fun remoteCompile(args: Array, errStream: RemoteOutputStream, outputFormat: OutputFormat): Int + + throws(RemoteException::class) + public fun remoteIncrementalCompile( + args: Array, + caches: Map, + outputStream: RemoteOutputStream, + outputFormat: OutputFormat): Int +} diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerFacade.java b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerFacade.java deleted file mode 100644 index adc151039fa..00000000000 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerFacade.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2010-2015 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.kotlin.rmi; - -import java.rmi.Remote; -import java.rmi.RemoteException; -import java.util.Collection; -import java.util.Map; - - -public interface CompilerFacade extends Remote { - - enum OutputFormat implements java.io.Serializable { - PLAIN, - XML - } - - interface RemoteIncrementalCache extends Remote { - Collection getObsoletePackageParts() throws RemoteException;; - - byte[] getPackageData(String fqName) throws RemoteException;; - - void close() throws RemoteException; - } - - - int remoteCompile(String[] args, RemoteOutputStream errStream, OutputFormat outputFormat) throws RemoteException; - - int remoteIncrementalCompile( - String[] args, - Map caches, - RemoteOutputStream outputStream, - OutputFormat outputFormat - ) throws RemoteException; -} diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt new file mode 100644 index 00000000000..a1dfb866393 --- /dev/null +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt @@ -0,0 +1,116 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi + +import java.io.File +import java.io.Serializable +import kotlin.platform.platformStatic +import kotlin.reflect.KMutableProperty1 +import kotlin.reflect.KProperty1 + + +public val COMPILER_JAR_NAME: String = "kotlin-compiler.jar" +public val COMPILER_SERVICE_RMI_NAME: String = "KotlinJvmCompilerService" +public val COMPILER_DAEMON_CLASS_FQN: String = "org.jetbrains.kotlin.rmi.service.CompileDaemon" +public val COMPILE_DAEMON_DEFAULT_PORT: Int = 17031 +public val COMPILE_DAEMON_ENABLED_PROPERTY: String ="kotlin.daemon.enabled" + + +fun> C.propToParams(p: P, conv: ((v: V) -> String) = { it.toString() } ) = + listOf("--daemon-" + p.name, conv(p.get(this))) + +class PropParser>(val dest: C, val prop: P, val parse: (s: String) -> V) { + fun apply(s: String) = prop.set(dest, parse(s)) +} + +fun Iterable.propParseFilter(parsers: List>) : Iterable { + var currentParser: PropParser<*,*,*>? = null + return filter { param -> + if (currentParser == null) { + currentParser = parsers.find { param.equals("--daemon-" + it.prop.name) } + if (currentParser != null) false + else true + } + else { + currentParser!!.apply(param) + currentParser = null + false + } + } +} + +// TODO: find out how to create more generic variant using first constructor +//fun C.propsToParams() { +// val kc = C::class +// kc.constructors.first(). +//} + +public interface CmdlineParams : Serializable { + public val asParams: Iterable + public val parsers: List> +} + +public fun Iterable.propParseFilter(vararg cs: CmdlineParams) : Iterable = + propParseFilter(cs.flatMap { it.parsers }) + + +public data class DaemonOptions( + public var port: Int = COMPILE_DAEMON_DEFAULT_PORT, + public var autoshutdownMemoryThreshold: Long = 0 /* 0 means unchecked */, + public var autoshutdownIdleSeconds: Int = 0 /* 0 means unchecked */, + public var startEcho: String = COMPILER_SERVICE_RMI_NAME +) : CmdlineParams { + + override val asParams: Iterable + get() = + propToParams(::port) + + propToParams(::autoshutdownMemoryThreshold) + + propToParams(::autoshutdownIdleSeconds) + + propToParams(::startEcho) + + override val parsers: List> + get() = listOf( PropParser(this, ::port, { it.toInt()}), + PropParser(this, ::autoshutdownMemoryThreshold, { it.toLong()}), + PropParser(this, ::autoshutdownIdleSeconds, { it.toInt()}), + PropParser(this, ::startEcho, { it.trim('"') })) +} + + +public data class CompilerId( + public var compilerClasspath: List = listOf(), + public var compilerVersion: String = "" + // TODO: checksum +) : CmdlineParams { + + override val asParams: Iterable + get() = + propToParams(::compilerClasspath, { it.joinToString(File.pathSeparator) }) + + propToParams(::compilerVersion) + + override val parsers: List> + get() = + listOf( PropParser(this, ::compilerClasspath, { it.trim('"').split(File.pathSeparator)}), + PropParser(this, ::compilerVersion, { it.trim('"') })) + + companion object { + public platformStatic fun makeCompilerId(libPath: File): CompilerId = + // TODO consider reading version and calculating checksum here + CompilerId(compilerClasspath = listOf(libPath.absolutePath)) + } +} + + diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.java b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.kt similarity index 57% rename from compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.java rename to compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.kt index 09694f0fdfb..a295bf34ba1 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.java +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/RemoteOutputStream.kt @@ -14,17 +14,20 @@ * limitations under the License. */ -package org.jetbrains.kotlin.rmi; +package org.jetbrains.kotlin.rmi -import java.io.IOException; -import java.rmi.Remote; -import java.rmi.RemoteException; +import java.io.IOException +import java.rmi.Remote +import java.rmi.RemoteException -public interface RemoteOutputStream extends Remote { +public interface RemoteOutputStream : Remote { - void close() throws IOException, RemoteException; + throws(IOException::class, RemoteException::class) + public fun close() - void write(byte[] data, int offset, int length) throws IOException, RemoteException; + throws(IOException::class, RemoteException::class) + public fun write(data: ByteArray, offset: Int, length: Int) - void write(int dataByte) throws IOException, RemoteException; + throws(IOException::class, RemoteException::class) + public fun write(dataByte: Int) } diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt new file mode 100644 index 00000000000..2d6b560dce0 --- /dev/null +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.service + +import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler +import org.jetbrains.kotlin.rmi.COMPILE_DAEMON_DEFAULT_PORT +import org.jetbrains.kotlin.rmi.CompilerId +import org.jetbrains.kotlin.rmi.DaemonOptions +import org.jetbrains.kotlin.rmi.propParseFilter +import org.jetbrains.kotlin.service.CompileServiceImpl +import java.rmi.RMISecurityManager +import java.rmi.registry.LocateRegistry +import kotlin.platform.platformStatic + +public class CompileDaemon { + + companion object { + + platformStatic public fun main(args: Array) { + + val compilerId = CompilerId() + val daemonOptions = DaemonOptions() + val filteredArgs = args.asIterable().propParseFilter(compilerId, daemonOptions) + + if (filteredArgs.any()) { + println("usage: ") + throw IllegalArgumentException("Unknown arguments") + } + + // TODO: find minimal set of permissions and restore security management +// if (System.getSecurityManager() == null) +// System.setSecurityManager (RMISecurityManager()) +// +// setDaemonPpermissions(daemonOptions.port) + + val registry = LocateRegistry.createRegistry(daemonOptions.port); + + val server = CompileServiceImpl(registry, K2JVMCompiler()) + + if (daemonOptions.startEcho.isNotEmpty()) + println(daemonOptions.startEcho) + } + } +} \ No newline at end of file diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServer.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServer.kt deleted file mode 100644 index f90f76485a7..00000000000 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServer.kt +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2010-2015 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.kotlin.rmi.service - -import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler -import org.jetbrains.kotlin.rmi.CompilerFacade -import org.jetbrains.kotlin.service.CompilerFacadeImpl -import java.rmi.RMISecurityManager -import java.rmi.registry.LocateRegistry -import java.rmi.server.UnicastRemoteObject -import kotlin.platform.platformStatic - -public class CompileServer { - - companion object { - platformStatic public fun main(args: Array) { - if (System.getSecurityManager() == null) - System.setSecurityManager (RMISecurityManager()) - - val registry = LocateRegistry.createRegistry(17031); - - val server = CompilerFacadeImpl(K2JVMCompiler()) - try { - UnicastRemoteObject.unexportObject(server, false) - } - catch (e: java.rmi.NoSuchObjectException) { - // ignoring if object already exported - } - - val stub = UnicastRemoteObject.exportObject(server, 0) as CompilerFacade - registry.rebind ("KotlinJvmCompilerService", stub); - } - } -} \ No newline at end of file diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt new file mode 100644 index 00000000000..244f957ebec --- /dev/null +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt @@ -0,0 +1,186 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.service + +import org.jetbrains.kotlin.cli.common.CLICompiler +import org.jetbrains.kotlin.config.Services +import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents +import org.jetbrains.kotlin.rmi.COMPILER_SERVICE_RMI_NAME +import org.jetbrains.kotlin.rmi.CompileService +import org.jetbrains.kotlin.rmi.CompilerId +import org.jetbrains.kotlin.rmi.RemoteOutputStream +import org.jetbrains.kotlin.rmi.service.RemoteIncrementalCacheClient +import org.jetbrains.kotlin.rmi.service.RemoteOutputStreamClient +import java.io.File +import java.io.FileNotFoundException +import java.io.IOException +import java.io.PrintStream +import java.net.URLClassLoader +import java.rmi.registry.Registry +import java.rmi.server.UnicastRemoteObject +import java.util.* +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.ReentrantReadWriteLock +import java.util.jar.Manifest +import java.util.logging.Logger +import kotlin.concurrent.read +import kotlin.concurrent.write + + +class CompileServiceImpl>(val registry: Registry, val compiler: Compiler) : CompileService, UnicastRemoteObject() { + + private val rwlock = ReentrantReadWriteLock() + private var alive = false + private val selfCompilerId by lazy { + // TODO: add classpath checksum calculated on init, add it to the interface and use to decide whether it was changed during the lifetime of the daemon + CompilerId( + compilerClasspath = System.getProperty("java.class.path") + ?.split(File.pathSeparator) + ?.map { File(it) } + ?.filter { it.exists() } + ?.map { it.absolutePath } + ?: listOf(), + compilerVersion = loadKotlinVersionFromResource() + ) + } + + init { + // assuming logically synchronized + try { + // cleanup for the case of incorrect restart + UnicastRemoteObject.unexportObject(this, false) + } + catch (e: java.rmi.NoSuchObjectException) { + // ignoring if object already exported + } + + val stub = UnicastRemoteObject.exportObject(this, 0) as CompileService + // TODO: use version-specific name + registry.rebind (COMPILER_SERVICE_RMI_NAME, stub); + alive = true + } + + public class IncrementalCompilationComponentsImpl(val idToCache: Map): IncrementalCompilationComponents { + // perf: cheap object, but still the pattern may be costly if there are too many calls to cache with the same id (which seems not to be the case now) + override fun getIncrementalCache(moduleId: String): IncrementalCache = RemoteIncrementalCacheClient(idToCache[moduleId]!!) + override fun getLookupTracker(): LookupTracker = LookupTracker.DO_NOTHING + } + + private fun createCompileServices(incrementalCaches: Map): Services = + Services.Builder() + .register(javaClass(), IncrementalCompilationComponentsImpl(incrementalCaches)) +// .register(javaClass(), object: CompilationCanceledStatus { +// override fun checkCanceled(): Unit = if (context.getCancelStatus().isCanceled()) throw CompilationCanceledException() +// }) + .build() + + + fun usedMemory(): Long { + System.gc() + val rt = Runtime.getRuntime() + return (rt.totalMemory() - rt.freeMemory()) + } + + private fun loadKotlinVersionFromResource(): String { + (javaClass.classLoader as? URLClassLoader) + ?.findResource("META-INF/MANIFEST.MF") + ?.let { + try { + return Manifest(it.openStream()).mainAttributes.getValue("Implementation-Version") ?: "" + } + catch (e: IOException) {} + } + return "" + } + + + fun checkedCompile(args: Array, body: () -> R): R { + try { + if (args.none()) + throw IllegalArgumentException("Error: empty arguments list.") + println("Starting compilation with args: " + args.joinToString(" ")) + val startMem = usedMemory() / 1024 + val startTime = System.nanoTime() + val res = body() + val endTime = System.nanoTime() + val endMem = usedMemory() / 1024 + println("Done") + println("Elapsed time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") + println("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)") + return res + } + catch (e: Exception) { + println("Error: $e") + throw e + } + } + + fun ifAlive(body: () -> R): R = rwlock.read { + if (!alive) throw IllegalStateException("Kotlin Compiler Service is not in alive state") + else body() + } + + fun ifAliveExclusive(body: () -> R): R = rwlock.write { + if (!alive) throw IllegalStateException("Kotlin Compiler Service is not in alive state") + else body() + } + + fun spy(msg: String, body: () -> R): R { + val res = body() + println(msg + " = " + res.toString()) + return res + } + + override fun getCompilerId(): CompilerId = ifAlive { selfCompilerId } + + override fun getUsedMemory(): Long = ifAlive { usedMemory() } + + override fun shutdown() { + ifAliveExclusive { + println("Shutdown started") + alive = false + UnicastRemoteObject.unexportObject(this, true) + println("Shutdown complete") + } + } + + override fun remoteCompile(args: Array, errStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int = + ifAlive { + checkedCompile(args) { + val strm = RemoteOutputStreamClient(errStream) + val printStrm = PrintStream(strm) + when (outputFormat) { + CompileService.OutputFormat.PLAIN -> compiler.exec(printStrm, *args) + CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStrm, Services.EMPTY, *args) + }.code + } + } + + override fun remoteIncrementalCompile(args: Array, caches: Map, errStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int = + ifAlive { + checkedCompile(args) { + val strm = RemoteOutputStreamClient(errStream) + val printStrm = PrintStream(strm) + when (outputFormat) { + CompileService.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation") + CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStrm, createCompileServices(caches), *args) + }.code + } + } +} diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompilerFacadeImpl.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompilerFacadeImpl.kt deleted file mode 100644 index 156c409242b..00000000000 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompilerFacadeImpl.kt +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2010-2015 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.kotlin.service - -import org.jetbrains.kotlin.cli.common.CLICompiler -import org.jetbrains.kotlin.config.Services -import org.jetbrains.kotlin.incremental.components.LookupTracker -import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache -import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents -import org.jetbrains.kotlin.rmi.CompilerFacade -import org.jetbrains.kotlin.rmi.RemoteOutputStream -import org.jetbrains.kotlin.rmi.service.RemoteIncrementalCacheClient -import org.jetbrains.kotlin.rmi.service.RemoteOutputStreamClient -import java.io.PrintStream -import java.rmi.server.UnicastRemoteObject -import java.util.concurrent.TimeUnit - - -class CompilerFacadeImpl>(val compiler: Compiler) : CompilerFacade, UnicastRemoteObject() { - - public class IncrementalCompilationComponentsImpl(val idToCache: Map): IncrementalCompilationComponents { - // perf: cheap object, but still the pattern may be costy if there are too many calls to cache with the same id (which seems not to be the case now) - override fun getIncrementalCache(moduleId: String): IncrementalCache = RemoteIncrementalCacheClient(idToCache[moduleId]!!) - override fun getLookupTracker(): LookupTracker = LookupTracker.DO_NOTHING - } - - private fun createCompileServices(incrementalCaches: Map): Services = - Services.Builder() - .register(javaClass(), IncrementalCompilationComponentsImpl(incrementalCaches)) -// .register(javaClass(), object: CompilationCanceledStatus { -// override fun checkCanceled(): Unit = if (context.getCancelStatus().isCanceled()) throw CompilationCanceledException() -// }) - .build() - - fun usedMemoryKb(): Long { - System.gc() - val rt = Runtime.getRuntime() - return (rt.totalMemory() - rt.freeMemory()) / 1024 - } - - fun checked(args: Array, body: () -> R): R { - try { - if (args.none()) - throw IllegalArgumentException("Error: empty arguments list.") - println("Starting compilation with args: " + args.joinToString(" ")) - val startMem = usedMemoryKb() - val startTime = System.nanoTime() - val res = body() - val endTime = System.nanoTime() - val endMem = usedMemoryKb() - println("Done") - println("Elapsed time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") - println("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)") - return res - } - catch (e: Exception) { - println("Error: $e") - throw e - } - } - - override fun remoteCompile(args: Array, errStream: RemoteOutputStream, outputFormat: CompilerFacade.OutputFormat): Int = - checked(args) { - val strm = RemoteOutputStreamClient(errStream) - val printStrm = PrintStream(strm) - when (outputFormat) { - CompilerFacade.OutputFormat.PLAIN -> compiler.exec(printStrm, *args) - CompilerFacade.OutputFormat.XML -> compiler.execAndOutputXml(printStrm, Services.EMPTY, *args) - }.code - } - - override fun remoteIncrementalCompile(args: Array, caches: Map, errStream: RemoteOutputStream, outputFormat: CompilerFacade.OutputFormat): Int = - checked(args) { - val strm = RemoteOutputStreamClient(errStream) - val printStrm = PrintStream(strm) - when (outputFormat) { - CompilerFacade.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation") - CompilerFacade.OutputFormat.XML -> compiler.execAndOutputXml(printStrm, createCompileServices(caches), *args) - }.code - } -} diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/DaemonPermissions.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/DaemonPermissions.kt new file mode 100644 index 00000000000..51d88d97837 --- /dev/null +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/DaemonPermissions.kt @@ -0,0 +1,91 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.rmi.service + +import java.io.FilePermission +import java.net.SocketPermission +import java.security.CodeSource +import java.security.Permission +import java.security.PermissionCollection +import java.security.Policy +import java.util.ArrayList +import java.util.Collections +import java.util.Enumeration +import java.util.PropertyPermission + + +public class DaemonPolicy(val port: Int) : Policy() { + + private fun createPermissions(): PermissionCollection { + val perms = DaemonPermissionCollection() + + val socketPermission = SocketPermission("localhost:$port-", "connect, accept, resolve") + val propertyPermission = PropertyPermission("localhost:$port", "read") + //val filePermission = FilePermission("<>", "read") + + perms.add(socketPermission) + perms.add(propertyPermission) + //perms.add(filePermission) + + return perms + } + + private val perms: PermissionCollection by lazy { createPermissions() } + + override fun getPermissions(codesource: CodeSource?): PermissionCollection { + return perms + } +} + + +class DaemonPermissionCollection : PermissionCollection() { + var perms = ArrayList() + + override fun add(p: Permission) { + perms.add(p) + } + + override fun implies(p: Permission): Boolean { + val i = perms.iterator() + while (i.hasNext()) { + if (i.next().implies(p)) { + return true + } + } + return false + } + + override fun elements(): Enumeration { + return Collections.enumeration(perms) + } + + override fun isReadOnly(): Boolean { + return false + } +// +// companion object { +// +// private val serialVersionUID = 614300921365729272L +// } + +} + + +public fun setDaemonPpermissions(port: Int) { + Policy.setPolicy(DaemonPolicy(port)) +} + diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt index 990a1c0885a..900c1e088d2 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCacheClient.kt @@ -17,12 +17,12 @@ package org.jetbrains.kotlin.rmi.service import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache -import org.jetbrains.kotlin.rmi.CompilerFacade +import org.jetbrains.kotlin.rmi.CompileService -public class RemoteIncrementalCacheClient(val cache: CompilerFacade.RemoteIncrementalCache): IncrementalCache { +public class RemoteIncrementalCacheClient(val cache: CompileService.RemoteIncrementalCache): IncrementalCache { override fun getObsoletePackageParts(): Collection = cache.getObsoletePackageParts() - override fun getPackageData(fqName: String): ByteArray = cache.getPackageData(fqName) + override fun getPackageData(fqName: String): ByteArray? = cache.getPackageData(fqName) override fun close(): Unit = cache.close() } diff --git a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/CompilerRunnerUtil.java b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/CompilerRunnerUtil.java index e8701e0c366..cc97d0376f3 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/CompilerRunnerUtil.java +++ b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/CompilerRunnerUtil.java @@ -56,7 +56,7 @@ public class CompilerRunnerUtil { } @Nullable - private static File getLibPath(@NotNull KotlinPaths paths, @NotNull MessageCollector messageCollector) { + public static File getLibPath(@NotNull KotlinPaths paths, @NotNull MessageCollector messageCollector) { File libs = paths.getLibPath(); if (libs.exists() && !libs.isFile()) return libs; diff --git a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java index 5da5119d20a..e313bd74d6a 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java +++ b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java @@ -31,7 +31,9 @@ import org.jetbrains.kotlin.cli.common.messages.MessageCollector; import org.jetbrains.kotlin.cli.common.messages.MessageCollectorUtil; import org.jetbrains.kotlin.config.CompilerSettings; import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache; -import org.jetbrains.kotlin.rmi.CompilerFacade; +import org.jetbrains.kotlin.rmi.CompileService; +import org.jetbrains.kotlin.rmi.CompilerId; +import org.jetbrains.kotlin.rmi.DaemonOptions; import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient; import org.jetbrains.kotlin.utils.UtilsPackage; @@ -42,6 +44,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.logging.Logger; import static org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation.NO_LOCATION; import static org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR; @@ -127,8 +130,13 @@ public class KotlinCompilerRunner { String[] argsArray = ArrayUtil.toStringArray(argumentsList); // trying the daemon first - if (incrementalCaches != null) { - CompilerFacade daemon = KotlinCompilerClient.Companion.connectToCompilerServer(); + if (incrementalCaches != null && KotlinCompilerClient.Companion.isDaemonEnabled()) { + File libPath = CompilerRunnerUtil.getLibPath(environment.getKotlinPaths(), messageCollector); + CompilerId compilerId = CompilerId.makeCompilerId(libPath); + DaemonOptions daemonOptions = new DaemonOptions(); + // TODO: find a proper logger + Logger log = Logger.getAnonymousLogger(); + CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonOptions, log); if (daemon != null) { Integer res = KotlinCompilerClient.Companion.incrementalCompile(daemon, argsArray, incrementalCaches, out); return res.toString(); From d8be8313398aa5649e6ac21cbc6c914943048cc0 Mon Sep 17 00:00:00 2001 From: ligee Date: Mon, 17 Aug 2015 10:57:41 +0200 Subject: [PATCH 05/14] Switching to stream for logging to simplify interaction with use sites --- .../rmi/kotlinr/src/KotlinCompilerClient.kt | 64 +++++++++---------- .../compilerRunner/KotlinCompilerRunner.java | 9 ++- 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt index 3dba28fc892..c38fa555925 100644 --- a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt +++ b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt @@ -20,12 +20,12 @@ import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache import org.jetbrains.kotlin.rmi.* import java.io.File import java.io.OutputStream +import java.io.PrintStream import java.rmi.ConnectException import java.rmi.Remote import java.rmi.registry.LocateRegistry import java.util.concurrent.TimeUnit import java.util.concurrent.locks.ReentrantReadWriteLock -import java.util.logging.Logger import kotlin.concurrent.read import kotlin.concurrent.thread import kotlin.concurrent.write @@ -38,25 +38,23 @@ public class KotlinCompilerClient { companion object { - val log: Logger by lazy { Logger.getLogger("KotlinDaemonClient") } + private fun connectToService(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream): CompileService? { - private fun connectToService(compilerId: CompilerId, daemonOptions: DaemonOptions, log: Logger): CompileService? { - - val compilerObj = connectToDaemon(compilerId, daemonOptions, log) ?: return null // no registry - no daemon running + val compilerObj = connectToDaemon(compilerId, daemonOptions, errStream) ?: return null // no registry - no daemon running return compilerObj as? CompileService ?: throw ClassCastException("Unable to cast compiler service, actual class received: ${compilerObj.javaClass}") } - private fun connectToDaemon(compilerId: CompilerId, daemonOptions: DaemonOptions, log: Logger): Remote? { + private fun connectToDaemon(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream): Remote? { try { val daemon = LocateRegistry.getRegistry("localhost", daemonOptions.port) ?.lookup(COMPILER_SERVICE_RMI_NAME) if (daemon != null) return daemon - log.info("Failed to find compile daemon") + errStream.println("[daemon client] daemon not found") } catch (e: ConnectException) { - log.info("Error connecting registry: " + e.toString()) + errStream.println("[daemon client] cannot connect to registry: " + e.getMessage()) // ignoring it - processing below } return null @@ -73,7 +71,7 @@ public class KotlinCompilerClient { } - private fun startDaemon(compilerId: CompilerId, daemonOptions: DaemonOptions, log: Logger) { + private fun startDaemon(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream) { val javaExecutable = listOf(System.getProperty("java.home"), "bin", "java").joinToString(File.separator) // TODO: add some specific environment variables to the cp and may be command line, to allow some specific startup configs val args = listOf(javaExecutable, @@ -81,7 +79,7 @@ public class KotlinCompilerClient { COMPILER_DAEMON_CLASS_FQN) + daemonOptions.asParams + compilerId.asParams - log.info("Starting the daemon as: " + args.joinToString(" ")) + errStream.println("[daemon client] starting the daemon as: " + args.joinToString(" ")) val processBuilder = ProcessBuilder(args).redirectErrorStream(true) // assuming daemon process is deaf and (mostly) silent, so do not handle streams val daemon = processBuilder.start() @@ -96,12 +94,12 @@ public class KotlinCompilerClient { .forEachLine { if (daemonOptions.startEcho.isNotEmpty() && it.contains(daemonOptions.startEcho)) lock.write { isEchoRead = true; return@forEachLine } - log.info("daemon: " + it) + errStream.println("[daemon] " + it) } } // trying to wait for process if (daemonOptions.startEcho.isNotEmpty()) { - log.info("waiting for daemon to respond") + errStream.println("[daemon client] waiting for daemon to respond") var waitMillis: Long = DAEMON_STARTUP_TIMEOUT_MILIS / DAEMON_STARTUP_CHECH_INTERVAL_MILIS while (waitMillis-- > 0) { Thread.sleep(DAEMON_STARTUP_CHECH_INTERVAL_MILIS) @@ -121,33 +119,33 @@ public class KotlinCompilerClient { lock.write { stdouThread.stop() } } - public fun checkCompilerId(compiler: CompileService, localId: CompilerId, log: Logger): Boolean { + public fun checkCompilerId(compiler: CompileService, localId: CompilerId, errStream: PrintStream): Boolean { val remoteId = compiler.getCompilerId() - log.info("remoteId = " + remoteId.toString()) - log.info("localId = " + localId.toString()) + errStream.println("[daemon client] remoteId = " + remoteId.toString()) + errStream.println("[daemon client] localId = " + localId.toString()) return (localId.compilerVersion.isEmpty() || localId.compilerVersion == remoteId.compilerVersion) && (localId.compilerClasspath.all { remoteId.compilerClasspath.contains(it) }) } - public fun connectToCompileService(compilerId: CompilerId, daemonOptions: DaemonOptions, log: Logger): CompileService? { - val service = connectToService(compilerId, daemonOptions, log) + public fun connectToCompileService(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream): CompileService? { + val service = connectToService(compilerId, daemonOptions, errStream) if (service != null) { - if (checkCompilerId(service, compilerId, log)) { - log.info("Found the suitable daemon") + if (checkCompilerId(service, compilerId, errStream)) { + errStream.println("[daemon client] found the suitable daemon") return service } - log.info("Compiler identity don't match: " + compilerId.asParams.joinToString(" ")) - log.info("Shutdown the daemon") + errStream.println("[daemon client] compiler identity don't match: " + compilerId.asParams.joinToString(" ")) + errStream.println("[daemon client] shutdown the daemon") service.shutdown() // TODO: find more reliable way Thread.sleep(1000) - log.info("Daemon shut down correctly, restarting") + errStream.println("[daemon client] daemon shut down correctly, restarting") } else - log.info("cannot connect to Compile Daemon, trying to start") - startDaemon(compilerId, daemonOptions, log) - log.info("Daemon started, trying to connect") - return connectToService(compilerId, daemonOptions, log) + errStream.println("[daemon client] cannot connect to Compile Daemon, trying to start") + startDaemon(compilerId, daemonOptions, errStream) + errStream.println("[daemon client] daemon started, trying to connect") + return connectToService(compilerId, daemonOptions, errStream) } public fun incrementalCompile(compiler: CompileService, args: Array, caches: Map, out: OutputStream): Int { @@ -175,7 +173,7 @@ public class KotlinCompilerClient { if (compilerId.compilerClasspath.none()) { // attempt to find compiler to use - log.info("compiler wasn't explicitly specified, attempt to find appropriate jar") + println("compiler wasn't explicitly specified, attempt to find appropriate jar") System.getProperty("java.class.path") ?.split(File.pathSeparator) ?.map { File(it).parent } @@ -189,20 +187,20 @@ public class KotlinCompilerClient { if (compilerId.compilerClasspath.none()) throw IllegalArgumentException("Cannot find compiler jar") else - log.info("desired compiler classpath: " + compilerId.compilerClasspath.joinToString(File.pathSeparator)) + println("desired compiler classpath: " + compilerId.compilerClasspath.joinToString(File.pathSeparator)) - connectToCompileService(compilerId, daemonOptions, log)?.let { - log.info("Executing daemon compilation with args: " + args.joinToString(" ")) + connectToCompileService(compilerId, daemonOptions, System.out)?.let { + println("Executing daemon compilation with args: " + args.joinToString(" ")) val outStrm = RemoteOutputStreamServer(System.out) try { val memBefore = it.getUsedMemory() / 1024 val startTime = System.nanoTime() val res = it.remoteCompile(args, outStrm, CompileService.OutputFormat.PLAIN) val endTime = System.nanoTime() - log.info("Compilation result code: $res") + println("Compilation result code: $res") val memAfter = it.getUsedMemory() / 1024 - log.info("Compilation time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") - log.info("Used memory $memAfter (${"%+d".format(memAfter - memBefore)} kb)") + println("Compilation time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") + println("Used memory $memAfter (${"%+d".format(memAfter - memBefore)} kb)") } finally { outStrm.disconnect() diff --git a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java index e313bd74d6a..4ce09052a0f 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java +++ b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java @@ -16,6 +16,7 @@ package org.jetbrains.kotlin.compilerRunner; +import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.text.StringUtil; import com.intellij.util.ArrayUtil; import com.intellij.util.Function; @@ -44,7 +45,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.logging.Logger; import static org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation.NO_LOCATION; import static org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR; @@ -132,11 +132,10 @@ public class KotlinCompilerRunner { // trying the daemon first if (incrementalCaches != null && KotlinCompilerClient.Companion.isDaemonEnabled()) { File libPath = CompilerRunnerUtil.getLibPath(environment.getKotlinPaths(), messageCollector); - CompilerId compilerId = CompilerId.makeCompilerId(libPath); + CompilerId compilerId = CompilerId.makeCompilerId(new File(libPath, "kotlin-compiler.jar")); DaemonOptions daemonOptions = new DaemonOptions(); - // TODO: find a proper logger - Logger log = Logger.getAnonymousLogger(); - CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonOptions, log); + // TODO: find proper stream to report daemon connection progress + CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonOptions, System.out); if (daemon != null) { Integer res = KotlinCompilerClient.Companion.incrementalCompile(daemon, argsArray, incrementalCaches, out); return res.toString(); From f08476cba94d60d917de67a433bd0567da7771ba Mon Sep 17 00:00:00 2001 From: ligee Date: Mon, 17 Aug 2015 12:47:14 +0200 Subject: [PATCH 06/14] Adding daemon stop command to kotlinr, minor fixes --- .../rmi/kotlinr/src/KotlinCompilerClient.kt | 145 +++++++++++------- .../org/jetbrains/kotlin/rmi/DaemonParams.kt | 12 +- 2 files changed, 99 insertions(+), 58 deletions(-) diff --git a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt index c38fa555925..ef19a22b6cb 100644 --- a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt +++ b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt @@ -31,8 +31,8 @@ import kotlin.concurrent.thread import kotlin.concurrent.write import kotlin.platform.platformStatic -val DAEMON_STARTUP_TIMEOUT_MILIS = 10000L -val DAEMON_STARTUP_CHECH_INTERVAL_MILIS = 100L +val DAEMON_STARTUP_TIMEOUT_MS = 10000L +val DAEMON_STARTUP_CHECK_INTERVAL_MS = 100L public class KotlinCompilerClient { @@ -54,7 +54,7 @@ public class KotlinCompilerClient { errStream.println("[daemon client] daemon not found") } catch (e: ConnectException) { - errStream.println("[daemon client] cannot connect to registry: " + e.getMessage()) + errStream.println("[daemon client] cannot connect to registry: " + (e.getCause()?.getMessage() ?: e.getMessage() ?: "unknown exception")) // ignoring it - processing below } return null @@ -97,26 +97,30 @@ public class KotlinCompilerClient { errStream.println("[daemon] " + it) } } - // trying to wait for process - if (daemonOptions.startEcho.isNotEmpty()) { - errStream.println("[daemon client] waiting for daemon to respond") - var waitMillis: Long = DAEMON_STARTUP_TIMEOUT_MILIS / DAEMON_STARTUP_CHECH_INTERVAL_MILIS - while (waitMillis-- > 0) { - Thread.sleep(DAEMON_STARTUP_CHECH_INTERVAL_MILIS) - if (!daemon.isAlive() || lock.read { isEchoRead } == true) break; + try { + // trying to wait for process + if (daemonOptions.startEcho.isNotEmpty()) { + errStream.println("[daemon client] waiting for daemon to respond") + var waitMillis: Long = DAEMON_STARTUP_TIMEOUT_MS / DAEMON_STARTUP_CHECK_INTERVAL_MS + while (waitMillis-- > 0) { + Thread.sleep(DAEMON_STARTUP_CHECK_INTERVAL_MS) + if (!daemon.isAlive() || lock.read { isEchoRead } == true) break; + } + if (!daemon.isAlive()) + throw Exception("Daemon terminated unexpectedly") + if (lock.read { isEchoRead } == false) + throw Exception("Unable to get response from daemon in $DAEMON_STARTUP_TIMEOUT_MS ms") } - if (!daemon.isAlive()) - throw Exception("Daemon terminated unexpectedly") - if (lock.read { isEchoRead } == false) - throw Exception("Unable to get response from daemon in $DAEMON_STARTUP_TIMEOUT_MILIS ms") - } - else + else // without startEcho defined waiting for max timeout - Thread.sleep(DAEMON_STARTUP_TIMEOUT_MILIS) - // assuming that all important output is already done, the rest should be routed to the log by the daemon itself - if (stdouThread.isAlive) + Thread.sleep(DAEMON_STARTUP_TIMEOUT_MS) + } + finally { + // assuming that all important output is already done, the rest should be routed to the log by the daemon itself + if (stdouThread.isAlive) // TODO: find better method to stop the thread, but seems it will require asynchronous consuming of the stream - lock.write { stdouThread.stop() } + lock.write { stdouThread.stop() } + } } public fun checkCompilerId(compiler: CompileService, localId: CompilerId, errStream: PrintStream): Boolean { @@ -127,22 +131,26 @@ public class KotlinCompilerClient { (localId.compilerClasspath.all { remoteId.compilerClasspath.contains(it) }) } - public fun connectToCompileService(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream): CompileService? { + public fun connectToCompileService(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream, autostart: Boolean = true, checkId: Boolean = true): CompileService? { val service = connectToService(compilerId, daemonOptions, errStream) if (service != null) { - if (checkCompilerId(service, compilerId, errStream)) { + if (!checkId || checkCompilerId(service, compilerId, errStream)) { errStream.println("[daemon client] found the suitable daemon") return service } errStream.println("[daemon client] compiler identity don't match: " + compilerId.asParams.joinToString(" ")) + if (!autostart) return null; errStream.println("[daemon client] shutdown the daemon") service.shutdown() // TODO: find more reliable way Thread.sleep(1000) errStream.println("[daemon client] daemon shut down correctly, restarting") } - else - errStream.println("[daemon client] cannot connect to Compile Daemon, trying to start") + else { + if (!autostart) return null; + else errStream.println("[daemon client] cannot connect to Compile Daemon, trying to start") + } + startDaemon(compilerId, daemonOptions, errStream) errStream.println("[daemon client] daemon started, trying to connect") return connectToService(compilerId, daemonOptions, errStream) @@ -165,48 +173,73 @@ public class KotlinCompilerClient { public fun isDaemonEnabled(): Boolean = System.getProperty(COMPILE_DAEMON_ENABLED_PROPERTY) != null + data class ClientOptions( + public var stop: Boolean = false + ) :CmdlineParams { + override val asParams: Iterable + get() = + if (stop) listOf("stop") else listOf() + + override val parsers: List> + get() = listOf( BoolPropParser(this, ::stop)) + } platformStatic public fun main(vararg args: String) { val compilerId = CompilerId() val daemonOptions = DaemonOptions() - val filteredArgs = args.asIterable().propParseFilter(compilerId, daemonOptions) + val clientOptions = ClientOptions() + val filteredArgs = args.asIterable().propParseFilter(compilerId, daemonOptions, clientOptions) - if (compilerId.compilerClasspath.none()) { - // attempt to find compiler to use - println("compiler wasn't explicitly specified, attempt to find appropriate jar") - System.getProperty("java.class.path") - ?.split(File.pathSeparator) - ?.map { File(it).parent } - ?.distinct() - ?.map { it?.walk() - ?.firstOrNull { it.getName().equals(COMPILER_JAR_NAME, ignoreCase = true) } } - ?.filterNotNull() - ?.firstOrNull() - ?.let { compilerId.compilerClasspath = listOf(it.absolutePath)} + if (!clientOptions.stop) { + if (compilerId.compilerClasspath.none()) { + // attempt to find compiler to use + println("compiler wasn't explicitly specified, attempt to find appropriate jar") + System.getProperty("java.class.path") + ?.split(File.pathSeparator) + ?.map { File(it).parent } + ?.distinct() + ?.map { + it?.walk() + ?.firstOrNull { it.getName().equals(COMPILER_JAR_NAME, ignoreCase = true) } + } + ?.filterNotNull() + ?.firstOrNull() + ?.let { compilerId.compilerClasspath = listOf(it.absolutePath) } + } + if (compilerId.compilerClasspath.none()) + throw IllegalArgumentException("Cannot find compiler jar") + else + println("desired compiler classpath: " + compilerId.compilerClasspath.joinToString(File.pathSeparator)) } - if (compilerId.compilerClasspath.none()) - throw IllegalArgumentException("Cannot find compiler jar") - else - println("desired compiler classpath: " + compilerId.compilerClasspath.joinToString(File.pathSeparator)) - connectToCompileService(compilerId, daemonOptions, System.out)?.let { - println("Executing daemon compilation with args: " + args.joinToString(" ")) - val outStrm = RemoteOutputStreamServer(System.out) - try { - val memBefore = it.getUsedMemory() / 1024 - val startTime = System.nanoTime() - val res = it.remoteCompile(args, outStrm, CompileService.OutputFormat.PLAIN) - val endTime = System.nanoTime() - println("Compilation result code: $res") - val memAfter = it.getUsedMemory() / 1024 - println("Compilation time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") - println("Used memory $memAfter (${"%+d".format(memAfter - memBefore)} kb)") + connectToCompileService(compilerId, daemonOptions, System.out, autostart = !clientOptions.stop, checkId = !clientOptions.stop)?.let { + when { + clientOptions.stop -> { + println("Shutdown the daemon") + it.shutdown() + println("Daemon shut down successfully") } - finally { - outStrm.disconnect() + else -> { + println("Executing daemon compilation with args: " + args.joinToString(" ")) + val outStrm = RemoteOutputStreamServer(System.out) + try { + val memBefore = it.getUsedMemory() / 1024 + val startTime = System.nanoTime() + val res = it.remoteCompile(args, outStrm, CompileService.OutputFormat.PLAIN) + val endTime = System.nanoTime() + println("Compilation result code: $res") + val memAfter = it.getUsedMemory() / 1024 + println("Compilation time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") + println("Used memory $memAfter (${"%+d".format(memAfter - memBefore)} kb)") + } + finally { + outStrm.disconnect() + } } } - ?: throw Exception("Unable to connect to daemon") + } + ?: if (clientOptions.stop) println("No daemon found to shut down") + else throw Exception("Unable to connect to daemon") } } } diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt index a1dfb866393..03ed6fc9ab9 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt @@ -33,16 +33,24 @@ public val COMPILE_DAEMON_ENABLED_PROPERTY: String ="kotlin.daemon.enabled" fun> C.propToParams(p: P, conv: ((v: V) -> String) = { it.toString() } ) = listOf("--daemon-" + p.name, conv(p.get(this))) -class PropParser>(val dest: C, val prop: P, val parse: (s: String) -> V) { +open class PropParser>(val dest: C, val prop: P, val parse: (s: String) -> V) { fun apply(s: String) = prop.set(dest, parse(s)) } +class BoolPropParser>(dest: C, prop: P): PropParser(dest, prop, { true }) + fun Iterable.propParseFilter(parsers: List>) : Iterable { var currentParser: PropParser<*,*,*>? = null return filter { param -> if (currentParser == null) { currentParser = parsers.find { param.equals("--daemon-" + it.prop.name) } - if (currentParser != null) false + if (currentParser != null) { + if (currentParser is BoolPropParser<*,*>) { + currentParser!!.apply("") + currentParser = null + } + false + } else true } else { From 39d6592e1fb027fb05e6b719e08e078bcac79d7f Mon Sep 17 00:00:00 2001 From: ligee Date: Mon, 17 Aug 2015 19:13:03 +0200 Subject: [PATCH 07/14] Adding compiler id digest calculation and check to detect replaced compiler jar, minor fixes and refactorings --- .../rmi/kotlinr/src/KotlinCompilerClient.kt | 32 ++++++------ .../org/jetbrains/kotlin/rmi/DaemonParams.kt | 50 ++++++++++++++++++- .../kotlin/rmi/service/CompileDaemon.kt | 3 +- .../kotlin/rmi/service/CompileServiceImpl.kt | 38 +++++++------- .../compilerRunner/KotlinCompilerRunner.java | 4 +- 5 files changed, 91 insertions(+), 36 deletions(-) diff --git a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt index ef19a22b6cb..e752f82a901 100644 --- a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt +++ b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt @@ -31,13 +31,22 @@ import kotlin.concurrent.thread import kotlin.concurrent.write import kotlin.platform.platformStatic -val DAEMON_STARTUP_TIMEOUT_MS = 10000L -val DAEMON_STARTUP_CHECK_INTERVAL_MS = 100L +fun Process.isAlive() = + try { + this.exitValue() + false + } + catch (e: IllegalThreadStateException) { + true + } public class KotlinCompilerClient { companion object { + val DAEMON_STARTUP_TIMEOUT_MS = 10000L + val DAEMON_STARTUP_CHECK_INTERVAL_MS = 100L + private fun connectToService(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream): CompileService? { val compilerObj = connectToDaemon(compilerId, daemonOptions, errStream) ?: return null // no registry - no daemon running @@ -61,16 +70,6 @@ public class KotlinCompilerClient { } - fun Process.isAlive() = - try { - this.exitValue() - false - } - catch (e: IllegalThreadStateException) { - true - } - - private fun startDaemon(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream) { val javaExecutable = listOf(System.getProperty("java.home"), "bin", "java").joinToString(File.separator) // TODO: add some specific environment variables to the cp and may be command line, to allow some specific startup configs @@ -128,7 +127,8 @@ public class KotlinCompilerClient { errStream.println("[daemon client] remoteId = " + remoteId.toString()) errStream.println("[daemon client] localId = " + localId.toString()) return (localId.compilerVersion.isEmpty() || localId.compilerVersion == remoteId.compilerVersion) && - (localId.compilerClasspath.all { remoteId.compilerClasspath.contains(it) }) + (localId.compilerClasspath.all { remoteId.compilerClasspath.contains(it) }) && + (localId.compilerDigest.isEmpty() || remoteId.compilerDigest.isEmpty() || localId.compilerDigest == remoteId.compilerDigest) } public fun connectToCompileService(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream, autostart: Boolean = true, checkId: Boolean = true): CompileService? { @@ -210,6 +210,8 @@ public class KotlinCompilerClient { throw IllegalArgumentException("Cannot find compiler jar") else println("desired compiler classpath: " + compilerId.compilerClasspath.joinToString(File.pathSeparator)) + + compilerId.updateDigest() } connectToCompileService(compilerId, daemonOptions, System.out, autostart = !clientOptions.stop, checkId = !clientOptions.stop)?.let { @@ -220,12 +222,12 @@ public class KotlinCompilerClient { println("Daemon shut down successfully") } else -> { - println("Executing daemon compilation with args: " + args.joinToString(" ")) + println("Executing daemon compilation with args: " + filteredArgs.joinToString(" ")) val outStrm = RemoteOutputStreamServer(System.out) try { val memBefore = it.getUsedMemory() / 1024 val startTime = System.nanoTime() - val res = it.remoteCompile(args, outStrm, CompileService.OutputFormat.PLAIN) + val res = it.remoteCompile(filteredArgs.toArrayList().toTypedArray(), outStrm, CompileService.OutputFormat.PLAIN) val endTime = System.nanoTime() println("Compilation result code: $res") val memAfter = it.getUsedMemory() / 1024 diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt index 03ed6fc9ab9..36fa8ad067f 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt @@ -18,6 +18,8 @@ package org.jetbrains.kotlin.rmi import java.io.File import java.io.Serializable +import java.security.DigestInputStream +import java.security.MessageDigest import kotlin.platform.platformStatic import kotlin.reflect.KMutableProperty1 import kotlin.reflect.KProperty1 @@ -98,8 +100,44 @@ public data class DaemonOptions( } +val COMPILER_ID_DIGEST = "MD5" + + +fun updateSingleFileDigest(file: File, md: MessageDigest) { + val stream = DigestInputStream(file.inputStream(), md) + val buf = ByteArray(1024) + while (stream.read(buf) == buf.size()) {} + stream.close() +} + +fun updateForAllClasses(dir: File, md: MessageDigest) { + dir.walk().forEach { updateEntryDigest(it, md) } +} + +fun updateEntryDigest(entry: File, md: MessageDigest) { + when { + entry.isDirectory + -> updateForAllClasses(entry, md) + entry.isFile && + (entry.getName().endsWith(".class", ignoreCase = true) || + entry.getName().endsWith(".jar", ignoreCase = true)) + -> updateSingleFileDigest(entry, md) + // else skip + } +} + +fun Iterable.getFilesClasspathDigest(): String { + val md = MessageDigest.getInstance(COMPILER_ID_DIGEST) + this.forEach { updateEntryDigest(it, md) } + return md.digest().joinToString("", transform = { "%02x".format(it) }) +} + +fun Iterable.getClasspathDigest(): String = map { File(it) }.getFilesClasspathDigest() + + public data class CompilerId( public var compilerClasspath: List = listOf(), + public var compilerDigest: String = "", public var compilerVersion: String = "" // TODO: checksum ) : CmdlineParams { @@ -107,17 +145,25 @@ public data class CompilerId( override val asParams: Iterable get() = propToParams(::compilerClasspath, { it.joinToString(File.pathSeparator) }) + + propToParams(::compilerDigest) + propToParams(::compilerVersion) override val parsers: List> get() = listOf( PropParser(this, ::compilerClasspath, { it.trim('"').split(File.pathSeparator)}), + PropParser(this, ::compilerDigest, { it.trim('"') }), PropParser(this, ::compilerVersion, { it.trim('"') })) + public fun updateDigest() { + compilerDigest = compilerClasspath.getClasspathDigest() + } + companion object { - public platformStatic fun makeCompilerId(libPath: File): CompilerId = + public platformStatic fun makeCompilerId(vararg paths: File): CompilerId = makeCompilerId(paths.asIterable()) + + public platformStatic fun makeCompilerId(paths: Iterable): CompilerId = // TODO consider reading version and calculating checksum here - CompilerId(compilerClasspath = listOf(libPath.absolutePath)) + CompilerId(compilerClasspath = paths.map { it.absolutePath }, compilerDigest = paths.getFilesClasspathDigest()) } } diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt index 2d6b560dce0..e6252f09853 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt @@ -48,8 +48,9 @@ public class CompileDaemon { // setDaemonPpermissions(daemonOptions.port) val registry = LocateRegistry.createRegistry(daemonOptions.port); + val compiler = K2JVMCompiler() - val server = CompileServiceImpl(registry, K2JVMCompiler()) + val server = CompileServiceImpl(registry, compiler, compilerId, daemonOptions) if (daemonOptions.startEcho.isNotEmpty()) println(daemonOptions.startEcho) diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt index 244f957ebec..6ec830a659e 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt @@ -17,14 +17,12 @@ package org.jetbrains.kotlin.service import org.jetbrains.kotlin.cli.common.CLICompiler +import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents -import org.jetbrains.kotlin.rmi.COMPILER_SERVICE_RMI_NAME -import org.jetbrains.kotlin.rmi.CompileService -import org.jetbrains.kotlin.rmi.CompilerId -import org.jetbrains.kotlin.rmi.RemoteOutputStream +import org.jetbrains.kotlin.rmi.* import org.jetbrains.kotlin.rmi.service.RemoteIncrementalCacheClient import org.jetbrains.kotlin.rmi.service.RemoteOutputStreamClient import java.io.File @@ -43,22 +41,28 @@ import kotlin.concurrent.read import kotlin.concurrent.write -class CompileServiceImpl>(val registry: Registry, val compiler: Compiler) : CompileService, UnicastRemoteObject() { +class CompileServiceImpl>( + val registry: Registry, + val compiler: Compiler, + val selfCompilerId: CompilerId, + val daemonOptions: DaemonOptions +) : CompileService, UnicastRemoteObject() { private val rwlock = ReentrantReadWriteLock() private var alive = false - private val selfCompilerId by lazy { - // TODO: add classpath checksum calculated on init, add it to the interface and use to decide whether it was changed during the lifetime of the daemon - CompilerId( - compilerClasspath = System.getProperty("java.class.path") - ?.split(File.pathSeparator) - ?.map { File(it) } - ?.filter { it.exists() } - ?.map { it.absolutePath } - ?: listOf(), - compilerVersion = loadKotlinVersionFromResource() - ) - } + + // TODO: consider matching compilerId coming from outside with actual one +// private val selfCompilerId by lazy { +// CompilerId( +// compilerClasspath = System.getProperty("java.class.path") +// ?.split(File.pathSeparator) +// ?.map { File(it) } +// ?.filter { it.exists() } +// ?.map { it.absolutePath } +// ?: listOf(), +// compilerVersion = loadKotlinVersionFromResource() +// ) +// } init { // assuming logically synchronized diff --git a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java index 4ce09052a0f..4921e35768b 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java +++ b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java @@ -132,10 +132,12 @@ public class KotlinCompilerRunner { // trying the daemon first if (incrementalCaches != null && KotlinCompilerClient.Companion.isDaemonEnabled()) { File libPath = CompilerRunnerUtil.getLibPath(environment.getKotlinPaths(), messageCollector); + // TODO: it may be a good idea to cache the compilerId, since making it means calculating digest over jar(s) and if \\ + // the lifetime of JPS process is small anyway, we can neglect the probability of changed compiler CompilerId compilerId = CompilerId.makeCompilerId(new File(libPath, "kotlin-compiler.jar")); DaemonOptions daemonOptions = new DaemonOptions(); // TODO: find proper stream to report daemon connection progress - CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonOptions, System.out); + CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonOptions, System.out, true, true); if (daemon != null) { Integer res = KotlinCompilerClient.Companion.incrementalCompile(daemon, argsArray, incrementalCaches, out); return res.toString(); From 5fee180d0950f00f71c297b601aff04dcf504afe Mon Sep 17 00:00:00 2001 From: ligee Date: Tue, 18 Aug 2015 17:44:20 +0200 Subject: [PATCH 08/14] Daemon log, controlling it from launching, proper stdout/err redirection technique, params to control jvm options of the daemon --- .../rmi/kotlinr/src/KotlinCompilerClient.kt | 16 ++-- .../org/jetbrains/kotlin/rmi/DaemonParams.kt | 12 +++ .../kotlin/rmi/service/CompileDaemon.kt | 79 ++++++++++++++++++- .../kotlin/rmi/service/CompileServiceImpl.kt | 18 +++-- 4 files changed, 108 insertions(+), 17 deletions(-) diff --git a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt index e752f82a901..754a74405d0 100644 --- a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt +++ b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt @@ -70,12 +70,13 @@ public class KotlinCompilerClient { } - private fun startDaemon(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream) { + private fun startDaemon(compilerId: CompilerId, daemonLaunchingOptions: DaemonLaunchingOptions, daemonOptions: DaemonOptions, errStream: PrintStream) { val javaExecutable = listOf(System.getProperty("java.home"), "bin", "java").joinToString(File.separator) // TODO: add some specific environment variables to the cp and may be command line, to allow some specific startup configs val args = listOf(javaExecutable, - "-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator), - COMPILER_DAEMON_CLASS_FQN) + + "-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator)) + + daemonLaunchingOptions.jvmParams + + COMPILER_DAEMON_CLASS_FQN + daemonOptions.asParams + compilerId.asParams errStream.println("[daemon client] starting the daemon as: " + args.joinToString(" ")) @@ -131,7 +132,7 @@ public class KotlinCompilerClient { (localId.compilerDigest.isEmpty() || remoteId.compilerDigest.isEmpty() || localId.compilerDigest == remoteId.compilerDigest) } - public fun connectToCompileService(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream, autostart: Boolean = true, checkId: Boolean = true): CompileService? { + public fun connectToCompileService(compilerId: CompilerId, daemonLaunchingOptions: DaemonLaunchingOptions, daemonOptions: DaemonOptions, errStream: PrintStream, autostart: Boolean = true, checkId: Boolean = true): CompileService? { val service = connectToService(compilerId, daemonOptions, errStream) if (service != null) { if (!checkId || checkCompilerId(service, compilerId, errStream)) { @@ -151,7 +152,7 @@ public class KotlinCompilerClient { else errStream.println("[daemon client] cannot connect to Compile Daemon, trying to start") } - startDaemon(compilerId, daemonOptions, errStream) + startDaemon(compilerId, daemonLaunchingOptions, daemonOptions, errStream) errStream.println("[daemon client] daemon started, trying to connect") return connectToService(compilerId, daemonOptions, errStream) } @@ -187,8 +188,9 @@ public class KotlinCompilerClient { platformStatic public fun main(vararg args: String) { val compilerId = CompilerId() val daemonOptions = DaemonOptions() + val daemonLaunchingOptions = DaemonLaunchingOptions() val clientOptions = ClientOptions() - val filteredArgs = args.asIterable().propParseFilter(compilerId, daemonOptions, clientOptions) + val filteredArgs = args.asIterable().propParseFilter(compilerId, daemonOptions, daemonLaunchingOptions, clientOptions) if (!clientOptions.stop) { if (compilerId.compilerClasspath.none()) { @@ -214,7 +216,7 @@ public class KotlinCompilerClient { compilerId.updateDigest() } - connectToCompileService(compilerId, daemonOptions, System.out, autostart = !clientOptions.stop, checkId = !clientOptions.stop)?.let { + connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, System.out, autostart = !clientOptions.stop, checkId = !clientOptions.stop)?.let { when { clientOptions.stop -> { println("Shutdown the daemon") diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt index 36fa8ad067f..58706ea4979 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt @@ -78,6 +78,18 @@ public fun Iterable.propParseFilter(vararg cs: CmdlineParams) : Iterable propParseFilter(cs.flatMap { it.parsers }) +public data class DaemonLaunchingOptions( + public var jvmParams: List = listOf() +) : CmdlineParams { + + override val asParams: Iterable + get() = + propToParams(::jvmParams, { it.joinToString("##") }) // TODO: consider some other options rather than using potentially dangerous delimiter + + override val parsers: List> + get() = listOf( PropParser(this, ::jvmParams, { it.split("##")})) // TODO: see appropriate comment in asParams +} + public data class DaemonOptions( public var port: Int = COMPILE_DAEMON_DEFAULT_PORT, public var autoshutdownMemoryThreshold: Long = 0 /* 0 means unchecked */, diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt index e6252f09853..54b7927c34a 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt @@ -17,19 +17,83 @@ package org.jetbrains.kotlin.rmi.service import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler -import org.jetbrains.kotlin.rmi.COMPILE_DAEMON_DEFAULT_PORT import org.jetbrains.kotlin.rmi.CompilerId import org.jetbrains.kotlin.rmi.DaemonOptions import org.jetbrains.kotlin.rmi.propParseFilter import org.jetbrains.kotlin.service.CompileServiceImpl +import java.io.OutputStream +import java.io.PrintStream import java.rmi.RMISecurityManager import java.rmi.registry.LocateRegistry +import java.text.SimpleDateFormat +import java.util.* +import java.util.logging.LogManager +import java.util.logging.Logger import kotlin.platform.platformStatic +class LogStream(name: String) : OutputStream() { + + val log by lazy { Logger.getLogger(name) } + + val lineBuf = StringBuilder() + + override fun write(byte: Int) { + if (byte.toChar() == '\n') flush() + else lineBuf.append(byte.toChar()) + } + + override fun write(data: ByteArray, offset: Int, length: Int) { + var ofs = offset + var lineStart = ofs + while (ofs < length) { + if (data[ofs].toChar() == '\n') { + flush(data, lineStart, ofs - lineStart) + lineStart = ofs + 1 + } + ofs++ + } + if (lineStart < length) + lineBuf.append(data, lineStart, length - lineStart) + } + + fun flush(data: ByteArray, offset: Int, length: Int) { + if (offset == 0 && length == data.size()) + log.info(lineBuf.toString() + data.toString()) + else + log.info(lineBuf.toString() + data.toString().substring(offset, length)) + lineBuf.setLength(0) + } + + override fun flush() { + log.info(lineBuf.toString()) + lineBuf.setLength(0) + } +} + + public class CompileDaemon { companion object { + init { + val logPath: String = System.getProperty("kotlin.daemon.log.path")?.trimEnd('/','\\') ?: "%t" + val logTime: String = SimpleDateFormat("yyyy-MM-dd.HH-mm-ss-SSS").format(Date()) + val cfg: String = + "handlers = java.util.logging.FileHandler\n" + + "java.util.logging.FileHandler.level = ALL\n" + + "java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter\n" + + "java.util.logging.FileHandler.encoding = UTF-8\n" + + "java.util.logging.FileHandler.limit = 1073741824\n" + // 1Mb + "java.util.logging.FileHandler.count = 3\n" + + "java.util.logging.FileHandler.append = false\n" + + "java.util.logging.FileHandler.pattern = $logPath/kotlin-daemon.$logTime.%g.log\n" + + "java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s\\n\n" + + LogManager.getLogManager().readConfiguration(cfg.byteInputStream()) + } + + val log by lazy { Logger.getLogger("daemon") } + platformStatic public fun main(args: Array) { val compilerId = CompilerId() @@ -37,10 +101,14 @@ public class CompileDaemon { val filteredArgs = args.asIterable().propParseFilter(compilerId, daemonOptions) if (filteredArgs.any()) { - println("usage: ") + val helpLine = "usage: " + log.info(helpLine) + println(helpLine) throw IllegalArgumentException("Unknown arguments") } + log.info("starting daemon") + // TODO: find minimal set of permissions and restore security management // if (System.getSecurityManager() == null) // System.setSecurityManager (RMISecurityManager()) @@ -54,6 +122,13 @@ public class CompileDaemon { if (daemonOptions.startEcho.isNotEmpty()) println(daemonOptions.startEcho) + + // this stops redirected streams reader(s) on the client side and prevent some situations with hanging threads + System.out.close() + System.err.close() + + System.setErr(PrintStream(LogStream("stderr"))) + System.setOut(PrintStream(LogStream("stdout"))) } } } \ No newline at end of file diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt index 6ec830a659e..f5bb4fede4c 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt @@ -48,6 +48,8 @@ class CompileServiceImpl>( val daemonOptions: DaemonOptions ) : CompileService, UnicastRemoteObject() { + val log by lazy { Logger.getLogger("compiler") } + private val rwlock = ReentrantReadWriteLock() private var alive = false @@ -118,19 +120,19 @@ class CompileServiceImpl>( try { if (args.none()) throw IllegalArgumentException("Error: empty arguments list.") - println("Starting compilation with args: " + args.joinToString(" ")) + log.info("Starting compilation with args: " + args.joinToString(" ")) val startMem = usedMemory() / 1024 val startTime = System.nanoTime() val res = body() val endTime = System.nanoTime() val endMem = usedMemory() / 1024 - println("Done") - println("Elapsed time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") - println("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)") + log.info("Done") + log.info("Elapsed time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") + log.info("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)") return res } catch (e: Exception) { - println("Error: $e") + log.info("Error: $e") throw e } } @@ -147,7 +149,7 @@ class CompileServiceImpl>( fun spy(msg: String, body: () -> R): R { val res = body() - println(msg + " = " + res.toString()) + log.info(msg + " = " + res.toString()) return res } @@ -157,10 +159,10 @@ class CompileServiceImpl>( override fun shutdown() { ifAliveExclusive { - println("Shutdown started") + log.info("Shutdown started") alive = false UnicastRemoteObject.unexportObject(this, true) - println("Shutdown complete") + log.info("Shutdown complete") } } From 6e3ce69faa8263eeffd8648a209a956b3b814f55 Mon Sep 17 00:00:00 2001 From: ligee Date: Wed, 19 Aug 2015 15:36:26 +0200 Subject: [PATCH 09/14] Shutting down daemon from kotlin plugin on idea exit, passing jvm params to daemon launch --- .idea/artifacts/KotlinPlugin.xml | 2 ++ compiler/rmi/kotlinr/src/KotlinCompilerClient.kt | 12 ++++++++++++ .../src/org/jetbrains/kotlin/rmi/DaemonParams.kt | 1 + idea/idea.iml | 1 + .../kotlin/idea/PluginStartupComponent.java | 2 ++ .../kotlin/compilerRunner/KotlinCompilerRunner.java | 5 ++++- 6 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.idea/artifacts/KotlinPlugin.xml b/.idea/artifacts/KotlinPlugin.xml index 3476cd93123..1ff79edf65e 100644 --- a/.idea/artifacts/KotlinPlugin.xml +++ b/.idea/artifacts/KotlinPlugin.xml @@ -44,6 +44,8 @@ + + diff --git a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt index 754a74405d0..ab5324dda2b 100644 --- a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt +++ b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt @@ -157,6 +157,11 @@ public class KotlinCompilerClient { return connectToService(compilerId, daemonOptions, errStream) } + public fun shutdownCompileService(): Unit { + KotlinCompilerClient.connectToCompileService(CompilerId(), DaemonLaunchingOptions(), DaemonOptions(), System.out, autostart = false, checkId = false) + ?.shutdown() + } + public fun incrementalCompile(compiler: CompileService, args: Array, caches: Map, out: OutputStream): Int { val outStrm = RemoteOutputStreamServer(out) @@ -174,6 +179,13 @@ public class KotlinCompilerClient { public fun isDaemonEnabled(): Boolean = System.getProperty(COMPILE_DAEMON_ENABLED_PROPERTY) != null + public fun configureDaemonLaunchingOptions(opts: DaemonLaunchingOptions) { + System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)?.let { + // TODO: find better way to pass and parse jvm options for daemon + opts.jvmParams = it.split("##") + } + } + data class ClientOptions( public var stop: Boolean = false ) :CmdlineParams { diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt index 58706ea4979..2c4f0c74f12 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt @@ -30,6 +30,7 @@ public val COMPILER_SERVICE_RMI_NAME: String = "KotlinJvmCompilerService" public val COMPILER_DAEMON_CLASS_FQN: String = "org.jetbrains.kotlin.rmi.service.CompileDaemon" public val COMPILE_DAEMON_DEFAULT_PORT: Int = 17031 public val COMPILE_DAEMON_ENABLED_PROPERTY: String ="kotlin.daemon.enabled" +public val COMPILE_DAEMON_JVM_OPTIONS_PROPERTY: String ="kotlin.daemon.jvm.options" fun> C.propToParams(p: P, conv: ((v: V) -> String) = { it.toString() } ) = diff --git a/idea/idea.iml b/idea/idea.iml index f0de3f1fc81..725e973dd63 100644 --- a/idea/idea.iml +++ b/idea/idea.iml @@ -50,5 +50,6 @@ + \ No newline at end of file diff --git a/idea/src/org/jetbrains/kotlin/idea/PluginStartupComponent.java b/idea/src/org/jetbrains/kotlin/idea/PluginStartupComponent.java index d9fbc93f52f..cfea2bc5d71 100644 --- a/idea/src/org/jetbrains/kotlin/idea/PluginStartupComponent.java +++ b/idea/src/org/jetbrains/kotlin/idea/PluginStartupComponent.java @@ -23,6 +23,7 @@ import org.jetbrains.kotlin.idea.caches.JarUserDataManager; import org.jetbrains.kotlin.idea.debugger.filter.FilterPackage; import org.jetbrains.kotlin.idea.decompiler.HasCompiledKotlinInJar; import org.jetbrains.kotlin.idea.framework.KotlinJavaScriptLibraryDetectionUtil; +import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient; import org.jetbrains.kotlin.utils.PathUtil; public class PluginStartupComponent implements ApplicationComponent { @@ -51,5 +52,6 @@ public class PluginStartupComponent implements ApplicationComponent { @Override public void disposeComponent() { + KotlinCompilerClient.Companion.shutdownCompileService(); } } diff --git a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java index 4921e35768b..25b917c6b6e 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java +++ b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java @@ -34,6 +34,7 @@ import org.jetbrains.kotlin.config.CompilerSettings; import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache; import org.jetbrains.kotlin.rmi.CompileService; import org.jetbrains.kotlin.rmi.CompilerId; +import org.jetbrains.kotlin.rmi.DaemonLaunchingOptions; import org.jetbrains.kotlin.rmi.DaemonOptions; import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient; import org.jetbrains.kotlin.utils.UtilsPackage; @@ -136,8 +137,10 @@ public class KotlinCompilerRunner { // the lifetime of JPS process is small anyway, we can neglect the probability of changed compiler CompilerId compilerId = CompilerId.makeCompilerId(new File(libPath, "kotlin-compiler.jar")); DaemonOptions daemonOptions = new DaemonOptions(); + DaemonLaunchingOptions daemonLaunchingOptions = new DaemonLaunchingOptions(); + KotlinCompilerClient.Companion.configureDaemonLaunchingOptions(daemonLaunchingOptions); // TODO: find proper stream to report daemon connection progress - CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonOptions, System.out, true, true); + CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, System.out, true, true); if (daemon != null) { Integer res = KotlinCompilerClient.Companion.incrementalCompile(daemon, argsArray, incrementalCaches, out); return res.toString(); From 434c30c1bcd28674a8ce556481aa411ffe76bd14 Mon Sep 17 00:00:00 2001 From: ligee Date: Thu, 20 Aug 2015 15:02:41 +0200 Subject: [PATCH 10/14] Simple test with daemon compilation, minor fixes mostly for easier testing --- .../rmi/kotlinr/src/KotlinCompilerClient.kt | 24 ++++-- .../jetbrains/kotlin/rmi/CompileService.kt | 2 +- .../kotlin/rmi/service/CompileServiceImpl.kt | 4 +- compiler/tests/compiler-tests.iml | 2 + .../kotlin/daemon/CompilerDaemonTest.kt | 76 +++++++++++++++++++ .../KotlinIntegrationTestBase.java | 2 +- 6 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt diff --git a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt index ab5324dda2b..7808fbdf82d 100644 --- a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt +++ b/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt @@ -157,19 +157,33 @@ public class KotlinCompilerClient { return connectToService(compilerId, daemonOptions, errStream) } - public fun shutdownCompileService(): Unit { - KotlinCompilerClient.connectToCompileService(CompilerId(), DaemonLaunchingOptions(), DaemonOptions(), System.out, autostart = false, checkId = false) + public fun shutdownCompileService(daemonOptions: DaemonOptions): Unit { + KotlinCompilerClient.connectToCompileService(CompilerId(), DaemonLaunchingOptions(), daemonOptions, System.out, autostart = false, checkId = false) ?.shutdown() } - public fun incrementalCompile(compiler: CompileService, args: Array, caches: Map, out: OutputStream): Int { + public fun shutdownCompileService(): Unit { + shutdownCompileService(DaemonOptions()) + } + + public fun compile(compiler: CompileService, args: Array, out: OutputStream): Int { + + val outStrm = RemoteOutputStreamServer(out) + try { + return compiler.remoteCompile(args, outStrm, CompileService.OutputFormat.PLAIN) + } + finally { + outStrm.disconnect() + } + } + + public fun incrementalCompile(compiler: CompileService, args: Array, caches: Map, out: OutputStream): Int { val outStrm = RemoteOutputStreamServer(out) val cacheServers = hashMapOf() try { caches.forEach { cacheServers.put( it.getKey(), RemoteIncrementalCacheServer( it.getValue())) } - val res = compiler.remoteIncrementalCompile(args, cacheServers, outStrm, CompileService.OutputFormat.XML) - return res + return compiler.remoteIncrementalCompile(args, cacheServers, outStrm, CompileService.OutputFormat.XML) } finally { cacheServers.forEach { it.getValue().disconnect() } diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt index 890c4eae466..5d5cd9c7530 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt @@ -51,7 +51,7 @@ public interface CompileService : Remote { throws(RemoteException::class) public fun remoteIncrementalCompile( - args: Array, + args: Array, caches: Map, outputStream: RemoteOutputStream, outputFormat: OutputFormat): Int diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt index f5bb4fede4c..2719bc17e7b 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt @@ -126,7 +126,7 @@ class CompileServiceImpl>( val res = body() val endTime = System.nanoTime() val endMem = usedMemory() / 1024 - log.info("Done") + log.info("Done with result " + res.toString()) log.info("Elapsed time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") log.info("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)") return res @@ -178,7 +178,7 @@ class CompileServiceImpl>( } } - override fun remoteIncrementalCompile(args: Array, caches: Map, errStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int = + override fun remoteIncrementalCompile(args: Array, caches: Map, errStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int = ifAlive { checkedCompile(args) { val strm = RemoteOutputStreamClient(errStream) diff --git a/compiler/tests/compiler-tests.iml b/compiler/tests/compiler-tests.iml index fb072c4387d..5448f557960 100644 --- a/compiler/tests/compiler-tests.iml +++ b/compiler/tests/compiler-tests.iml @@ -26,5 +26,7 @@ + + \ No newline at end of file diff --git a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt new file mode 100644 index 00000000000..dd5842f9e5e --- /dev/null +++ b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt @@ -0,0 +1,76 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.daemon + +import junit.framework.TestCase +import org.jetbrains.kotlin.cli.CliBaseTest +import org.jetbrains.kotlin.integration.KotlinIntegrationTestBase +import org.jetbrains.kotlin.rmi.CompilerId +import org.jetbrains.kotlin.rmi.DaemonLaunchingOptions +import org.jetbrains.kotlin.rmi.DaemonOptions +import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient +import org.jetbrains.kotlin.test.JetTestUtils +import java.io.ByteArrayOutputStream +import java.io.File + +val KOTLIN_DAEMON_TEST_PORT = 19753 + +public class CompilerDaemonTest : KotlinIntegrationTestBase() { + + data class CompilerResults(val resultCode: Int, val out: String) + + val daemonOptions = DaemonOptions(port = KOTLIN_DAEMON_TEST_PORT) + val daemonLaunchingOptions = DaemonLaunchingOptions() + val compilerId by lazy { CompilerId.makeCompilerId( File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-compiler.jar"), + File("dependencies/bootstrap-compiler/Kotlin/kotlinc/lib/kotlin-runtime.jar"), + File("dependencies/bootstrap-compiler/Kotlin/kotlinc/lib/kotlin-reflect.jar")) } + + private fun compileOnDaemon(args: Array): CompilerResults { + val daemon = KotlinCompilerClient.connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, System.err, autostart = true, checkId = true) + TestCase.assertNotNull("failed to connect daemon", daemon) + val strm = ByteArrayOutputStream() + val code = KotlinCompilerClient.compile(daemon!!, args, strm) + return CompilerResults(code, strm.toString()) + } + + private fun runDaemonCompilerTwice(logName: String, vararg arguments: String): Unit { + KotlinCompilerClient.shutdownCompileService(daemonOptions) + + try { + val res1 = compileOnDaemon(arguments) + TestCase.assertEquals("first compilation failed:\n${res1.out}", 0, res1.resultCode) + val res2 = compileOnDaemon(arguments) + TestCase.assertEquals("second compilation failed:\n${res2.out}", 0, res2.resultCode) + TestCase.assertEquals("build results differ", CliBaseTest.removePerfOutput(res1.out), CliBaseTest.removePerfOutput(res2.out)) + // TODO: add performance comparison assert + } + finally { + KotlinCompilerClient.shutdownCompileService(daemonOptions) + } + } + + private fun getTestBaseDir(): String = JetTestUtils.getTestDataPathBase() + "/integration/smoke/" + getTestName(true) + + private fun run(logName: String, vararg args: String): Int = runJava(getTestBaseDir(), logName, *args) + + public fun testHelloApp() { + val jar = tmpdir.absolutePath + File.separator + "hello.jar" + + runDaemonCompilerTwice("hello.compile", "-include-runtime", File(getTestBaseDir(), "hello.kt").absolutePath, "-d", jar) + run("hello.run", "-cp", jar, "Hello.HelloPackage") + } +} diff --git a/compiler/tests/org/jetbrains/kotlin/integration/KotlinIntegrationTestBase.java b/compiler/tests/org/jetbrains/kotlin/integration/KotlinIntegrationTestBase.java index 4766c779fa4..2b81e49fc34 100644 --- a/compiler/tests/org/jetbrains/kotlin/integration/KotlinIntegrationTestBase.java +++ b/compiler/tests/org/jetbrains/kotlin/integration/KotlinIntegrationTestBase.java @@ -133,7 +133,7 @@ public abstract class KotlinIntegrationTestBase extends TestCaseWithTmpdir { return runtime; } - protected static File getCompilerLib() { + public static File getCompilerLib() { File file = PathUtil.getKotlinPathsForDistDirectory().getLibPath().getAbsoluteFile(); assertTrue("Lib directory doesn't exist. Run 'ant dist'", file.isDirectory()); return file; From 61de1c3212715e67b3c5314edabf6352ec30f78f Mon Sep 17 00:00:00 2001 From: Ilya Chernikov Date: Thu, 20 Aug 2015 17:36:26 +0200 Subject: [PATCH 11/14] Fixes after review --- .../rmi/kotlinr}/KotlinCompilerClient.kt | 86 ++++++++++--------- .../kotlinr}/RemoteIncrementalCacheServer.kt | 0 .../rmi/kotlinr}/RemoteOutputStreamServer.kt | 4 +- .../org/jetbrains/kotlin/rmi/DaemonParams.kt | 9 +- .../kotlin/rmi/service/CompileDaemon.kt | 5 +- .../kotlin/rmi/service/CompileServiceImpl.kt | 11 +++ .../kotlin/rmi/service/DaemonPermissions.kt | 20 +---- .../compilerRunner/KotlinCompilerRunner.java | 7 +- 8 files changed, 72 insertions(+), 70 deletions(-) rename compiler/rmi/kotlinr/src/{ => org/jetbrains/kotlin/rmi/kotlinr}/KotlinCompilerClient.kt (80%) rename compiler/rmi/kotlinr/src/{ => org/jetbrains/kotlin/rmi/kotlinr}/RemoteIncrementalCacheServer.kt (100%) rename compiler/rmi/kotlinr/src/{ => org/jetbrains/kotlin/rmi/kotlinr}/RemoteOutputStreamServer.kt (91%) diff --git a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt similarity index 80% rename from compiler/rmi/kotlinr/src/KotlinCompilerClient.kt rename to compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt index 7808fbdf82d..87fbae84c5f 100644 --- a/compiler/rmi/kotlinr/src/KotlinCompilerClient.kt +++ b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt @@ -24,6 +24,7 @@ import java.io.PrintStream import java.rmi.ConnectException import java.rmi.Remote import java.rmi.registry.LocateRegistry +import java.util.concurrent.Semaphore import java.util.concurrent.TimeUnit import java.util.concurrent.locks.ReentrantReadWriteLock import kotlin.concurrent.read @@ -45,7 +46,6 @@ public class KotlinCompilerClient { companion object { val DAEMON_STARTUP_TIMEOUT_MS = 10000L - val DAEMON_STARTUP_CHECK_INTERVAL_MS = 100L private fun connectToService(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream): CompileService? { @@ -71,9 +71,13 @@ public class KotlinCompilerClient { private fun startDaemon(compilerId: CompilerId, daemonLaunchingOptions: DaemonLaunchingOptions, daemonOptions: DaemonOptions, errStream: PrintStream) { - val javaExecutable = listOf(System.getProperty("java.home"), "bin", "java").joinToString(File.separator) + val javaExecutable = File(System.getProperty("java.home"), "bin").let { + val javaw = File(it, "javaw.exe") + if (javaw.exists()) javaw + else File(it, "java") + } // TODO: add some specific environment variables to the cp and may be command line, to allow some specific startup configs - val args = listOf(javaExecutable, + val args = listOf(javaExecutable.absolutePath, "-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator)) + daemonLaunchingOptions.jvmParams + COMPILER_DAEMON_CLASS_FQN + @@ -84,16 +88,18 @@ public class KotlinCompilerClient { // assuming daemon process is deaf and (mostly) silent, so do not handle streams val daemon = processBuilder.start() - val lock = ReentrantReadWriteLock() - var isEchoRead = false + var isEchoRead = Semaphore(1) + isEchoRead.acquire() - val stdouThread = + val stdoutThread = thread { daemon.getInputStream() .reader() .forEachLine { - if (daemonOptions.startEcho.isNotEmpty() && it.contains(daemonOptions.startEcho)) - lock.write { isEchoRead = true; return@forEachLine } + if (daemonOptions.startEcho.isNotEmpty() && it.contains(daemonOptions.startEcho)) { + isEchoRead.release() + return@forEachLine + } errStream.println("[daemon] " + it) } } @@ -101,14 +107,10 @@ public class KotlinCompilerClient { // trying to wait for process if (daemonOptions.startEcho.isNotEmpty()) { errStream.println("[daemon client] waiting for daemon to respond") - var waitMillis: Long = DAEMON_STARTUP_TIMEOUT_MS / DAEMON_STARTUP_CHECK_INTERVAL_MS - while (waitMillis-- > 0) { - Thread.sleep(DAEMON_STARTUP_CHECK_INTERVAL_MS) - if (!daemon.isAlive() || lock.read { isEchoRead } == true) break; - } + val succeeded = isEchoRead.tryAcquire(DAEMON_STARTUP_TIMEOUT_MS, TimeUnit.MILLISECONDS) if (!daemon.isAlive()) throw Exception("Daemon terminated unexpectedly") - if (lock.read { isEchoRead } == false) + if (!succeeded) throw Exception("Unable to get response from daemon in $DAEMON_STARTUP_TIMEOUT_MS ms") } else @@ -117,9 +119,9 @@ public class KotlinCompilerClient { } finally { // assuming that all important output is already done, the rest should be routed to the log by the daemon itself - if (stdouThread.isAlive) + if (stdoutThread.isAlive) // TODO: find better method to stop the thread, but seems it will require asynchronous consuming of the stream - lock.write { stdouThread.stop() } + stdoutThread.stop() } } @@ -182,7 +184,7 @@ public class KotlinCompilerClient { val outStrm = RemoteOutputStreamServer(out) val cacheServers = hashMapOf() try { - caches.forEach { cacheServers.put( it.getKey(), RemoteIncrementalCacheServer( it.getValue())) } + caches.mapValuesTo(cacheServers, { RemoteIncrementalCacheServer( it.getValue()) }) return compiler.remoteIncrementalCompile(args, cacheServers, outStrm, CompileService.OutputFormat.XML) } finally { @@ -242,34 +244,36 @@ public class KotlinCompilerClient { compilerId.updateDigest() } - connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, System.out, autostart = !clientOptions.stop, checkId = !clientOptions.stop)?.let { - when { - clientOptions.stop -> { - println("Shutdown the daemon") - it.shutdown() - println("Daemon shut down successfully") + val daemon = connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, System.out, autostart = !clientOptions.stop, checkId = !clientOptions.stop) + + if (daemon == null) { + if (clientOptions.stop) println("No daemon found to shut down") + else throw Exception("Unable to connect to daemon") + } + else when { + clientOptions.stop -> { + println("Shutdown the daemon") + daemon.shutdown() + println("Daemon shut down successfully") + } + else -> { + println("Executing daemon compilation with args: " + filteredArgs.joinToString(" ")) + val outStrm = RemoteOutputStreamServer(System.out) + try { + val memBefore = daemon.getUsedMemory() / 1024 + val startTime = System.nanoTime() + val res = daemon.remoteCompile(filteredArgs.toArrayList().toTypedArray(), outStrm, CompileService.OutputFormat.PLAIN) + val endTime = System.nanoTime() + println("Compilation result code: $res") + val memAfter = daemon.getUsedMemory() / 1024 + println("Compilation time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") + println("Used memory $memAfter (${"%+d".format(memAfter - memBefore)} kb)") } - else -> { - println("Executing daemon compilation with args: " + filteredArgs.joinToString(" ")) - val outStrm = RemoteOutputStreamServer(System.out) - try { - val memBefore = it.getUsedMemory() / 1024 - val startTime = System.nanoTime() - val res = it.remoteCompile(filteredArgs.toArrayList().toTypedArray(), outStrm, CompileService.OutputFormat.PLAIN) - val endTime = System.nanoTime() - println("Compilation result code: $res") - val memAfter = it.getUsedMemory() / 1024 - println("Compilation time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") - println("Used memory $memAfter (${"%+d".format(memAfter - memBefore)} kb)") - } - finally { - outStrm.disconnect() - } + finally { + outStrm.disconnect() } } } - ?: if (clientOptions.stop) println("No daemon found to shut down") - else throw Exception("Unable to connect to daemon") } } } diff --git a/compiler/rmi/kotlinr/src/RemoteIncrementalCacheServer.kt b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteIncrementalCacheServer.kt similarity index 100% rename from compiler/rmi/kotlinr/src/RemoteIncrementalCacheServer.kt rename to compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteIncrementalCacheServer.kt diff --git a/compiler/rmi/kotlinr/src/RemoteOutputStreamServer.kt b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteOutputStreamServer.kt similarity index 91% rename from compiler/rmi/kotlinr/src/RemoteOutputStreamServer.kt rename to compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteOutputStreamServer.kt index 7a96d0dc927..4b67c389486 100644 --- a/compiler/rmi/kotlinr/src/RemoteOutputStreamServer.kt +++ b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/RemoteOutputStreamServer.kt @@ -35,8 +35,8 @@ class RemoteOutputStreamServer(val out: OutputStream) : RemoteOutputStream { out.close() } - override fun write(data: ByteArray, start: Int, length: Int) { - out.write(data, start, length) + override fun write(data: ByteArray, offset: Int, length: Int) { + out.write(data, offset, length) } override fun write(dataByte: Int) { diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt index 2c4f0c74f12..87a48243b63 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt @@ -117,10 +117,11 @@ val COMPILER_ID_DIGEST = "MD5" fun updateSingleFileDigest(file: File, md: MessageDigest) { - val stream = DigestInputStream(file.inputStream(), md) - val buf = ByteArray(1024) - while (stream.read(buf) == buf.size()) {} - stream.close() + DigestInputStream(file.inputStream(), md).use { + val buf = ByteArray(1024) + while (it.read(buf) != -1) { } + it.close() + } } fun updateForAllClasses(dir: File, md: MessageDigest) { diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt index 54b7927c34a..184699b1df2 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt @@ -57,10 +57,7 @@ class LogStream(name: String) : OutputStream() { } fun flush(data: ByteArray, offset: Int, length: Int) { - if (offset == 0 && length == data.size()) - log.info(lineBuf.toString() + data.toString()) - else - log.info(lineBuf.toString() + data.toString().substring(offset, length)) + log.info(lineBuf.toString() + data.toString().substring(offset, length)) lineBuf.setLength(0) } diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt index 2719bc17e7b..fd2b51fbaf2 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt @@ -29,6 +29,7 @@ import java.io.File import java.io.FileNotFoundException import java.io.IOException import java.io.PrintStream +import java.lang.management.ManagementFactory import java.net.URLClassLoader import java.rmi.registry.Registry import java.rmi.server.UnicastRemoteObject @@ -103,6 +104,13 @@ class CompileServiceImpl>( return (rt.totalMemory() - rt.freeMemory()) } + fun usedMemoryMX(): Long { + System.gc() + val memoryMXBean= ManagementFactory.getMemoryMXBean() + val memHeap=memoryMXBean.getHeapMemoryUsage() + return memHeap.used + } + private fun loadKotlinVersionFromResource(): String { (javaClass.classLoader as? URLClassLoader) ?.findResource("META-INF/MANIFEST.MF") @@ -121,14 +129,17 @@ class CompileServiceImpl>( if (args.none()) throw IllegalArgumentException("Error: empty arguments list.") log.info("Starting compilation with args: " + args.joinToString(" ")) + val startMemMX = usedMemoryMX() / 1024 val startMem = usedMemory() / 1024 val startTime = System.nanoTime() val res = body() val endTime = System.nanoTime() val endMem = usedMemory() / 1024 + val endMemMX = usedMemoryMX() / 1024 log.info("Done with result " + res.toString()) log.info("Elapsed time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") log.info("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)") + log.info("Used memory (from MemoryMXBean): $endMemMX kb (${"%+d".format(endMemMX - startMemMX)} kb)") return res } catch (e: Exception) { diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/DaemonPermissions.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/DaemonPermissions.kt index 51d88d97837..909e1a6dd1f 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/DaemonPermissions.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/DaemonPermissions.kt @@ -53,29 +53,17 @@ public class DaemonPolicy(val port: Int) : Policy() { class DaemonPermissionCollection : PermissionCollection() { - var perms = ArrayList() + val perms = ArrayList() override fun add(p: Permission) { perms.add(p) } - override fun implies(p: Permission): Boolean { - val i = perms.iterator() - while (i.hasNext()) { - if (i.next().implies(p)) { - return true - } - } - return false - } + override fun implies(p: Permission): Boolean = perms.any { implies(p) } - override fun elements(): Enumeration { - return Collections.enumeration(perms) - } + override fun elements(): Enumeration = Collections.enumeration(perms) - override fun isReadOnly(): Boolean { - return false - } + override fun isReadOnly(): Boolean = false // // companion object { // diff --git a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java index 25b917c6b6e..658343708d3 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java +++ b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java @@ -62,7 +62,8 @@ public class KotlinCompilerRunner { CompilerSettings compilerSettings, MessageCollector messageCollector, CompilerEnvironment environment, - Map incrementalCaches, File moduleFile, + Map incrementalCaches, + File moduleFile, OutputItemsCollector collector ) { K2JVMCompilerArguments arguments = mergeBeans(commonArguments, k2jvmArguments); @@ -78,7 +79,8 @@ public class KotlinCompilerRunner { @NotNull CompilerSettings compilerSettings, @NotNull MessageCollector messageCollector, @NotNull CompilerEnvironment environment, - Map incrementalCaches, @NotNull OutputItemsCollector collector, + Map incrementalCaches, + @NotNull OutputItemsCollector collector, @NotNull Collection sourceFiles, @NotNull List libraryFiles, @NotNull File outputFile @@ -156,7 +158,6 @@ public class KotlinCompilerRunner { // exec() returns an ExitCode object, class of which is loaded with a different class loader, // so we take it's contents through reflection return getReturnCodeFromObject(rc); - } catch (Throwable e) { MessageCollectorUtil.reportException(messageCollector, e); From 2d45a37884404f072a5b3aee29fce73596d48abc Mon Sep 17 00:00:00 2001 From: Ilya Chernikov Date: Fri, 21 Aug 2015 19:35:02 +0200 Subject: [PATCH 12/14] Passing JPS process JVM memory options to daemon, refactoring options processing --- .../rmi/kotlinr/KotlinCompilerClient.kt | 25 +-- .../org/jetbrains/kotlin/rmi/DaemonParams.kt | 148 ++++++++++++++---- .../kotlin/rmi/service/CompileDaemon.kt | 7 +- .../compilerRunner/KotlinCompilerRunner.java | 12 +- 4 files changed, 134 insertions(+), 58 deletions(-) diff --git a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt index 87fbae84c5f..d6b863930e3 100644 --- a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt +++ b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt @@ -31,6 +31,7 @@ import kotlin.concurrent.read import kotlin.concurrent.thread import kotlin.concurrent.write import kotlin.platform.platformStatic +import kotlin.reflect.KProperty1 fun Process.isAlive() = try { @@ -79,10 +80,10 @@ public class KotlinCompilerClient { // TODO: add some specific environment variables to the cp and may be command line, to allow some specific startup configs val args = listOf(javaExecutable.absolutePath, "-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator)) + - daemonLaunchingOptions.jvmParams + + daemonLaunchingOptions.extractors.flatMap { it.extract("-") } + COMPILER_DAEMON_CLASS_FQN + - daemonOptions.asParams + - compilerId.asParams + daemonOptions.extractors.flatMap { it.extract(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } + + compilerId.extractors.flatMap { it.extract(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } errStream.println("[daemon client] starting the daemon as: " + args.joinToString(" ")) val processBuilder = ProcessBuilder(args).redirectErrorStream(true) // assuming daemon process is deaf and (mostly) silent, so do not handle streams @@ -141,7 +142,7 @@ public class KotlinCompilerClient { errStream.println("[daemon client] found the suitable daemon") return service } - errStream.println("[daemon client] compiler identity don't match: " + compilerId.asParams.joinToString(" ")) + errStream.println("[daemon client] compiler identity don't match: " + compilerId.extractors.flatMap { it.extract("") }.joinToString(" ")) if (!autostart) return null; errStream.println("[daemon client] shutdown the daemon") service.shutdown() @@ -193,21 +194,11 @@ public class KotlinCompilerClient { } } - public fun isDaemonEnabled(): Boolean = System.getProperty(COMPILE_DAEMON_ENABLED_PROPERTY) != null - - public fun configureDaemonLaunchingOptions(opts: DaemonLaunchingOptions) { - System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)?.let { - // TODO: find better way to pass and parse jvm options for daemon - opts.jvmParams = it.split("##") - } - } - data class ClientOptions( public var stop: Boolean = false ) :CmdlineParams { - override val asParams: Iterable - get() = - if (stop) listOf("stop") else listOf() + override val extractors: List> + get() = listOf( BoolPropExtractor(this, ::stop)) override val parsers: List> get() = listOf( BoolPropParser(this, ::stop)) @@ -218,7 +209,7 @@ public class KotlinCompilerClient { val daemonOptions = DaemonOptions() val daemonLaunchingOptions = DaemonLaunchingOptions() val clientOptions = ClientOptions() - val filteredArgs = args.asIterable().propParseFilter(compilerId, daemonOptions, daemonLaunchingOptions, clientOptions) + val filteredArgs = args.asIterable().filterSetProps(compilerId, daemonOptions, daemonLaunchingOptions, clientOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) if (!clientOptions.stop) { if (compilerId.compilerClasspath.none()) { diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt index 87a48243b63..80c6302de7d 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt @@ -18,6 +18,7 @@ package org.jetbrains.kotlin.rmi import java.io.File import java.io.Serializable +import java.lang.management.ManagementFactory import java.security.DigestInputStream import java.security.MessageDigest import kotlin.platform.platformStatic @@ -31,29 +32,78 @@ public val COMPILER_DAEMON_CLASS_FQN: String = "org.jetbrains.kotlin.rmi.service public val COMPILE_DAEMON_DEFAULT_PORT: Int = 17031 public val COMPILE_DAEMON_ENABLED_PROPERTY: String ="kotlin.daemon.enabled" public val COMPILE_DAEMON_JVM_OPTIONS_PROPERTY: String ="kotlin.daemon.jvm.options" +public val COMPILE_DAEMON_OPTIONS_PROPERTY: String ="kotlin.daemon.options" +public val COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX: String ="--daemon-" -fun> C.propToParams(p: P, conv: ((v: V) -> String) = { it.toString() } ) = - listOf("--daemon-" + p.name, conv(p.get(this))) +open class PropExtractor>(val dest: C, + val prop: P, + val name: String, + val convert: ((v: V) -> String?) = { it.toString() }, + val skipIf: ((v: V) -> Boolean) = { false }, + val mergeWithDelimiter: String? = null) +{ + constructor(dest: C, prop: P, convert: ((v: V) -> String?) = { it.toString() }, skipIf: ((v: V) -> Boolean) = { false }) : this(dest, prop, prop.name, convert, skipIf) + open fun extract(prefix: String = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX): List = + when { + skipIf(prop.get(dest)) -> listOf() + mergeWithDelimiter != null -> listOf(prefix + name + mergeWithDelimiter + convert(prop.get(dest))).filterNotNull() + else -> listOf(prefix + name, convert(prop.get(dest))).filterNotNull() + } +} -open class PropParser>(val dest: C, val prop: P, val parse: (s: String) -> V) { +class BoolPropExtractor>(dest: C, prop: P, name: String? = null) + : PropExtractor(dest, prop, name ?: prop.name, convert = { null }, skipIf = { !prop.get(dest) }) + +class RestPropExtractor>>(dest: C, prop: P) : PropExtractor, P>(dest, prop, convert = { null }) { + override fun extract(prefix: String): List = prop.get(dest).map { prefix + it } +} + + +open class PropParser>(val dest: C, + val prop: P, alternativeNames: List, + val parse: (s: String) -> V, + val allowMergedArg: Boolean = false) { + val names = listOf(prop.name) + alternativeNames + constructor(dest: C, prop: P, parse: (s: String) -> V, allowMergedArg: Boolean = false) : this(dest, prop, listOf(), parse, allowMergedArg) fun apply(s: String) = prop.set(dest, parse(s)) } class BoolPropParser>(dest: C, prop: P): PropParser(dest, prop, { true }) -fun Iterable.propParseFilter(parsers: List>) : Iterable { +class RestPropParser>>(dest: C, prop: P): PropParser, P>(dest, prop, { arrayListOf() }) { + fun add(s: String) { prop.get(dest).add(s) } +} + + +fun Iterable.filterSetProps(parsers: List>, prefix: String, restParser: RestPropParser<*,*>? = null) : Iterable { var currentParser: PropParser<*,*,*>? = null - return filter { param -> + var matchingOption = "" + val res = filter { param -> if (currentParser == null) { - currentParser = parsers.find { param.equals("--daemon-" + it.prop.name) } - if (currentParser != null) { - if (currentParser is BoolPropParser<*,*>) { - currentParser!!.apply("") - currentParser = null + val parser = parsers.find { it.names.any { name -> + if (param.startsWith(prefix + name)) { matchingOption = prefix + name; true } + else false } } + if (parser != null) { + val optionLength = matchingOption.length() + when { + parser is BoolPropParser<*,*> -> + if (param.length() > optionLength) throw IllegalArgumentException("Invalid switch option '$param', expecting $matchingOption without arguments") + else parser.apply("") + param.length() > optionLength -> + if (param[optionLength] != '=') { + if (parser.allowMergedArg) parser.apply(param.substring(optionLength)) + else throw IllegalArgumentException("Invalid option syntax '$param', expecting $matchingOption[= ]") + } + else parser.apply(param.substring(optionLength + 1)) + else -> currentParser = parser } false } + else if (restParser != null && param.startsWith(prefix)) { + restParser.add(param.removePrefix(prefix)) + false + } else true } else { @@ -62,6 +112,8 @@ fun Iterable.propParseFilter(parsers: List>) : Iterabl false } } + if (currentParser != null) throw IllegalArgumentException("Expecting argument for the option $matchingOption") + return res } // TODO: find out how to create more generic variant using first constructor @@ -70,25 +122,35 @@ fun Iterable.propParseFilter(parsers: List>) : Iterabl // kc.constructors.first(). //} + + public interface CmdlineParams : Serializable { - public val asParams: Iterable + public val extractors: List> public val parsers: List> } -public fun Iterable.propParseFilter(vararg cs: CmdlineParams) : Iterable = - propParseFilter(cs.flatMap { it.parsers }) +public fun Iterable.filterSetProps(vararg cs: CmdlineParams, prefix: String) : Iterable = + filterSetProps(cs.flatMap { it.parsers }, prefix) public data class DaemonLaunchingOptions( - public var jvmParams: List = listOf() + public var maxMemory: String = "", + public var maxPermSize: String = "", + public var reservedCodeCacheSize: String = "", + public var otherJvmParams: MutableCollection = arrayListOf() ) : CmdlineParams { - override val asParams: Iterable - get() = - propToParams(::jvmParams, { it.joinToString("##") }) // TODO: consider some other options rather than using potentially dangerous delimiter + override val extractors: List> + get() = listOf( PropExtractor(this, ::maxMemory, "Xmx", skipIf = { it.isEmpty() }, mergeWithDelimiter = ""), + PropExtractor(this, ::maxPermSize, "XX:MaxPermSize", skipIf = { it.isEmpty() }, mergeWithDelimiter = "="), + PropExtractor(this, ::reservedCodeCacheSize, "XX:ReservedCodeCacheSize", skipIf = { it.isEmpty() }, mergeWithDelimiter = "="), + RestPropExtractor(this, ::otherJvmParams)) override val parsers: List> - get() = listOf( PropParser(this, ::jvmParams, { it.split("##")})) // TODO: see appropriate comment in asParams + get() = listOf( PropParser(this, ::maxMemory, listOf("Xmx"), { it }, allowMergedArg = true), + PropParser(this, ::maxPermSize, listOf("XX:MaxPermSize"), { it }, allowMergedArg = true), + PropParser(this, ::reservedCodeCacheSize, listOf("XX:ReservedCodeCacheSize"), { it }, allowMergedArg = true)) + // otherJvmParams is missing here deliberately, it is used explicitly as a restParser param to filterSetProps } public data class DaemonOptions( @@ -98,12 +160,11 @@ public data class DaemonOptions( public var startEcho: String = COMPILER_SERVICE_RMI_NAME ) : CmdlineParams { - override val asParams: Iterable - get() = - propToParams(::port) + - propToParams(::autoshutdownMemoryThreshold) + - propToParams(::autoshutdownIdleSeconds) + - propToParams(::startEcho) + override val extractors: List> + get() = listOf( PropExtractor(this, ::port), + PropExtractor(this, ::autoshutdownMemoryThreshold, skipIf = { it == 0L }), + PropExtractor(this, ::autoshutdownIdleSeconds, skipIf = { it == 0 }), + PropExtractor(this, ::startEcho)) override val parsers: List> get() = listOf( PropParser(this, ::port, { it.toInt()}), @@ -156,11 +217,10 @@ public data class CompilerId( // TODO: checksum ) : CmdlineParams { - override val asParams: Iterable - get() = - propToParams(::compilerClasspath, { it.joinToString(File.pathSeparator) }) + - propToParams(::compilerDigest) + - propToParams(::compilerVersion) + override val extractors: List> + get() = listOf( PropExtractor(this, ::compilerClasspath, convert = { it.joinToString(File.pathSeparator) }), + PropExtractor(this, ::compilerDigest), + PropExtractor(this, ::compilerVersion, skipIf = { it.isEmpty() })) override val parsers: List> get() = @@ -176,9 +236,37 @@ public data class CompilerId( public platformStatic fun makeCompilerId(vararg paths: File): CompilerId = makeCompilerId(paths.asIterable()) public platformStatic fun makeCompilerId(paths: Iterable): CompilerId = - // TODO consider reading version and calculating checksum here + // TODO consider reading version here CompilerId(compilerClasspath = paths.map { it.absolutePath }, compilerDigest = paths.getFilesClasspathDigest()) } } +public fun isDaemonEnabled(): Boolean = System.getProperty(COMPILE_DAEMON_ENABLED_PROPERTY) != null + + +public fun configureDaemonLaunchingOptions(opts: DaemonLaunchingOptions, inheritMemoryLimits: Boolean): DaemonLaunchingOptions { + // note: sequence matters, explicit override in COMPILE_DAEMON_JVM_OPTIONS_PROPERTY should be done after inputArguments processing + if (inheritMemoryLimits) + ManagementFactory.getRuntimeMXBean().inputArguments.filterSetProps(opts.parsers, "-") + + System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)?.let { + opts.otherJvmParams.addAll( it.trim('"', '\'').split(",").filterSetProps(opts.parsers, "-", RestPropParser(opts, DaemonLaunchingOptions::otherJvmParams))) + } + return opts +} + +public fun configureDaemonLaunchingOptions(inheritMemoryLimits: Boolean): DaemonLaunchingOptions = + configureDaemonLaunchingOptions(DaemonLaunchingOptions(), inheritMemoryLimits = inheritMemoryLimits) + +jvmOverloads public fun configureDaemonOptions(opts: DaemonOptions = DaemonOptions()): DaemonOptions { + System.getProperty(COMPILE_DAEMON_OPTIONS_PROPERTY)?.let { + val unrecognized = it.trim('"', '\'').split(",").filterSetProps(opts.parsers, "") + if (unrecognized.any()) + throw IllegalArgumentException( + "Unrecognized daemon options passed via property $COMPILE_DAEMON_OPTIONS_PROPERTY: " + unrecognized.joinToString(" ") + + "\nSupported options: " + opts.extractors.joinToString(", ", transform = { it.name })) + } + return opts +} + diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt index 184699b1df2..3c274d52f72 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt @@ -17,9 +17,10 @@ package org.jetbrains.kotlin.rmi.service import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler +import org.jetbrains.kotlin.rmi.COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX import org.jetbrains.kotlin.rmi.CompilerId import org.jetbrains.kotlin.rmi.DaemonOptions -import org.jetbrains.kotlin.rmi.propParseFilter +import org.jetbrains.kotlin.rmi.filterSetProps import org.jetbrains.kotlin.service.CompileServiceImpl import java.io.OutputStream import java.io.PrintStream @@ -95,7 +96,7 @@ public class CompileDaemon { val compilerId = CompilerId() val daemonOptions = DaemonOptions() - val filteredArgs = args.asIterable().propParseFilter(compilerId, daemonOptions) + val filteredArgs = args.asIterable().filterSetProps(compilerId, daemonOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) if (filteredArgs.any()) { val helpLine = "usage: " @@ -115,7 +116,7 @@ public class CompileDaemon { val registry = LocateRegistry.createRegistry(daemonOptions.port); val compiler = K2JVMCompiler() - val server = CompileServiceImpl(registry, compiler, compilerId, daemonOptions) + CompileServiceImpl(registry, compiler, compilerId, daemonOptions) if (daemonOptions.startEcho.isNotEmpty()) println(daemonOptions.startEcho) diff --git a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java index 658343708d3..c5460c78d48 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java +++ b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java @@ -32,10 +32,7 @@ import org.jetbrains.kotlin.cli.common.messages.MessageCollector; import org.jetbrains.kotlin.cli.common.messages.MessageCollectorUtil; import org.jetbrains.kotlin.config.CompilerSettings; import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache; -import org.jetbrains.kotlin.rmi.CompileService; -import org.jetbrains.kotlin.rmi.CompilerId; -import org.jetbrains.kotlin.rmi.DaemonLaunchingOptions; -import org.jetbrains.kotlin.rmi.DaemonOptions; +import org.jetbrains.kotlin.rmi.*; import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient; import org.jetbrains.kotlin.utils.UtilsPackage; @@ -133,14 +130,13 @@ public class KotlinCompilerRunner { String[] argsArray = ArrayUtil.toStringArray(argumentsList); // trying the daemon first - if (incrementalCaches != null && KotlinCompilerClient.Companion.isDaemonEnabled()) { + if (incrementalCaches != null && RmiPackage.isDaemonEnabled()) { File libPath = CompilerRunnerUtil.getLibPath(environment.getKotlinPaths(), messageCollector); // TODO: it may be a good idea to cache the compilerId, since making it means calculating digest over jar(s) and if \\ // the lifetime of JPS process is small anyway, we can neglect the probability of changed compiler CompilerId compilerId = CompilerId.makeCompilerId(new File(libPath, "kotlin-compiler.jar")); - DaemonOptions daemonOptions = new DaemonOptions(); - DaemonLaunchingOptions daemonLaunchingOptions = new DaemonLaunchingOptions(); - KotlinCompilerClient.Companion.configureDaemonLaunchingOptions(daemonLaunchingOptions); + DaemonOptions daemonOptions = RmiPackage.configureDaemonOptions(); + DaemonLaunchingOptions daemonLaunchingOptions = RmiPackage.configureDaemonLaunchingOptions(true); // TODO: find proper stream to report daemon connection progress CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, System.out, true, true); if (daemon != null) { From 9bee97e810fb7aedcda233391fb6d3d76ebe35ed Mon Sep 17 00:00:00 2001 From: Ilya Chernikov Date: Fri, 28 Aug 2015 14:18:39 +0200 Subject: [PATCH 13/14] Refactoring after review --- build.xml | 4 + .../rmi/kotlinr/KotlinCompilerClient.kt | 36 +-- .../org/jetbrains/kotlin/rmi/DaemonParams.kt | 263 ++++++++++-------- .../kotlin/rmi/service/CompileDaemon.kt | 7 +- .../kotlin/rmi/service/CompileServiceImpl.kt | 41 ++- .../kotlin/daemon/CompilerDaemonTest.kt | 4 +- .../compilerRunner/KotlinCompilerRunner.java | 5 +- 7 files changed, 196 insertions(+), 164 deletions(-) diff --git a/build.xml b/build.xml index 6ef2272dc24..f1084a2b0b9 100644 --- a/build.xml +++ b/build.xml @@ -117,6 +117,8 @@ + + @@ -200,6 +202,8 @@ + + diff --git a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt index d6b863930e3..1850cafdaa6 100644 --- a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt +++ b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt @@ -26,12 +26,7 @@ import java.rmi.Remote import java.rmi.registry.LocateRegistry import java.util.concurrent.Semaphore import java.util.concurrent.TimeUnit -import java.util.concurrent.locks.ReentrantReadWriteLock -import kotlin.concurrent.read import kotlin.concurrent.thread -import kotlin.concurrent.write -import kotlin.platform.platformStatic -import kotlin.reflect.KProperty1 fun Process.isAlive() = try { @@ -71,7 +66,7 @@ public class KotlinCompilerClient { } - private fun startDaemon(compilerId: CompilerId, daemonLaunchingOptions: DaemonLaunchingOptions, daemonOptions: DaemonOptions, errStream: PrintStream) { + private fun startDaemon(compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, daemonOptions: DaemonOptions, errStream: PrintStream) { val javaExecutable = File(System.getProperty("java.home"), "bin").let { val javaw = File(it, "javaw.exe") if (javaw.exists()) javaw @@ -80,10 +75,10 @@ public class KotlinCompilerClient { // TODO: add some specific environment variables to the cp and may be command line, to allow some specific startup configs val args = listOf(javaExecutable.absolutePath, "-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator)) + - daemonLaunchingOptions.extractors.flatMap { it.extract("-") } + + daemonJVMOptions.mappers.flatMap { it.toArgs("-") } + COMPILER_DAEMON_CLASS_FQN + - daemonOptions.extractors.flatMap { it.extract(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } + - compilerId.extractors.flatMap { it.extract(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } + daemonOptions.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } + + compilerId.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } errStream.println("[daemon client] starting the daemon as: " + args.joinToString(" ")) val processBuilder = ProcessBuilder(args).redirectErrorStream(true) // assuming daemon process is deaf and (mostly) silent, so do not handle streams @@ -135,14 +130,14 @@ public class KotlinCompilerClient { (localId.compilerDigest.isEmpty() || remoteId.compilerDigest.isEmpty() || localId.compilerDigest == remoteId.compilerDigest) } - public fun connectToCompileService(compilerId: CompilerId, daemonLaunchingOptions: DaemonLaunchingOptions, daemonOptions: DaemonOptions, errStream: PrintStream, autostart: Boolean = true, checkId: Boolean = true): CompileService? { + public fun connectToCompileService(compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, daemonOptions: DaemonOptions, errStream: PrintStream, autostart: Boolean = true, checkId: Boolean = true): CompileService? { val service = connectToService(compilerId, daemonOptions, errStream) if (service != null) { if (!checkId || checkCompilerId(service, compilerId, errStream)) { errStream.println("[daemon client] found the suitable daemon") return service } - errStream.println("[daemon client] compiler identity don't match: " + compilerId.extractors.flatMap { it.extract("") }.joinToString(" ")) + errStream.println("[daemon client] compiler identity don't match: " + compilerId.mappers.flatMap { it.toArgs("") }.joinToString(" ")) if (!autostart) return null; errStream.println("[daemon client] shutdown the daemon") service.shutdown() @@ -155,13 +150,13 @@ public class KotlinCompilerClient { else errStream.println("[daemon client] cannot connect to Compile Daemon, trying to start") } - startDaemon(compilerId, daemonLaunchingOptions, daemonOptions, errStream) + startDaemon(compilerId, daemonJVMOptions, daemonOptions, errStream) errStream.println("[daemon client] daemon started, trying to connect") return connectToService(compilerId, daemonOptions, errStream) } public fun shutdownCompileService(daemonOptions: DaemonOptions): Unit { - KotlinCompilerClient.connectToCompileService(CompilerId(), DaemonLaunchingOptions(), daemonOptions, System.out, autostart = false, checkId = false) + KotlinCompilerClient.connectToCompileService(CompilerId(), DaemonJVMOptions(), daemonOptions, System.out, autostart = false, checkId = false) ?.shutdown() } @@ -196,20 +191,17 @@ public class KotlinCompilerClient { data class ClientOptions( public var stop: Boolean = false - ) :CmdlineParams { - override val extractors: List> - get() = listOf( BoolPropExtractor(this, ::stop)) - - override val parsers: List> - get() = listOf( BoolPropParser(this, ::stop)) + ) : OptionsGroup { + override val mappers: List> + get() = listOf( BoolPropMapper(this, ::stop)) } - platformStatic public fun main(vararg args: String) { + jvmStatic public fun main(vararg args: String) { val compilerId = CompilerId() val daemonOptions = DaemonOptions() - val daemonLaunchingOptions = DaemonLaunchingOptions() + val daemonLaunchingOptions = DaemonJVMOptions() val clientOptions = ClientOptions() - val filteredArgs = args.asIterable().filterSetProps(compilerId, daemonOptions, daemonLaunchingOptions, clientOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) + val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, daemonLaunchingOptions, clientOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) if (!clientOptions.stop) { if (compilerId.compilerClasspath.none()) { diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt index 80c6302de7d..815ea746007 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt @@ -21,9 +21,7 @@ import java.io.Serializable import java.lang.management.ManagementFactory import java.security.DigestInputStream import java.security.MessageDigest -import kotlin.platform.platformStatic import kotlin.reflect.KMutableProperty1 -import kotlin.reflect.KProperty1 public val COMPILER_JAR_NAME: String = "kotlin-compiler.jar" @@ -34,85 +32,147 @@ public val COMPILE_DAEMON_ENABLED_PROPERTY: String ="kotlin.daemon.enabled" public val COMPILE_DAEMON_JVM_OPTIONS_PROPERTY: String ="kotlin.daemon.jvm.options" public val COMPILE_DAEMON_OPTIONS_PROPERTY: String ="kotlin.daemon.options" public val COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX: String ="--daemon-" +public val COMPILE_DAEMON_TIMEOUT_INFINITE_S: Int = 0 +public val COMPILE_DAEMON_MEMORY_THRESHOLD_INFINITE: Long = 0L +val COMPILER_ID_DIGEST = "MD5" -open class PropExtractor>(val dest: C, - val prop: P, - val name: String, - val convert: ((v: V) -> String?) = { it.toString() }, - val skipIf: ((v: V) -> Boolean) = { false }, - val mergeWithDelimiter: String? = null) +//open class PropExtractor>(val dest: C, +// val prop: P, +// val name: String, +// val convert: ((v: V) -> String?) = { it.toString() }, +// val skipIf: ((v: V) -> Boolean) = { false }, +// val mergeWithDelimiter: String? = null) +//{ +// constructor(dest: C, prop: P, convert: ((v: V) -> String?) = { it.toString() }, skipIf: ((v: V) -> Boolean) = { false }) : this(dest, prop, prop.name, convert, skipIf) +// open fun extract(prefix: String = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX): List = +// when { +// skipIf(prop.get(dest)) -> listOf() +// mergeWithDelimiter != null -> listOf(prefix + name + mergeWithDelimiter + convert(prop.get(dest))).filterNotNull() +// else -> listOf(prefix + name, convert(prop.get(dest))).filterNotNull() +// } +//} +// +//class BoolPropExtractor>(dest: C, prop: P, name: String? = null) +// : PropExtractor(dest, prop, name ?: prop.name, convert = { null }, skipIf = { !prop.get(dest) }) +// +//class RestPropExtractor>>(dest: C, prop: P) : PropExtractor, P>(dest, prop, convert = { null }) { +// override fun extract(prefix: String): List = prop.get(dest).map { prefix + it } +//} +// +// +//open class PropParser>(val dest: C, +// val prop: P, alternativeNames: List, +// val parse: (s: String) -> V, +// val allowMergedArg: Boolean = false) { +// val names = listOf(prop.name) + alternativeNames +// constructor(dest: C, prop: P, parse: (s: String) -> V, allowMergedArg: Boolean = false) : this(dest, prop, listOf(), parse, allowMergedArg) +// fun apply(s: String) = prop.set(dest, parse(s)) +//} +// +//class BoolPropParser>(dest: C, prop: P): PropParser(dest, prop, { true }) +// +//class RestPropParser>>(dest: C, prop: P): PropParser, P>(dest, prop, { arrayListOf() }) { +// fun add(s: String) { prop.get(dest).add(s) } +//} + +// -------------------------------------------------------- + +open class PropMapper>(val dest: C, + val prop: P, + val names: List = listOf(prop.name), + val fromString: (s: String) -> V, + val toString: ((v: V) -> String?) = { it.toString() }, + val skipIf: ((v: V) -> Boolean) = { false }, + val mergeDelimiter: String? = null) { - constructor(dest: C, prop: P, convert: ((v: V) -> String?) = { it.toString() }, skipIf: ((v: V) -> Boolean) = { false }) : this(dest, prop, prop.name, convert, skipIf) - open fun extract(prefix: String = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX): List = + open fun toArgs(prefix: String = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX): List = when { skipIf(prop.get(dest)) -> listOf() - mergeWithDelimiter != null -> listOf(prefix + name + mergeWithDelimiter + convert(prop.get(dest))).filterNotNull() - else -> listOf(prefix + name, convert(prop.get(dest))).filterNotNull() + mergeDelimiter != null -> listOf(prefix + names.first() + mergeDelimiter + toString(prop.get(dest))).filterNotNull() + else -> listOf(prefix + names.first(), toString(prop.get(dest))).filterNotNull() } + fun apply(s: String) = prop.set(dest, fromString(s)) } -class BoolPropExtractor>(dest: C, prop: P, name: String? = null) - : PropExtractor(dest, prop, name ?: prop.name, convert = { null }, skipIf = { !prop.get(dest) }) +class StringPropMapper>(dest: C, + prop: P, + names: List = listOf(), + fromString: ((String) -> String) = { it }, + toString: ((String) -> String?) = { it.toString() }, + skipIf: ((String) -> Boolean) = { it.isEmpty() }, + mergeDelimiter: String? = null) +: PropMapper(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name), + fromString = fromString, toString = toString, skipIf = skipIf, mergeDelimiter = mergeDelimiter) -class RestPropExtractor>>(dest: C, prop: P) : PropExtractor, P>(dest, prop, convert = { null }) { - override fun extract(prefix: String): List = prop.get(dest).map { prefix + it } -} +class BoolPropMapper>(dest: C, prop: P, names: List = listOf()) + : PropMapper(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name), + fromString = { true }, toString = { null }, skipIf = { !prop.get(dest) }) - -open class PropParser>(val dest: C, - val prop: P, alternativeNames: List, - val parse: (s: String) -> V, - val allowMergedArg: Boolean = false) { - val names = listOf(prop.name) + alternativeNames - constructor(dest: C, prop: P, parse: (s: String) -> V, allowMergedArg: Boolean = false) : this(dest, prop, listOf(), parse, allowMergedArg) - fun apply(s: String) = prop.set(dest, parse(s)) -} - -class BoolPropParser>(dest: C, prop: P): PropParser(dest, prop, { true }) - -class RestPropParser>>(dest: C, prop: P): PropParser, P>(dest, prop, { arrayListOf() }) { +class RestPropMapper>>(dest: C, prop: P) + : PropMapper, P>(dest = dest, prop = prop, toString = { null }, fromString = { arrayListOf() }) +{ + override fun toArgs(prefix: String): List = prop.get(dest).map { prefix + it } fun add(s: String) { prop.get(dest).add(s) } } +// ------------------------------------------ -fun Iterable.filterSetProps(parsers: List>, prefix: String, restParser: RestPropParser<*,*>? = null) : Iterable { - var currentParser: PropParser<*,*,*>? = null +fun Iterable.filterExtractProps(propMappers: List>, prefix: String, restParser: RestPropMapper<*,*>? = null) : Iterable { + var currentPropMapper: PropMapper<*,*,*>? = null var matchingOption = "" val res = filter { param -> - if (currentParser == null) { - val parser = parsers.find { it.names.any { name -> - if (param.startsWith(prefix + name)) { matchingOption = prefix + name; true } - else false } } - if (parser != null) { - val optionLength = matchingOption.length() - when { - parser is BoolPropParser<*,*> -> - if (param.length() > optionLength) throw IllegalArgumentException("Invalid switch option '$param', expecting $matchingOption without arguments") - else parser.apply("") - param.length() > optionLength -> - if (param[optionLength] != '=') { - if (parser.allowMergedArg) parser.apply(param.substring(optionLength)) - else throw IllegalArgumentException("Invalid option syntax '$param', expecting $matchingOption[= ]") - } - else parser.apply(param.substring(optionLength + 1)) - else -> currentParser = parser + if (currentPropMapper == null) { + val propMapper = propMappers.find { + it !is RestPropMapper<*,*> && + it.names.any { name -> + if (param.startsWith(prefix + name)) { + matchingOption = prefix + name + true + } + else { + false + } } - false } - else if (restParser != null && param.startsWith(prefix)) { - restParser.add(param.removePrefix(prefix)) - false + when { + propMapper != null -> { + val optionLength = matchingOption.length() + when { + propMapper is BoolPropMapper<*,*> -> { + if (param.length() > optionLength) + throw IllegalArgumentException("Invalid switch option '$param', expecting $matchingOption without arguments") + propMapper.apply("") + } + param.length() > optionLength -> + if (param[optionLength] != '=') { + if (propMapper.mergeDelimiter == null) + throw IllegalArgumentException("Invalid option syntax '$param', expecting $matchingOption[= ]") + propMapper.apply(param.substring(optionLength)) + } + else { + propMapper.apply(param.substring(optionLength + 1)) + } + else -> + currentPropMapper = propMapper + } + false + } + restParser != null && param.startsWith(prefix) -> { + restParser.add(param.removePrefix(prefix)) + false + } + else -> true } - else true } else { - currentParser!!.apply(param) - currentParser = null + currentPropMapper!!.apply(param) + currentPropMapper = null false } } - if (currentParser != null) throw IllegalArgumentException("Expecting argument for the option $matchingOption") + if (currentPropMapper != null) + throw IllegalArgumentException("Expecting argument for the option $matchingOption") return res } @@ -124,59 +184,46 @@ fun Iterable.filterSetProps(parsers: List>, prefix: St -public interface CmdlineParams : Serializable { - public val extractors: List> - public val parsers: List> +public interface OptionsGroup : Serializable { + public val mappers: List> } -public fun Iterable.filterSetProps(vararg cs: CmdlineParams, prefix: String) : Iterable = - filterSetProps(cs.flatMap { it.parsers }, prefix) +public fun Iterable.filterExtractProps(vararg groups: OptionsGroup, prefix: String) : Iterable = + filterExtractProps(groups.flatMap { it.mappers }, prefix) -public data class DaemonLaunchingOptions( +public data class DaemonJVMOptions( public var maxMemory: String = "", public var maxPermSize: String = "", public var reservedCodeCacheSize: String = "", public var otherJvmParams: MutableCollection = arrayListOf() -) : CmdlineParams { +) : OptionsGroup { - override val extractors: List> - get() = listOf( PropExtractor(this, ::maxMemory, "Xmx", skipIf = { it.isEmpty() }, mergeWithDelimiter = ""), - PropExtractor(this, ::maxPermSize, "XX:MaxPermSize", skipIf = { it.isEmpty() }, mergeWithDelimiter = "="), - PropExtractor(this, ::reservedCodeCacheSize, "XX:ReservedCodeCacheSize", skipIf = { it.isEmpty() }, mergeWithDelimiter = "="), - RestPropExtractor(this, ::otherJvmParams)) + override val mappers: List> + get() = listOf( StringPropMapper(this, ::maxMemory, listOf("Xmx"), mergeDelimiter = ""), + StringPropMapper(this, ::maxPermSize, listOf("XX:MaxPermSize"), mergeDelimiter = "="), + StringPropMapper(this, ::reservedCodeCacheSize, listOf("XX:ReservedCodeCacheSize"), mergeDelimiter = "="), + restMapper) - override val parsers: List> - get() = listOf( PropParser(this, ::maxMemory, listOf("Xmx"), { it }, allowMergedArg = true), - PropParser(this, ::maxPermSize, listOf("XX:MaxPermSize"), { it }, allowMergedArg = true), - PropParser(this, ::reservedCodeCacheSize, listOf("XX:ReservedCodeCacheSize"), { it }, allowMergedArg = true)) - // otherJvmParams is missing here deliberately, it is used explicitly as a restParser param to filterSetProps + val restMapper: RestPropMapper<*,*> + get() = RestPropMapper(this, ::otherJvmParams) } public data class DaemonOptions( public var port: Int = COMPILE_DAEMON_DEFAULT_PORT, - public var autoshutdownMemoryThreshold: Long = 0 /* 0 means unchecked */, - public var autoshutdownIdleSeconds: Int = 0 /* 0 means unchecked */, + public var autoshutdownMemoryThreshold: Long = COMPILE_DAEMON_MEMORY_THRESHOLD_INFINITE, + public var autoshutdownIdleSeconds: Int = COMPILE_DAEMON_TIMEOUT_INFINITE_S, public var startEcho: String = COMPILER_SERVICE_RMI_NAME -) : CmdlineParams { +) : OptionsGroup { - override val extractors: List> - get() = listOf( PropExtractor(this, ::port), - PropExtractor(this, ::autoshutdownMemoryThreshold, skipIf = { it == 0L }), - PropExtractor(this, ::autoshutdownIdleSeconds, skipIf = { it == 0 }), - PropExtractor(this, ::startEcho)) - - override val parsers: List> - get() = listOf( PropParser(this, ::port, { it.toInt()}), - PropParser(this, ::autoshutdownMemoryThreshold, { it.toLong()}), - PropParser(this, ::autoshutdownIdleSeconds, { it.toInt()}), - PropParser(this, ::startEcho, { it.trim('"') })) + override val mappers: List> + get() = listOf( PropMapper(this, ::port, fromString = { it.toInt() }), + PropMapper(this, ::autoshutdownMemoryThreshold, fromString = { it.toLong() }, skipIf = { it == 0L }), + PropMapper(this, ::autoshutdownIdleSeconds, fromString = { it.toInt() }, skipIf = { it == 0 }), + PropMapper(this, ::startEcho, fromString = { it.trim('"') })) } -val COMPILER_ID_DIGEST = "MD5" - - fun updateSingleFileDigest(file: File, md: MessageDigest) { DigestInputStream(file.inputStream(), md).use { val buf = ByteArray(1024) @@ -215,27 +262,21 @@ public data class CompilerId( public var compilerDigest: String = "", public var compilerVersion: String = "" // TODO: checksum -) : CmdlineParams { +) : OptionsGroup { - override val extractors: List> - get() = listOf( PropExtractor(this, ::compilerClasspath, convert = { it.joinToString(File.pathSeparator) }), - PropExtractor(this, ::compilerDigest), - PropExtractor(this, ::compilerVersion, skipIf = { it.isEmpty() })) - - override val parsers: List> - get() = - listOf( PropParser(this, ::compilerClasspath, { it.trim('"').split(File.pathSeparator)}), - PropParser(this, ::compilerDigest, { it.trim('"') }), - PropParser(this, ::compilerVersion, { it.trim('"') })) + override val mappers: List> + get() = listOf( PropMapper(this, ::compilerClasspath, toString = { it.joinToString(File.pathSeparator) }, fromString = { it.trim('"').split(File.pathSeparator)}), + StringPropMapper(this, ::compilerDigest), + StringPropMapper(this, ::compilerVersion)) public fun updateDigest() { compilerDigest = compilerClasspath.getClasspathDigest() } companion object { - public platformStatic fun makeCompilerId(vararg paths: File): CompilerId = makeCompilerId(paths.asIterable()) + public jvmStatic fun makeCompilerId(vararg paths: File): CompilerId = makeCompilerId(paths.asIterable()) - public platformStatic fun makeCompilerId(paths: Iterable): CompilerId = + public jvmStatic fun makeCompilerId(paths: Iterable): CompilerId = // TODO consider reading version here CompilerId(compilerClasspath = paths.map { it.absolutePath }, compilerDigest = paths.getFilesClasspathDigest()) } @@ -245,27 +286,27 @@ public data class CompilerId( public fun isDaemonEnabled(): Boolean = System.getProperty(COMPILE_DAEMON_ENABLED_PROPERTY) != null -public fun configureDaemonLaunchingOptions(opts: DaemonLaunchingOptions, inheritMemoryLimits: Boolean): DaemonLaunchingOptions { +public fun configureDaemonLaunchingOptions(opts: DaemonJVMOptions, inheritMemoryLimits: Boolean): DaemonJVMOptions { // note: sequence matters, explicit override in COMPILE_DAEMON_JVM_OPTIONS_PROPERTY should be done after inputArguments processing if (inheritMemoryLimits) - ManagementFactory.getRuntimeMXBean().inputArguments.filterSetProps(opts.parsers, "-") + ManagementFactory.getRuntimeMXBean().inputArguments.filterExtractProps(opts.mappers, "-") System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)?.let { - opts.otherJvmParams.addAll( it.trim('"', '\'').split(",").filterSetProps(opts.parsers, "-", RestPropParser(opts, DaemonLaunchingOptions::otherJvmParams))) + opts.otherJvmParams.addAll( it.trim('"', '\'').split(",").filterExtractProps(opts.mappers, "-", opts.restMapper)) } return opts } -public fun configureDaemonLaunchingOptions(inheritMemoryLimits: Boolean): DaemonLaunchingOptions = - configureDaemonLaunchingOptions(DaemonLaunchingOptions(), inheritMemoryLimits = inheritMemoryLimits) +public fun configureDaemonLaunchingOptions(inheritMemoryLimits: Boolean): DaemonJVMOptions = + configureDaemonLaunchingOptions(DaemonJVMOptions(), inheritMemoryLimits = inheritMemoryLimits) jvmOverloads public fun configureDaemonOptions(opts: DaemonOptions = DaemonOptions()): DaemonOptions { System.getProperty(COMPILE_DAEMON_OPTIONS_PROPERTY)?.let { - val unrecognized = it.trim('"', '\'').split(",").filterSetProps(opts.parsers, "") + val unrecognized = it.trim('"', '\'').split(",").filterExtractProps(opts.mappers, "") if (unrecognized.any()) throw IllegalArgumentException( "Unrecognized daemon options passed via property $COMPILE_DAEMON_OPTIONS_PROPERTY: " + unrecognized.joinToString(" ") + - "\nSupported options: " + opts.extractors.joinToString(", ", transform = { it.name })) + "\nSupported options: " + opts.mappers.joinToString(", ", transform = { it.names.first() })) } return opts } diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt index 3c274d52f72..84c70919de2 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt @@ -17,10 +17,7 @@ package org.jetbrains.kotlin.rmi.service import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler -import org.jetbrains.kotlin.rmi.COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX -import org.jetbrains.kotlin.rmi.CompilerId -import org.jetbrains.kotlin.rmi.DaemonOptions -import org.jetbrains.kotlin.rmi.filterSetProps +import org.jetbrains.kotlin.rmi.* import org.jetbrains.kotlin.service.CompileServiceImpl import java.io.OutputStream import java.io.PrintStream @@ -96,7 +93,7 @@ public class CompileDaemon { val compilerId = CompilerId() val daemonOptions = DaemonOptions() - val filteredArgs = args.asIterable().filterSetProps(compilerId, daemonOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) + val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) if (filteredArgs.any()) { val helpLine = "usage: " diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt index fd2b51fbaf2..f3ca7bd58a9 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt @@ -17,7 +17,6 @@ package org.jetbrains.kotlin.service import org.jetbrains.kotlin.cli.common.CLICompiler -import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache @@ -25,15 +24,12 @@ import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompil import org.jetbrains.kotlin.rmi.* import org.jetbrains.kotlin.rmi.service.RemoteIncrementalCacheClient import org.jetbrains.kotlin.rmi.service.RemoteOutputStreamClient -import java.io.File -import java.io.FileNotFoundException import java.io.IOException import java.io.PrintStream import java.lang.management.ManagementFactory import java.net.URLClassLoader import java.rmi.registry.Registry import java.rmi.server.UnicastRemoteObject -import java.util.* import java.util.concurrent.TimeUnit import java.util.concurrent.locks.ReentrantReadWriteLock import java.util.jar.Manifest @@ -86,12 +82,14 @@ class CompileServiceImpl>( public class IncrementalCompilationComponentsImpl(val idToCache: Map): IncrementalCompilationComponents { // perf: cheap object, but still the pattern may be costly if there are too many calls to cache with the same id (which seems not to be the case now) override fun getIncrementalCache(moduleId: String): IncrementalCache = RemoteIncrementalCacheClient(idToCache[moduleId]!!) + // TODO: add appropriate proxy into interaction when lookup tracker is needed override fun getLookupTracker(): LookupTracker = LookupTracker.DO_NOTHING } private fun createCompileServices(incrementalCaches: Map): Services = Services.Builder() - .register(javaClass(), IncrementalCompilationComponentsImpl(incrementalCaches)) + .register(IncrementalCompilationComponents::class.java, IncrementalCompilationComponentsImpl(incrementalCaches)) + // TODO: add remote proxy for cancellation status tracking // .register(javaClass(), object: CompilationCanceledStatus { // override fun checkCanceled(): Unit = if (context.getCancelStatus().isCanceled()) throw CompilationCanceledException() // }) @@ -111,6 +109,7 @@ class CompileServiceImpl>( return memHeap.used } + // TODO: consider using version as a part of compiler ID or drop this function private fun loadKotlinVersionFromResource(): String { (javaClass.classLoader as? URLClassLoader) ?.findResource("META-INF/MANIFEST.MF") @@ -158,6 +157,7 @@ class CompileServiceImpl>( else body() } + // sometimes used for debugging fun spy(msg: String, body: () -> R): R { val res = body() log.info(msg + " = " + res.toString()) @@ -178,26 +178,25 @@ class CompileServiceImpl>( } override fun remoteCompile(args: Array, errStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int = - ifAlive { - checkedCompile(args) { - val strm = RemoteOutputStreamClient(errStream) - val printStrm = PrintStream(strm) - when (outputFormat) { - CompileService.OutputFormat.PLAIN -> compiler.exec(printStrm, *args) - CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStrm, Services.EMPTY, *args) - }.code - } + doCompile(args, errStream) { printStream -> + when (outputFormat) { + CompileService.OutputFormat.PLAIN -> compiler.exec(printStream, *args) + CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStream, Services.EMPTY, *args) + }.code } - override fun remoteIncrementalCompile(args: Array, caches: Map, errStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int = + override fun remoteIncrementalCompile(args: Array, caches: Map, outputStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int = + doCompile(args, outputStream) { printStream -> + when (outputFormat) { + CompileService.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation") + CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStream, createCompileServices(caches), *args) + }.code + } + + fun doCompile(args: Array, errStream: RemoteOutputStream, body: (PrintStream) -> Int): Int = ifAlive { checkedCompile(args) { - val strm = RemoteOutputStreamClient(errStream) - val printStrm = PrintStream(strm) - when (outputFormat) { - CompileService.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation") - CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStrm, createCompileServices(caches), *args) - }.code + body( PrintStream( RemoteOutputStreamClient(errStream))) } } } diff --git a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt index dd5842f9e5e..804afae51dc 100644 --- a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt +++ b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt @@ -20,7 +20,7 @@ import junit.framework.TestCase import org.jetbrains.kotlin.cli.CliBaseTest import org.jetbrains.kotlin.integration.KotlinIntegrationTestBase import org.jetbrains.kotlin.rmi.CompilerId -import org.jetbrains.kotlin.rmi.DaemonLaunchingOptions +import org.jetbrains.kotlin.rmi.DaemonJVMOptions import org.jetbrains.kotlin.rmi.DaemonOptions import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient import org.jetbrains.kotlin.test.JetTestUtils @@ -34,7 +34,7 @@ public class CompilerDaemonTest : KotlinIntegrationTestBase() { data class CompilerResults(val resultCode: Int, val out: String) val daemonOptions = DaemonOptions(port = KOTLIN_DAEMON_TEST_PORT) - val daemonLaunchingOptions = DaemonLaunchingOptions() + val daemonLaunchingOptions = DaemonJVMOptions() val compilerId by lazy { CompilerId.makeCompilerId( File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-compiler.jar"), File("dependencies/bootstrap-compiler/Kotlin/kotlinc/lib/kotlin-runtime.jar"), File("dependencies/bootstrap-compiler/Kotlin/kotlinc/lib/kotlin-reflect.jar")) } diff --git a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java index c5460c78d48..3f62f14e35b 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java +++ b/jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.java @@ -16,7 +16,6 @@ package org.jetbrains.kotlin.compilerRunner; -import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.util.text.StringUtil; import com.intellij.util.ArrayUtil; import com.intellij.util.Function; @@ -136,9 +135,9 @@ public class KotlinCompilerRunner { // the lifetime of JPS process is small anyway, we can neglect the probability of changed compiler CompilerId compilerId = CompilerId.makeCompilerId(new File(libPath, "kotlin-compiler.jar")); DaemonOptions daemonOptions = RmiPackage.configureDaemonOptions(); - DaemonLaunchingOptions daemonLaunchingOptions = RmiPackage.configureDaemonLaunchingOptions(true); + DaemonJVMOptions daemonJVMOptions = RmiPackage.configureDaemonLaunchingOptions(true); // TODO: find proper stream to report daemon connection progress - CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, System.out, true, true); + CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonJVMOptions, daemonOptions, System.out, true, true); if (daemon != null) { Integer res = KotlinCompilerClient.Companion.incrementalCompile(daemon, argsArray, incrementalCaches, out); return res.toString(); From 7c1c62882393ad423e755d2fd814ba7c5718cd82 Mon Sep 17 00:00:00 2001 From: Ilya Chernikov Date: Mon, 31 Aug 2015 19:10:28 +0200 Subject: [PATCH 14/14] Next round of refactoring after review --- .../org/jetbrains/kotlin/rmi/DaemonParams.kt | 144 ++++++------------ .../kotlin/rmi/service/CompileServiceImpl.kt | 9 +- 2 files changed, 55 insertions(+), 98 deletions(-) diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt index 815ea746007..eacf096d24b 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt @@ -37,46 +37,6 @@ public val COMPILE_DAEMON_MEMORY_THRESHOLD_INFINITE: Long = 0L val COMPILER_ID_DIGEST = "MD5" -//open class PropExtractor>(val dest: C, -// val prop: P, -// val name: String, -// val convert: ((v: V) -> String?) = { it.toString() }, -// val skipIf: ((v: V) -> Boolean) = { false }, -// val mergeWithDelimiter: String? = null) -//{ -// constructor(dest: C, prop: P, convert: ((v: V) -> String?) = { it.toString() }, skipIf: ((v: V) -> Boolean) = { false }) : this(dest, prop, prop.name, convert, skipIf) -// open fun extract(prefix: String = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX): List = -// when { -// skipIf(prop.get(dest)) -> listOf() -// mergeWithDelimiter != null -> listOf(prefix + name + mergeWithDelimiter + convert(prop.get(dest))).filterNotNull() -// else -> listOf(prefix + name, convert(prop.get(dest))).filterNotNull() -// } -//} -// -//class BoolPropExtractor>(dest: C, prop: P, name: String? = null) -// : PropExtractor(dest, prop, name ?: prop.name, convert = { null }, skipIf = { !prop.get(dest) }) -// -//class RestPropExtractor>>(dest: C, prop: P) : PropExtractor, P>(dest, prop, convert = { null }) { -// override fun extract(prefix: String): List = prop.get(dest).map { prefix + it } -//} -// -// -//open class PropParser>(val dest: C, -// val prop: P, alternativeNames: List, -// val parse: (s: String) -> V, -// val allowMergedArg: Boolean = false) { -// val names = listOf(prop.name) + alternativeNames -// constructor(dest: C, prop: P, parse: (s: String) -> V, allowMergedArg: Boolean = false) : this(dest, prop, listOf(), parse, allowMergedArg) -// fun apply(s: String) = prop.set(dest, parse(s)) -//} -// -//class BoolPropParser>(dest: C, prop: P): PropParser(dest, prop, { true }) -// -//class RestPropParser>>(dest: C, prop: P): PropParser, P>(dest, prop, { arrayListOf() }) { -// fun add(s: String) { prop.get(dest).add(s) } -//} - -// -------------------------------------------------------- open class PropMapper>(val dest: C, val prop: P, @@ -89,10 +49,10 @@ open class PropMapper>(val dest: C, open fun toArgs(prefix: String = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX): List = when { skipIf(prop.get(dest)) -> listOf() - mergeDelimiter != null -> listOf(prefix + names.first() + mergeDelimiter + toString(prop.get(dest))).filterNotNull() + mergeDelimiter != null -> listOf( listOf(prefix + names.first(), toString(prop.get(dest))).filterNotNull().joinToString(mergeDelimiter)) else -> listOf(prefix + names.first(), toString(prop.get(dest))).filterNotNull() } - fun apply(s: String) = prop.set(dest, fromString(s)) + open fun apply(s: String) = prop.set(dest, fromString(s)) } class StringPropMapper>(dest: C, @@ -113,69 +73,65 @@ class RestPropMapper>>(dest : PropMapper, P>(dest = dest, prop = prop, toString = { null }, fromString = { arrayListOf() }) { override fun toArgs(prefix: String): List = prop.get(dest).map { prefix + it } + override fun apply(s: String) = add(s) fun add(s: String) { prop.get(dest).add(s) } } -// ------------------------------------------ + +inline fun Iterable.firstMapOrNull(mappingPredicate: (T) -> Pair): R? { + for (element in this) { + val (found, mapped) = mappingPredicate(element) + if (found) return mapped + } + return null +} + fun Iterable.filterExtractProps(propMappers: List>, prefix: String, restParser: RestPropMapper<*,*>? = null) : Iterable { - var currentPropMapper: PropMapper<*,*,*>? = null - var matchingOption = "" - val res = filter { param -> - if (currentPropMapper == null) { - val propMapper = propMappers.find { - it !is RestPropMapper<*,*> && - it.names.any { name -> - if (param.startsWith(prefix + name)) { - matchingOption = prefix + name - true + + val iter = iterator() + val rest = arrayListOf() + + while (iter.hasNext()) { + val param = iter.next() + val (propMapper, matchingOption) = propMappers.firstMapOrNull { + val name = if (it !is RestPropMapper<*,*>) it.names.firstOrNull { param.startsWith(prefix + it) } else null + Pair(name != null, Pair(it, name)) + } ?: Pair(null, null) + + when { + propMapper != null -> { + val optionLength = prefix.length() + matchingOption!!.length() + when { + propMapper is BoolPropMapper<*,*> -> { + if (param.length() > optionLength) + throw IllegalArgumentException("Invalid switch option '$param', expecting $prefix$matchingOption without arguments") + propMapper.apply("") } - else { - false - } - } - } - when { - propMapper != null -> { - val optionLength = matchingOption.length() - when { - propMapper is BoolPropMapper<*,*> -> { - if (param.length() > optionLength) - throw IllegalArgumentException("Invalid switch option '$param', expecting $matchingOption without arguments") - propMapper.apply("") + param.length() > optionLength -> + if (param[optionLength] != '=') { + if (propMapper.mergeDelimiter == null) + throw IllegalArgumentException("Invalid option syntax '$param', expecting $prefix$matchingOption[= ]") + propMapper.apply(param.substring(optionLength)) } - param.length() > optionLength -> - if (param[optionLength] != '=') { - if (propMapper.mergeDelimiter == null) - throw IllegalArgumentException("Invalid option syntax '$param', expecting $matchingOption[= ]") - propMapper.apply(param.substring(optionLength)) - } - else { - propMapper.apply(param.substring(optionLength + 1)) - } - else -> - currentPropMapper = propMapper + else { + propMapper.apply(param.substring(optionLength + 1)) + } + else -> { + if (!iter.hasNext()) throw IllegalArgumentException("Expecting argument for the option $prefix$matchingOption") + propMapper.apply(iter.next()) } - false } - restParser != null && param.startsWith(prefix) -> { - restParser.add(param.removePrefix(prefix)) - false - } - else -> true } - } - else { - currentPropMapper!!.apply(param) - currentPropMapper = null - false + restParser != null && param.startsWith(prefix) -> + restParser.add(param.removePrefix(prefix)) + else -> rest.add(param) } } - if (currentPropMapper != null) - throw IllegalArgumentException("Expecting argument for the option $matchingOption") - return res + return rest } + // TODO: find out how to create more generic variant using first constructor //fun C.propsToParams() { // val kc = C::class @@ -196,7 +152,7 @@ public data class DaemonJVMOptions( public var maxMemory: String = "", public var maxPermSize: String = "", public var reservedCodeCacheSize: String = "", - public var otherJvmParams: MutableCollection = arrayListOf() + public var jvmParams: MutableCollection = arrayListOf() ) : OptionsGroup { override val mappers: List> @@ -206,7 +162,7 @@ public data class DaemonJVMOptions( restMapper) val restMapper: RestPropMapper<*,*> - get() = RestPropMapper(this, ::otherJvmParams) + get() = RestPropMapper(this, ::jvmParams) } public data class DaemonOptions( @@ -292,7 +248,7 @@ public fun configureDaemonLaunchingOptions(opts: DaemonJVMOptions, inheritMemory ManagementFactory.getRuntimeMXBean().inputArguments.filterExtractProps(opts.mappers, "-") System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)?.let { - opts.otherJvmParams.addAll( it.trim('"', '\'').split(",").filterExtractProps(opts.mappers, "-", opts.restMapper)) + opts.jvmParams.addAll(it.trim('"', '\'').split(",").filterExtractProps(opts.mappers, "-", opts.restMapper)) } return opts } diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt index f3ca7bd58a9..797a6e7cb5c 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt @@ -17,6 +17,7 @@ package org.jetbrains.kotlin.service import org.jetbrains.kotlin.cli.common.CLICompiler +import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache @@ -182,7 +183,7 @@ class CompileServiceImpl>( when (outputFormat) { CompileService.OutputFormat.PLAIN -> compiler.exec(printStream, *args) CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStream, Services.EMPTY, *args) - }.code + } } override fun remoteIncrementalCompile(args: Array, caches: Map, outputStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int = @@ -190,13 +191,13 @@ class CompileServiceImpl>( when (outputFormat) { CompileService.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation") CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStream, createCompileServices(caches), *args) - }.code + } } - fun doCompile(args: Array, errStream: RemoteOutputStream, body: (PrintStream) -> Int): Int = + fun doCompile(args: Array, errStream: RemoteOutputStream, body: (PrintStream) -> ExitCode): Int = ifAlive { checkedCompile(args) { - body( PrintStream( RemoteOutputStreamClient(errStream))) + body( PrintStream( RemoteOutputStreamClient(errStream))).code } } }