Merge pull request #743 from JetBrains/rr/compile-service

Kotlin compile daemon
This commit is contained in:
Ilya Chernikov
2015-09-01 09:15:13 +02:00
27 changed files with 1391 additions and 16 deletions
+2
View File
@@ -19,6 +19,8 @@
<element id="module-output" name="util" />
<element id="module-output" name="util.runtime" />
<element id="file-copy" path="$PROJECT_DIR$/resources/kotlinManifest.properties" />
<element id="module-output" name="kotlinr" />
<element id="module-output" name="rmi-interface" />
</root>
</artifact>
</component>
+2
View File
@@ -44,6 +44,8 @@
<element id="module-output" name="idea-core" />
<element id="module-output" name="idea-js" />
<element id="module-output" name="container" />
<element id="module-output" name="rmi-interface" />
<element id="module-output" name="kotlinr" />
</element>
<element id="library" level="project" name="javax.inject" />
<element id="directory" name="jps">
+3
View File
@@ -50,10 +50,13 @@
<module fileurl="file://$PROJECT_DIR$/js/js.translator/js.translator.iml" filepath="$PROJECT_DIR$/js/js.translator/js.translator.iml" group="compiler/js" />
<module fileurl="file://$PROJECT_DIR$/jps-plugin/kannotator-jps-plugin-test/kannotator-jps-plugin-test.iml" filepath="$PROJECT_DIR$/jps-plugin/kannotator-jps-plugin-test/kannotator-jps-plugin-test.iml" group="ide/jps" />
<module fileurl="file://$PROJECT_DIR$/idea/kotlin-android-plugin/kotlin-android-plugin.iml" filepath="$PROJECT_DIR$/idea/kotlin-android-plugin/kotlin-android-plugin.iml" group="ide" />
<module fileurl="file://$PROJECT_DIR$/compiler/rmi/kotlinr/kotlinr.iml" filepath="$PROJECT_DIR$/compiler/rmi/kotlinr/kotlinr.iml" group="rmi" />
<module fileurl="file://$PROJECT_DIR$/compiler/light-classes/light-classes.iml" filepath="$PROJECT_DIR$/compiler/light-classes/light-classes.iml" group="compiler/java" />
<module fileurl="file://$PROJECT_DIR$/compiler/plugin-api/plugin-api.iml" filepath="$PROJECT_DIR$/compiler/plugin-api/plugin-api.iml" group="compiler" />
<module fileurl="file://$PROJECT_DIR$/compiler/preloader/preloader.iml" filepath="$PROJECT_DIR$/compiler/preloader/preloader.iml" group="compiler/cli" />
<module fileurl="file://$PROJECT_DIR$/core/reflection.jvm/reflection.jvm.iml" filepath="$PROJECT_DIR$/core/reflection.jvm/reflection.jvm.iml" group="core" />
<module fileurl="file://$PROJECT_DIR$/compiler/rmi/rmi-interface/rmi-interface.iml" filepath="$PROJECT_DIR$/compiler/rmi/rmi-interface/rmi-interface.iml" group="rmi" />
<module fileurl="file://$PROJECT_DIR$/compiler/rmi/rmi-server/rmi-server.iml" filepath="$PROJECT_DIR$/compiler/rmi/rmi-server/rmi-server.iml" group="rmi" />
<module fileurl="file://$PROJECT_DIR$/core/runtime.jvm/runtime.jvm.iml" filepath="$PROJECT_DIR$/core/runtime.jvm/runtime.jvm.iml" group="core" />
<module fileurl="file://$PROJECT_DIR$/compiler/serialization/serialization.iml" filepath="$PROJECT_DIR$/compiler/serialization/serialization.iml" group="compiler" />
<module fileurl="file://$PROJECT_DIR$/compiler/util/util.iml" filepath="$PROJECT_DIR$/compiler/util/util.iml" />
+27 -1
View File
@@ -86,6 +86,8 @@
<include name="compiler/builtins-serializer/src"/>
<include name="compiler/cli/src"/>
<include name="compiler/cli/cli-common/src"/>
<include name="compiler/rmi/rmi-server/src"/>
<include name="compiler/rmi/rmi-interface/src"/>
<include name="compiler/container/src"/>
<include name="compiler/frontend/src"/>
<include name="compiler/frontend.java/src"/>
@@ -115,6 +117,8 @@
<include name="backend-common/**"/>
<include name="cli/**"/>
<include name="cli-common/**"/>
<include name="rmi-server/**"/>
<include name="rmi-interface/**"/>
<include name="util/**"/>
<include name="util.runtime/**"/>
<include name="light-classes/**"/>
@@ -198,6 +202,8 @@
<fileset dir="compiler/builtins-serializer/src"/>
<fileset dir="compiler/cli/src"/>
<fileset dir="compiler/cli/cli-common/src"/>
<include name="compiler/rmi/rmi-server/src"/>
<include name="compiler/rmi/rmi-interface/src"/>
<fileset dir="compiler/container/src"/>
<fileset dir="compiler/frontend/src"/>
<fileset dir="compiler/frontend.java/src"/>
@@ -535,6 +541,26 @@
</jar>
</target>
<target name="kotlinr">
<cleandir dir="${output}/classes/kotlinr"/>
<kotlinc output="${output}/classes/kotlinr">
<src>
<pathelement path="compiler/rmi/kotlinr/src"/>
</src>
<classpath>
<pathelement path="${bootstrap.runtime}"/>
<pathelement path="${bootstrap.reflect}"/>
<pathelement path="${kotlin-home}/lib/kotlin-compiler.jar"/>
</classpath>
</kotlinc>
<jar destfile="${kotlin-home}/lib/kotlinr.jar">
<fileset dir="${output}/classes/kotlinr"/>
<zipfileset file="${kotlin-home}/build.txt" prefix="META-INF"/>
</jar>
</target>
<target name="android-compiler-plugin">
<cleandir dir="${output}/classes/android-compiler-plugin"/>
@@ -839,7 +865,7 @@
depends="builtins,stdlib,core,reflection,pack-runtime,pack-runtime-sources"/>
<target name="dist"
depends="clean,init,prepare-dist,preloader,serialize-builtins,compiler,compiler-sources,ant-tools,jdk-annotations,android-sdk-annotations,runtime,kotlin-js-stdlib,android-compiler-plugin"
depends="clean,init,prepare-dist,preloader,serialize-builtins,compiler,compiler-sources,ant-tools,jdk-annotations,android-sdk-annotations,runtime,kotlin-js-stdlib,android-compiler-plugin,kotlinr"
description="Builds redistributables from sources"/>
<target name="dist-quick"
+14
View File
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="kotlin-runtime" level="project" />
<orderEntry type="module" module-name="rmi-interface" />
<orderEntry type="module" module-name="frontend.java" />
</component>
</module>
@@ -0,0 +1,263 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.rmi.kotlinr
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
import org.jetbrains.kotlin.rmi.*
import java.io.File
import java.io.OutputStream
import java.io.PrintStream
import java.rmi.ConnectException
import java.rmi.Remote
import java.rmi.registry.LocateRegistry
import java.util.concurrent.Semaphore
import java.util.concurrent.TimeUnit
import kotlin.concurrent.thread
fun Process.isAlive() =
try {
this.exitValue()
false
}
catch (e: IllegalThreadStateException) {
true
}
public class KotlinCompilerClient {
companion object {
val DAEMON_STARTUP_TIMEOUT_MS = 10000L
private fun connectToService(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream): CompileService? {
val compilerObj = connectToDaemon(compilerId, daemonOptions, errStream) ?: return null // no registry - no daemon running
return compilerObj as? CompileService ?:
throw ClassCastException("Unable to cast compiler service, actual class received: ${compilerObj.javaClass}")
}
private fun connectToDaemon(compilerId: CompilerId, daemonOptions: DaemonOptions, errStream: PrintStream): Remote? {
try {
val daemon = LocateRegistry.getRegistry("localhost", daemonOptions.port)
?.lookup(COMPILER_SERVICE_RMI_NAME)
if (daemon != null)
return daemon
errStream.println("[daemon client] daemon not found")
}
catch (e: ConnectException) {
errStream.println("[daemon client] cannot connect to registry: " + (e.getCause()?.getMessage() ?: e.getMessage() ?: "unknown exception"))
// ignoring it - processing below
}
return null
}
private fun startDaemon(compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, daemonOptions: DaemonOptions, errStream: PrintStream) {
val javaExecutable = File(System.getProperty("java.home"), "bin").let {
val javaw = File(it, "javaw.exe")
if (javaw.exists()) javaw
else File(it, "java")
}
// TODO: add some specific environment variables to the cp and may be command line, to allow some specific startup configs
val args = listOf(javaExecutable.absolutePath,
"-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator)) +
daemonJVMOptions.mappers.flatMap { it.toArgs("-") } +
COMPILER_DAEMON_CLASS_FQN +
daemonOptions.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } +
compilerId.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) }
errStream.println("[daemon client] starting the daemon as: " + args.joinToString(" "))
val processBuilder = ProcessBuilder(args).redirectErrorStream(true)
// assuming daemon process is deaf and (mostly) silent, so do not handle streams
val daemon = processBuilder.start()
var isEchoRead = Semaphore(1)
isEchoRead.acquire()
val stdoutThread =
thread {
daemon.getInputStream()
.reader()
.forEachLine {
if (daemonOptions.startEcho.isNotEmpty() && it.contains(daemonOptions.startEcho)) {
isEchoRead.release()
return@forEachLine
}
errStream.println("[daemon] " + it)
}
}
try {
// trying to wait for process
if (daemonOptions.startEcho.isNotEmpty()) {
errStream.println("[daemon client] waiting for daemon to respond")
val succeeded = isEchoRead.tryAcquire(DAEMON_STARTUP_TIMEOUT_MS, TimeUnit.MILLISECONDS)
if (!daemon.isAlive())
throw Exception("Daemon terminated unexpectedly")
if (!succeeded)
throw Exception("Unable to get response from daemon in $DAEMON_STARTUP_TIMEOUT_MS ms")
}
else
// without startEcho defined waiting for max timeout
Thread.sleep(DAEMON_STARTUP_TIMEOUT_MS)
}
finally {
// assuming that all important output is already done, the rest should be routed to the log by the daemon itself
if (stdoutThread.isAlive)
// TODO: find better method to stop the thread, but seems it will require asynchronous consuming of the stream
stdoutThread.stop()
}
}
public fun checkCompilerId(compiler: CompileService, localId: CompilerId, errStream: PrintStream): Boolean {
val remoteId = compiler.getCompilerId()
errStream.println("[daemon client] remoteId = " + remoteId.toString())
errStream.println("[daemon client] localId = " + localId.toString())
return (localId.compilerVersion.isEmpty() || localId.compilerVersion == remoteId.compilerVersion) &&
(localId.compilerClasspath.all { remoteId.compilerClasspath.contains(it) }) &&
(localId.compilerDigest.isEmpty() || remoteId.compilerDigest.isEmpty() || localId.compilerDigest == remoteId.compilerDigest)
}
public fun connectToCompileService(compilerId: CompilerId, daemonJVMOptions: DaemonJVMOptions, daemonOptions: DaemonOptions, errStream: PrintStream, autostart: Boolean = true, checkId: Boolean = true): CompileService? {
val service = connectToService(compilerId, daemonOptions, errStream)
if (service != null) {
if (!checkId || checkCompilerId(service, compilerId, errStream)) {
errStream.println("[daemon client] found the suitable daemon")
return service
}
errStream.println("[daemon client] compiler identity don't match: " + compilerId.mappers.flatMap { it.toArgs("") }.joinToString(" "))
if (!autostart) return null;
errStream.println("[daemon client] shutdown the daemon")
service.shutdown()
// TODO: find more reliable way
Thread.sleep(1000)
errStream.println("[daemon client] daemon shut down correctly, restarting")
}
else {
if (!autostart) return null;
else errStream.println("[daemon client] cannot connect to Compile Daemon, trying to start")
}
startDaemon(compilerId, daemonJVMOptions, daemonOptions, errStream)
errStream.println("[daemon client] daemon started, trying to connect")
return connectToService(compilerId, daemonOptions, errStream)
}
public fun shutdownCompileService(daemonOptions: DaemonOptions): Unit {
KotlinCompilerClient.connectToCompileService(CompilerId(), DaemonJVMOptions(), daemonOptions, System.out, autostart = false, checkId = false)
?.shutdown()
}
public fun shutdownCompileService(): Unit {
shutdownCompileService(DaemonOptions())
}
public fun compile(compiler: CompileService, args: Array<out String>, out: OutputStream): Int {
val outStrm = RemoteOutputStreamServer(out)
try {
return compiler.remoteCompile(args, outStrm, CompileService.OutputFormat.PLAIN)
}
finally {
outStrm.disconnect()
}
}
public fun incrementalCompile(compiler: CompileService, args: Array<out String>, caches: Map<String, IncrementalCache>, out: OutputStream): Int {
val outStrm = RemoteOutputStreamServer(out)
val cacheServers = hashMapOf<String, RemoteIncrementalCacheServer>()
try {
caches.mapValuesTo(cacheServers, { RemoteIncrementalCacheServer( it.getValue()) })
return compiler.remoteIncrementalCompile(args, cacheServers, outStrm, CompileService.OutputFormat.XML)
}
finally {
cacheServers.forEach { it.getValue().disconnect() }
outStrm.disconnect()
}
}
data class ClientOptions(
public var stop: Boolean = false
) : OptionsGroup {
override val mappers: List<PropMapper<*, *, *>>
get() = listOf( BoolPropMapper(this, ::stop))
}
jvmStatic public fun main(vararg args: String) {
val compilerId = CompilerId()
val daemonOptions = DaemonOptions()
val daemonLaunchingOptions = DaemonJVMOptions()
val clientOptions = ClientOptions()
val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, daemonLaunchingOptions, clientOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX)
if (!clientOptions.stop) {
if (compilerId.compilerClasspath.none()) {
// attempt to find compiler to use
println("compiler wasn't explicitly specified, attempt to find appropriate jar")
System.getProperty("java.class.path")
?.split(File.pathSeparator)
?.map { File(it).parent }
?.distinct()
?.map {
it?.walk()
?.firstOrNull { it.getName().equals(COMPILER_JAR_NAME, ignoreCase = true) }
}
?.filterNotNull()
?.firstOrNull()
?.let { compilerId.compilerClasspath = listOf(it.absolutePath) }
}
if (compilerId.compilerClasspath.none())
throw IllegalArgumentException("Cannot find compiler jar")
else
println("desired compiler classpath: " + compilerId.compilerClasspath.joinToString(File.pathSeparator))
compilerId.updateDigest()
}
val daemon = connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, System.out, autostart = !clientOptions.stop, checkId = !clientOptions.stop)
if (daemon == null) {
if (clientOptions.stop) println("No daemon found to shut down")
else throw Exception("Unable to connect to daemon")
}
else when {
clientOptions.stop -> {
println("Shutdown the daemon")
daemon.shutdown()
println("Daemon shut down successfully")
}
else -> {
println("Executing daemon compilation with args: " + filteredArgs.joinToString(" "))
val outStrm = RemoteOutputStreamServer(System.out)
try {
val memBefore = daemon.getUsedMemory() / 1024
val startTime = System.nanoTime()
val res = daemon.remoteCompile(filteredArgs.toArrayList().toTypedArray(), outStrm, CompileService.OutputFormat.PLAIN)
val endTime = System.nanoTime()
println("Compilation result code: $res")
val memAfter = daemon.getUsedMemory() / 1024
println("Compilation time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms")
println("Used memory $memAfter (${"%+d".format(memAfter - memBefore)} kb)")
}
finally {
outStrm.disconnect()
}
}
}
}
}
}
@@ -0,0 +1,41 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.rmi.kotlinr
import org.jetbrains.kotlin.rmi.CompileService
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache;
import java.rmi.server.UnicastRemoteObject
public class RemoteIncrementalCacheServer(val cache: IncrementalCache) : CompileService.RemoteIncrementalCache {
init {
UnicastRemoteObject.exportObject(this, 0)
}
override fun getObsoletePackageParts(): Collection<String> = cache.getObsoletePackageParts()
override fun getPackageData(fqName: String): ByteArray? = cache.getPackageData(fqName)
override fun close() {
cache.close()
}
public fun disconnect() {
UnicastRemoteObject.unexportObject(this, true)
}
}
@@ -0,0 +1,45 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.rmi.kotlinr
import org.jetbrains.kotlin.rmi.RemoteOutputStream
import java.io.OutputStream
import java.rmi.server.UnicastRemoteObject
class RemoteOutputStreamServer(val out: OutputStream) : RemoteOutputStream {
init {
UnicastRemoteObject.exportObject(this, 0)
}
public fun disconnect() {
UnicastRemoteObject.unexportObject(this, true)
}
override fun close() {
out.close()
}
override fun write(data: ByteArray, offset: Int, length: Int) {
out.write(data, offset, length)
}
override fun write(dataByte: Int) {
out.write(dataByte)
}
}
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<module version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="kotlin-runtime" level="project" />
<orderEntry type="module" module-name="cli-common" />
<orderEntry type="module" module-name="frontend.java" />
</component>
</module>
@@ -0,0 +1,58 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.rmi
import java.rmi.Remote
import java.rmi.RemoteException
public interface CompileService : Remote {
public enum class OutputFormat : java.io.Serializable {
PLAIN,
XML
}
public interface RemoteIncrementalCache : Remote {
throws(RemoteException::class)
public fun getObsoletePackageParts(): Collection<String>
throws(RemoteException::class)
public fun getPackageData(fqName: String): ByteArray?
throws(RemoteException::class)
public fun close()
}
throws(RemoteException::class)
public fun getCompilerId(): CompilerId
throws(RemoteException::class)
public fun getUsedMemory(): Long
throws(RemoteException::class)
public fun shutdown()
throws(RemoteException::class)
public fun remoteCompile(args: Array<out String>, errStream: RemoteOutputStream, outputFormat: OutputFormat): Int
throws(RemoteException::class)
public fun remoteIncrementalCompile(
args: Array<out String>,
caches: Map<String, RemoteIncrementalCache>,
outputStream: RemoteOutputStream,
outputFormat: OutputFormat): Int
}
@@ -0,0 +1,269 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.rmi
import java.io.File
import java.io.Serializable
import java.lang.management.ManagementFactory
import java.security.DigestInputStream
import java.security.MessageDigest
import kotlin.reflect.KMutableProperty1
public val COMPILER_JAR_NAME: String = "kotlin-compiler.jar"
public val COMPILER_SERVICE_RMI_NAME: String = "KotlinJvmCompilerService"
public val COMPILER_DAEMON_CLASS_FQN: String = "org.jetbrains.kotlin.rmi.service.CompileDaemon"
public val COMPILE_DAEMON_DEFAULT_PORT: Int = 17031
public val COMPILE_DAEMON_ENABLED_PROPERTY: String ="kotlin.daemon.enabled"
public val COMPILE_DAEMON_JVM_OPTIONS_PROPERTY: String ="kotlin.daemon.jvm.options"
public val COMPILE_DAEMON_OPTIONS_PROPERTY: String ="kotlin.daemon.options"
public val COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX: String ="--daemon-"
public val COMPILE_DAEMON_TIMEOUT_INFINITE_S: Int = 0
public val COMPILE_DAEMON_MEMORY_THRESHOLD_INFINITE: Long = 0L
val COMPILER_ID_DIGEST = "MD5"
open class PropMapper<C, V, P: KMutableProperty1<C, V>>(val dest: C,
val prop: P,
val names: List<String> = listOf(prop.name),
val fromString: (s: String) -> V,
val toString: ((v: V) -> String?) = { it.toString() },
val skipIf: ((v: V) -> Boolean) = { false },
val mergeDelimiter: String? = null)
{
open fun toArgs(prefix: String = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX): List<String> =
when {
skipIf(prop.get(dest)) -> listOf<String>()
mergeDelimiter != null -> listOf( listOf(prefix + names.first(), toString(prop.get(dest))).filterNotNull().joinToString(mergeDelimiter))
else -> listOf(prefix + names.first(), toString(prop.get(dest))).filterNotNull()
}
open fun apply(s: String) = prop.set(dest, fromString(s))
}
class StringPropMapper<C, P: KMutableProperty1<C, String>>(dest: C,
prop: P,
names: List<String> = listOf(),
fromString: ((String) -> String) = { it },
toString: ((String) -> String?) = { it.toString() },
skipIf: ((String) -> Boolean) = { it.isEmpty() },
mergeDelimiter: String? = null)
: PropMapper<C, String, P>(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name),
fromString = fromString, toString = toString, skipIf = skipIf, mergeDelimiter = mergeDelimiter)
class BoolPropMapper<C, P: KMutableProperty1<C, Boolean>>(dest: C, prop: P, names: List<String> = listOf())
: PropMapper<C, Boolean, P>(dest = dest, prop = prop, names = if (names.any()) names else listOf(prop.name),
fromString = { true }, toString = { null }, skipIf = { !prop.get(dest) })
class RestPropMapper<C, P: KMutableProperty1<C, MutableCollection<String>>>(dest: C, prop: P)
: PropMapper<C, MutableCollection<String>, P>(dest = dest, prop = prop, toString = { null }, fromString = { arrayListOf() })
{
override fun toArgs(prefix: String): List<String> = prop.get(dest).map { prefix + it }
override fun apply(s: String) = add(s)
fun add(s: String) { prop.get(dest).add(s) }
}
inline fun <T, R: Any> Iterable<T>.firstMapOrNull(mappingPredicate: (T) -> Pair<Boolean, R?>): R? {
for (element in this) {
val (found, mapped) = mappingPredicate(element)
if (found) return mapped
}
return null
}
fun Iterable<String>.filterExtractProps(propMappers: List<PropMapper<*,*,*>>, prefix: String, restParser: RestPropMapper<*,*>? = null) : Iterable<String> {
val iter = iterator()
val rest = arrayListOf<String>()
while (iter.hasNext()) {
val param = iter.next()
val (propMapper, matchingOption) = propMappers.firstMapOrNull {
val name = if (it !is RestPropMapper<*,*>) it.names.firstOrNull { param.startsWith(prefix + it) } else null
Pair(name != null, Pair(it, name))
} ?: Pair(null, null)
when {
propMapper != null -> {
val optionLength = prefix.length() + matchingOption!!.length()
when {
propMapper is BoolPropMapper<*,*> -> {
if (param.length() > optionLength)
throw IllegalArgumentException("Invalid switch option '$param', expecting $prefix$matchingOption without arguments")
propMapper.apply("")
}
param.length() > optionLength ->
if (param[optionLength] != '=') {
if (propMapper.mergeDelimiter == null)
throw IllegalArgumentException("Invalid option syntax '$param', expecting $prefix$matchingOption[= ]<arg>")
propMapper.apply(param.substring(optionLength))
}
else {
propMapper.apply(param.substring(optionLength + 1))
}
else -> {
if (!iter.hasNext()) throw IllegalArgumentException("Expecting argument for the option $prefix$matchingOption")
propMapper.apply(iter.next())
}
}
}
restParser != null && param.startsWith(prefix) ->
restParser.add(param.removePrefix(prefix))
else -> rest.add(param)
}
}
return rest
}
// TODO: find out how to create more generic variant using first constructor
//fun<C> C.propsToParams() {
// val kc = C::class
// kc.constructors.first().
//}
public interface OptionsGroup : Serializable {
public val mappers: List<PropMapper<*,*,*>>
}
public fun Iterable<String>.filterExtractProps(vararg groups: OptionsGroup, prefix: String) : Iterable<String> =
filterExtractProps(groups.flatMap { it.mappers }, prefix)
public data class DaemonJVMOptions(
public var maxMemory: String = "",
public var maxPermSize: String = "",
public var reservedCodeCacheSize: String = "",
public var jvmParams: MutableCollection<String> = arrayListOf()
) : OptionsGroup {
override val mappers: List<PropMapper<*,*,*>>
get() = listOf( StringPropMapper(this, ::maxMemory, listOf("Xmx"), mergeDelimiter = ""),
StringPropMapper(this, ::maxPermSize, listOf("XX:MaxPermSize"), mergeDelimiter = "="),
StringPropMapper(this, ::reservedCodeCacheSize, listOf("XX:ReservedCodeCacheSize"), mergeDelimiter = "="),
restMapper)
val restMapper: RestPropMapper<*,*>
get() = RestPropMapper(this, ::jvmParams)
}
public data class DaemonOptions(
public var port: Int = COMPILE_DAEMON_DEFAULT_PORT,
public var autoshutdownMemoryThreshold: Long = COMPILE_DAEMON_MEMORY_THRESHOLD_INFINITE,
public var autoshutdownIdleSeconds: Int = COMPILE_DAEMON_TIMEOUT_INFINITE_S,
public var startEcho: String = COMPILER_SERVICE_RMI_NAME
) : OptionsGroup {
override val mappers: List<PropMapper<*, *, *>>
get() = listOf( PropMapper(this, ::port, fromString = { it.toInt() }),
PropMapper(this, ::autoshutdownMemoryThreshold, fromString = { it.toLong() }, skipIf = { it == 0L }),
PropMapper(this, ::autoshutdownIdleSeconds, fromString = { it.toInt() }, skipIf = { it == 0 }),
PropMapper(this, ::startEcho, fromString = { it.trim('"') }))
}
fun updateSingleFileDigest(file: File, md: MessageDigest) {
DigestInputStream(file.inputStream(), md).use {
val buf = ByteArray(1024)
while (it.read(buf) != -1) { }
it.close()
}
}
fun updateForAllClasses(dir: File, md: MessageDigest) {
dir.walk().forEach { updateEntryDigest(it, md) }
}
fun updateEntryDigest(entry: File, md: MessageDigest) {
when {
entry.isDirectory
-> updateForAllClasses(entry, md)
entry.isFile &&
(entry.getName().endsWith(".class", ignoreCase = true) ||
entry.getName().endsWith(".jar", ignoreCase = true))
-> updateSingleFileDigest(entry, md)
// else skip
}
}
fun Iterable<File>.getFilesClasspathDigest(): String {
val md = MessageDigest.getInstance(COMPILER_ID_DIGEST)
this.forEach { updateEntryDigest(it, md) }
return md.digest().joinToString("", transform = { "%02x".format(it) })
}
fun Iterable<String>.getClasspathDigest(): String = map { File(it) }.getFilesClasspathDigest()
public data class CompilerId(
public var compilerClasspath: List<String> = listOf(),
public var compilerDigest: String = "",
public var compilerVersion: String = ""
// TODO: checksum
) : OptionsGroup {
override val mappers: List<PropMapper<*, *, *>>
get() = listOf( PropMapper(this, ::compilerClasspath, toString = { it.joinToString(File.pathSeparator) }, fromString = { it.trim('"').split(File.pathSeparator)}),
StringPropMapper(this, ::compilerDigest),
StringPropMapper(this, ::compilerVersion))
public fun updateDigest() {
compilerDigest = compilerClasspath.getClasspathDigest()
}
companion object {
public jvmStatic fun makeCompilerId(vararg paths: File): CompilerId = makeCompilerId(paths.asIterable())
public jvmStatic fun makeCompilerId(paths: Iterable<File>): CompilerId =
// TODO consider reading version here
CompilerId(compilerClasspath = paths.map { it.absolutePath }, compilerDigest = paths.getFilesClasspathDigest())
}
}
public fun isDaemonEnabled(): Boolean = System.getProperty(COMPILE_DAEMON_ENABLED_PROPERTY) != null
public fun configureDaemonLaunchingOptions(opts: DaemonJVMOptions, inheritMemoryLimits: Boolean): DaemonJVMOptions {
// note: sequence matters, explicit override in COMPILE_DAEMON_JVM_OPTIONS_PROPERTY should be done after inputArguments processing
if (inheritMemoryLimits)
ManagementFactory.getRuntimeMXBean().inputArguments.filterExtractProps(opts.mappers, "-")
System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)?.let {
opts.jvmParams.addAll(it.trim('"', '\'').split(",").filterExtractProps(opts.mappers, "-", opts.restMapper))
}
return opts
}
public fun configureDaemonLaunchingOptions(inheritMemoryLimits: Boolean): DaemonJVMOptions =
configureDaemonLaunchingOptions(DaemonJVMOptions(), inheritMemoryLimits = inheritMemoryLimits)
jvmOverloads public fun configureDaemonOptions(opts: DaemonOptions = DaemonOptions()): DaemonOptions {
System.getProperty(COMPILE_DAEMON_OPTIONS_PROPERTY)?.let {
val unrecognized = it.trim('"', '\'').split(",").filterExtractProps(opts.mappers, "")
if (unrecognized.any())
throw IllegalArgumentException(
"Unrecognized daemon options passed via property $COMPILE_DAEMON_OPTIONS_PROPERTY: " + unrecognized.joinToString(" ") +
"\nSupported options: " + opts.mappers.joinToString(", ", transform = { it.names.first() }))
}
return opts
}
@@ -0,0 +1,33 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.rmi
import java.io.IOException
import java.rmi.Remote
import java.rmi.RemoteException
public interface RemoteOutputStream : Remote {
throws(IOException::class, RemoteException::class)
public fun close()
throws(IOException::class, RemoteException::class)
public fun write(data: ByteArray, offset: Int, length: Int)
throws(IOException::class, RemoteException::class)
public fun write(dataByte: Int)
}
+16
View File
@@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="kotlin-runtime" level="project" />
<orderEntry type="module" module-name="util" />
<orderEntry type="module" module-name="frontend.java" />
<orderEntry type="module" module-name="cli" />
<orderEntry type="module" module-name="rmi-interface" />
</component>
</module>
@@ -0,0 +1,129 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.rmi.service
import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
import org.jetbrains.kotlin.rmi.*
import org.jetbrains.kotlin.service.CompileServiceImpl
import java.io.OutputStream
import java.io.PrintStream
import java.rmi.RMISecurityManager
import java.rmi.registry.LocateRegistry
import java.text.SimpleDateFormat
import java.util.*
import java.util.logging.LogManager
import java.util.logging.Logger
import kotlin.platform.platformStatic
class LogStream(name: String) : OutputStream() {
val log by lazy { Logger.getLogger(name) }
val lineBuf = StringBuilder()
override fun write(byte: Int) {
if (byte.toChar() == '\n') flush()
else lineBuf.append(byte.toChar())
}
override fun write(data: ByteArray, offset: Int, length: Int) {
var ofs = offset
var lineStart = ofs
while (ofs < length) {
if (data[ofs].toChar() == '\n') {
flush(data, lineStart, ofs - lineStart)
lineStart = ofs + 1
}
ofs++
}
if (lineStart < length)
lineBuf.append(data, lineStart, length - lineStart)
}
fun flush(data: ByteArray, offset: Int, length: Int) {
log.info(lineBuf.toString() + data.toString().substring(offset, length))
lineBuf.setLength(0)
}
override fun flush() {
log.info(lineBuf.toString())
lineBuf.setLength(0)
}
}
public class CompileDaemon {
companion object {
init {
val logPath: String = System.getProperty("kotlin.daemon.log.path")?.trimEnd('/','\\') ?: "%t"
val logTime: String = SimpleDateFormat("yyyy-MM-dd.HH-mm-ss-SSS").format(Date())
val cfg: String =
"handlers = java.util.logging.FileHandler\n" +
"java.util.logging.FileHandler.level = ALL\n" +
"java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter\n" +
"java.util.logging.FileHandler.encoding = UTF-8\n" +
"java.util.logging.FileHandler.limit = 1073741824\n" + // 1Mb
"java.util.logging.FileHandler.count = 3\n" +
"java.util.logging.FileHandler.append = false\n" +
"java.util.logging.FileHandler.pattern = $logPath/kotlin-daemon.$logTime.%g.log\n" +
"java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s\\n\n"
LogManager.getLogManager().readConfiguration(cfg.byteInputStream())
}
val log by lazy { Logger.getLogger("daemon") }
platformStatic public fun main(args: Array<String>) {
val compilerId = CompilerId()
val daemonOptions = DaemonOptions()
val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX)
if (filteredArgs.any()) {
val helpLine = "usage: <daemon> <compilerId options> <daemon options>"
log.info(helpLine)
println(helpLine)
throw IllegalArgumentException("Unknown arguments")
}
log.info("starting daemon")
// TODO: find minimal set of permissions and restore security management
// if (System.getSecurityManager() == null)
// System.setSecurityManager (RMISecurityManager())
//
// setDaemonPpermissions(daemonOptions.port)
val registry = LocateRegistry.createRegistry(daemonOptions.port);
val compiler = K2JVMCompiler()
CompileServiceImpl(registry, compiler, compilerId, daemonOptions)
if (daemonOptions.startEcho.isNotEmpty())
println(daemonOptions.startEcho)
// this stops redirected streams reader(s) on the client side and prevent some situations with hanging threads
System.out.close()
System.err.close()
System.setErr(PrintStream(LogStream("stderr")))
System.setOut(PrintStream(LogStream("stdout")))
}
}
}
@@ -0,0 +1,203 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.service
import org.jetbrains.kotlin.cli.common.CLICompiler
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
import org.jetbrains.kotlin.rmi.*
import org.jetbrains.kotlin.rmi.service.RemoteIncrementalCacheClient
import org.jetbrains.kotlin.rmi.service.RemoteOutputStreamClient
import java.io.IOException
import java.io.PrintStream
import java.lang.management.ManagementFactory
import java.net.URLClassLoader
import java.rmi.registry.Registry
import java.rmi.server.UnicastRemoteObject
import java.util.concurrent.TimeUnit
import java.util.concurrent.locks.ReentrantReadWriteLock
import java.util.jar.Manifest
import java.util.logging.Logger
import kotlin.concurrent.read
import kotlin.concurrent.write
class CompileServiceImpl<Compiler: CLICompiler<*>>(
val registry: Registry,
val compiler: Compiler,
val selfCompilerId: CompilerId,
val daemonOptions: DaemonOptions
) : CompileService, UnicastRemoteObject() {
val log by lazy { Logger.getLogger("compiler") }
private val rwlock = ReentrantReadWriteLock()
private var alive = false
// TODO: consider matching compilerId coming from outside with actual one
// private val selfCompilerId by lazy {
// CompilerId(
// compilerClasspath = System.getProperty("java.class.path")
// ?.split(File.pathSeparator)
// ?.map { File(it) }
// ?.filter { it.exists() }
// ?.map { it.absolutePath }
// ?: listOf(),
// compilerVersion = loadKotlinVersionFromResource()
// )
// }
init {
// assuming logically synchronized
try {
// cleanup for the case of incorrect restart
UnicastRemoteObject.unexportObject(this, false)
}
catch (e: java.rmi.NoSuchObjectException) {
// ignoring if object already exported
}
val stub = UnicastRemoteObject.exportObject(this, 0) as CompileService
// TODO: use version-specific name
registry.rebind (COMPILER_SERVICE_RMI_NAME, stub);
alive = true
}
public class IncrementalCompilationComponentsImpl(val idToCache: Map<String, CompileService.RemoteIncrementalCache>): IncrementalCompilationComponents {
// perf: cheap object, but still the pattern may be costly if there are too many calls to cache with the same id (which seems not to be the case now)
override fun getIncrementalCache(moduleId: String): IncrementalCache = RemoteIncrementalCacheClient(idToCache[moduleId]!!)
// TODO: add appropriate proxy into interaction when lookup tracker is needed
override fun getLookupTracker(): LookupTracker = LookupTracker.DO_NOTHING
}
private fun createCompileServices(incrementalCaches: Map<String, CompileService.RemoteIncrementalCache>): Services =
Services.Builder()
.register(IncrementalCompilationComponents::class.java, IncrementalCompilationComponentsImpl(incrementalCaches))
// TODO: add remote proxy for cancellation status tracking
// .register(javaClass<CompilationCanceledStatus>(), object: CompilationCanceledStatus {
// override fun checkCanceled(): Unit = if (context.getCancelStatus().isCanceled()) throw CompilationCanceledException()
// })
.build()
fun usedMemory(): Long {
System.gc()
val rt = Runtime.getRuntime()
return (rt.totalMemory() - rt.freeMemory())
}
fun usedMemoryMX(): Long {
System.gc()
val memoryMXBean= ManagementFactory.getMemoryMXBean()
val memHeap=memoryMXBean.getHeapMemoryUsage()
return memHeap.used
}
// TODO: consider using version as a part of compiler ID or drop this function
private fun loadKotlinVersionFromResource(): String {
(javaClass.classLoader as? URLClassLoader)
?.findResource("META-INF/MANIFEST.MF")
?.let {
try {
return Manifest(it.openStream()).mainAttributes.getValue("Implementation-Version") ?: ""
}
catch (e: IOException) {}
}
return ""
}
fun<R> checkedCompile(args: Array<out String>, body: () -> R): R {
try {
if (args.none())
throw IllegalArgumentException("Error: empty arguments list.")
log.info("Starting compilation with args: " + args.joinToString(" "))
val startMemMX = usedMemoryMX() / 1024
val startMem = usedMemory() / 1024
val startTime = System.nanoTime()
val res = body()
val endTime = System.nanoTime()
val endMem = usedMemory() / 1024
val endMemMX = usedMemoryMX() / 1024
log.info("Done with result " + res.toString())
log.info("Elapsed time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms")
log.info("Used memory: $endMem kb (${"%+d".format(endMem - startMem)} kb)")
log.info("Used memory (from MemoryMXBean): $endMemMX kb (${"%+d".format(endMemMX - startMemMX)} kb)")
return res
}
catch (e: Exception) {
log.info("Error: $e")
throw e
}
}
fun<R> ifAlive(body: () -> R): R = rwlock.read {
if (!alive) throw IllegalStateException("Kotlin Compiler Service is not in alive state")
else body()
}
fun<R> ifAliveExclusive(body: () -> R): R = rwlock.write {
if (!alive) throw IllegalStateException("Kotlin Compiler Service is not in alive state")
else body()
}
// sometimes used for debugging
fun<R> spy(msg: String, body: () -> R): R {
val res = body()
log.info(msg + " = " + res.toString())
return res
}
override fun getCompilerId(): CompilerId = ifAlive { selfCompilerId }
override fun getUsedMemory(): Long = ifAlive { usedMemory() }
override fun shutdown() {
ifAliveExclusive {
log.info("Shutdown started")
alive = false
UnicastRemoteObject.unexportObject(this, true)
log.info("Shutdown complete")
}
}
override fun remoteCompile(args: Array<out String>, errStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int =
doCompile(args, errStream) { printStream ->
when (outputFormat) {
CompileService.OutputFormat.PLAIN -> compiler.exec(printStream, *args)
CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStream, Services.EMPTY, *args)
}
}
override fun remoteIncrementalCompile(args: Array<out String>, caches: Map<String, CompileService.RemoteIncrementalCache>, outputStream: RemoteOutputStream, outputFormat: CompileService.OutputFormat): Int =
doCompile(args, outputStream) { printStream ->
when (outputFormat) {
CompileService.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation")
CompileService.OutputFormat.XML -> compiler.execAndOutputXml(printStream, createCompileServices(caches), *args)
}
}
fun doCompile(args: Array<out String>, errStream: RemoteOutputStream, body: (PrintStream) -> ExitCode): Int =
ifAlive {
checkedCompile(args) {
body( PrintStream( RemoteOutputStreamClient(errStream))).code
}
}
}
@@ -0,0 +1,79 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.rmi.service
import java.io.FilePermission
import java.net.SocketPermission
import java.security.CodeSource
import java.security.Permission
import java.security.PermissionCollection
import java.security.Policy
import java.util.ArrayList
import java.util.Collections
import java.util.Enumeration
import java.util.PropertyPermission
public class DaemonPolicy(val port: Int) : Policy() {
private fun createPermissions(): PermissionCollection {
val perms = DaemonPermissionCollection()
val socketPermission = SocketPermission("localhost:$port-", "connect, accept, resolve")
val propertyPermission = PropertyPermission("localhost:$port", "read")
//val filePermission = FilePermission("<<ALL FILES>>", "read")
perms.add(socketPermission)
perms.add(propertyPermission)
//perms.add(filePermission)
return perms
}
private val perms: PermissionCollection by lazy { createPermissions() }
override fun getPermissions(codesource: CodeSource?): PermissionCollection {
return perms
}
}
class DaemonPermissionCollection : PermissionCollection() {
val perms = ArrayList<Permission>()
override fun add(p: Permission) {
perms.add(p)
}
override fun implies(p: Permission): Boolean = perms.any { implies(p) }
override fun elements(): Enumeration<Permission> = Collections.enumeration(perms)
override fun isReadOnly(): Boolean = false
//
// companion object {
//
// private val serialVersionUID = 614300921365729272L
// }
}
public fun setDaemonPpermissions(port: Int) {
Policy.setPolicy(DaemonPolicy(port))
}
@@ -0,0 +1,28 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.rmi.service
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
import org.jetbrains.kotlin.rmi.CompileService
public class RemoteIncrementalCacheClient(val cache: CompileService.RemoteIncrementalCache): IncrementalCache {
override fun getObsoletePackageParts(): Collection<String> = cache.getObsoletePackageParts()
override fun getPackageData(fqName: String): ByteArray? = cache.getPackageData(fqName)
override fun close(): Unit = cache.close()
}
@@ -0,0 +1,34 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.rmi.service
import org.jetbrains.kotlin.rmi.RemoteOutputStream
import java.io.OutputStream
class RemoteOutputStreamClient(val remote: RemoteOutputStream): OutputStream() {
override fun write(data: ByteArray) {
remote.write(data, 0, data.size())
}
override fun write(data: ByteArray, offset: Int, length: Int) {
remote.write(data, offset, length)
}
override fun write(byte: Int) {
remote.write(byte)
}
}
+2
View File
@@ -26,5 +26,7 @@
<orderEntry type="module" module-name="js.serializer" />
<orderEntry type="module" module-name="js.translator" scope="TEST" />
<orderEntry type="module" module-name="util" />
<orderEntry type="module" module-name="kotlinr" />
<orderEntry type="module" module-name="rmi-interface" />
</component>
</module>
@@ -0,0 +1,76 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.daemon
import junit.framework.TestCase
import org.jetbrains.kotlin.cli.CliBaseTest
import org.jetbrains.kotlin.integration.KotlinIntegrationTestBase
import org.jetbrains.kotlin.rmi.CompilerId
import org.jetbrains.kotlin.rmi.DaemonJVMOptions
import org.jetbrains.kotlin.rmi.DaemonOptions
import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient
import org.jetbrains.kotlin.test.JetTestUtils
import java.io.ByteArrayOutputStream
import java.io.File
val KOTLIN_DAEMON_TEST_PORT = 19753
public class CompilerDaemonTest : KotlinIntegrationTestBase() {
data class CompilerResults(val resultCode: Int, val out: String)
val daemonOptions = DaemonOptions(port = KOTLIN_DAEMON_TEST_PORT)
val daemonLaunchingOptions = DaemonJVMOptions()
val compilerId by lazy { CompilerId.makeCompilerId( File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-compiler.jar"),
File("dependencies/bootstrap-compiler/Kotlin/kotlinc/lib/kotlin-runtime.jar"),
File("dependencies/bootstrap-compiler/Kotlin/kotlinc/lib/kotlin-reflect.jar")) }
private fun compileOnDaemon(args: Array<out String>): CompilerResults {
val daemon = KotlinCompilerClient.connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, System.err, autostart = true, checkId = true)
TestCase.assertNotNull("failed to connect daemon", daemon)
val strm = ByteArrayOutputStream()
val code = KotlinCompilerClient.compile(daemon!!, args, strm)
return CompilerResults(code, strm.toString())
}
private fun runDaemonCompilerTwice(logName: String, vararg arguments: String): Unit {
KotlinCompilerClient.shutdownCompileService(daemonOptions)
try {
val res1 = compileOnDaemon(arguments)
TestCase.assertEquals("first compilation failed:\n${res1.out}", 0, res1.resultCode)
val res2 = compileOnDaemon(arguments)
TestCase.assertEquals("second compilation failed:\n${res2.out}", 0, res2.resultCode)
TestCase.assertEquals("build results differ", CliBaseTest.removePerfOutput(res1.out), CliBaseTest.removePerfOutput(res2.out))
// TODO: add performance comparison assert
}
finally {
KotlinCompilerClient.shutdownCompileService(daemonOptions)
}
}
private fun getTestBaseDir(): String = JetTestUtils.getTestDataPathBase() + "/integration/smoke/" + getTestName(true)
private fun run(logName: String, vararg args: String): Int = runJava(getTestBaseDir(), logName, *args)
public fun testHelloApp() {
val jar = tmpdir.absolutePath + File.separator + "hello.jar"
runDaemonCompilerTwice("hello.compile", "-include-runtime", File(getTestBaseDir(), "hello.kt").absolutePath, "-d", jar)
run("hello.run", "-cp", jar, "Hello.HelloPackage")
}
}
@@ -133,7 +133,7 @@ public abstract class KotlinIntegrationTestBase extends TestCaseWithTmpdir {
return runtime;
}
protected static File getCompilerLib() {
public static File getCompilerLib() {
File file = PathUtil.getKotlinPathsForDistDirectory().getLibPath().getAbsoluteFile();
assertTrue("Lib directory doesn't exist. Run 'ant dist'", file.isDirectory());
return file;
+1
View File
@@ -50,5 +50,6 @@
<orderEntry type="module" module-name="idea-test-framework" scope="TEST" />
<orderEntry type="module" module-name="js.serializer" />
<orderEntry type="module" module-name="idea-js" scope="TEST" />
<orderEntry type="module" module-name="kotlinr" />
</component>
</module>
@@ -23,6 +23,7 @@ import org.jetbrains.kotlin.idea.caches.JarUserDataManager;
import org.jetbrains.kotlin.idea.debugger.filter.FilterPackage;
import org.jetbrains.kotlin.idea.decompiler.HasCompiledKotlinInJar;
import org.jetbrains.kotlin.idea.framework.KotlinJavaScriptLibraryDetectionUtil;
import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient;
import org.jetbrains.kotlin.utils.PathUtil;
public class PluginStartupComponent implements ApplicationComponent {
@@ -51,5 +52,6 @@ public class PluginStartupComponent implements ApplicationComponent {
@Override
public void disposeComponent() {
KotlinCompilerClient.Companion.shutdownCompileService();
}
}
+2
View File
@@ -21,5 +21,7 @@
<orderEntry type="library" name="asm" level="project" />
<orderEntry type="module" module-name="util" />
<orderEntry type="module" module-name="preloader" />
<orderEntry type="module" module-name="kotlinr" />
<orderEntry type="module" module-name="rmi-interface" />
</component>
</module>
@@ -56,7 +56,7 @@ public class CompilerRunnerUtil {
}
@Nullable
private static File getLibPath(@NotNull KotlinPaths paths, @NotNull MessageCollector messageCollector) {
public static File getLibPath(@NotNull KotlinPaths paths, @NotNull MessageCollector messageCollector) {
File libs = paths.getLibPath();
if (libs.exists() && !libs.isFile()) return libs;
@@ -30,6 +30,9 @@ import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments;
import org.jetbrains.kotlin.cli.common.messages.MessageCollector;
import org.jetbrains.kotlin.cli.common.messages.MessageCollectorUtil;
import org.jetbrains.kotlin.config.CompilerSettings;
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache;
import org.jetbrains.kotlin.rmi.*;
import org.jetbrains.kotlin.rmi.kotlinr.KotlinCompilerClient;
import org.jetbrains.kotlin.utils.UtilsPackage;
import java.io.*;
@@ -38,6 +41,7 @@ import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import static org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation.NO_LOCATION;
import static org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR;
@@ -54,13 +58,15 @@ public class KotlinCompilerRunner {
CompilerSettings compilerSettings,
MessageCollector messageCollector,
CompilerEnvironment environment,
Map<String, IncrementalCache> incrementalCaches,
File moduleFile,
OutputItemsCollector collector
) {
K2JVMCompilerArguments arguments = mergeBeans(commonArguments, k2jvmArguments);
setupK2JvmArguments(moduleFile, arguments);
runCompiler(K2JVM_COMPILER, arguments, compilerSettings.getAdditionalArguments(), messageCollector, collector, environment);
runCompiler(K2JVM_COMPILER, arguments, compilerSettings.getAdditionalArguments(), messageCollector, collector, environment,
incrementalCaches);
}
public static void runK2JsCompiler(
@@ -69,6 +75,7 @@ public class KotlinCompilerRunner {
@NotNull CompilerSettings compilerSettings,
@NotNull MessageCollector messageCollector,
@NotNull CompilerEnvironment environment,
Map<String, IncrementalCache> incrementalCaches,
@NotNull OutputItemsCollector collector,
@NotNull Collection<File> sourceFiles,
@NotNull List<String> libraryFiles,
@@ -77,7 +84,8 @@ public class KotlinCompilerRunner {
K2JSCompilerArguments arguments = mergeBeans(commonArguments, k2jsArguments);
setupK2JsArguments(outputFile, sourceFiles, libraryFiles, arguments);
runCompiler(K2JS_COMPILER, arguments, compilerSettings.getAdditionalArguments(), messageCollector, collector, environment);
runCompiler(K2JS_COMPILER, arguments, compilerSettings.getAdditionalArguments(), messageCollector, collector, environment,
incrementalCaches);
}
private static void runCompiler(
@@ -86,12 +94,13 @@ public class KotlinCompilerRunner {
String additionalArguments,
MessageCollector messageCollector,
OutputItemsCollector collector,
CompilerEnvironment environment
CompilerEnvironment environment,
Map<String, IncrementalCache> incrementalCaches
) {
ByteArrayOutputStream stream = new ByteArrayOutputStream();
PrintStream out = new PrintStream(stream);
String exitCode = execCompiler(compilerClassName, arguments, additionalArguments, environment, out, messageCollector);
String exitCode = execCompiler(compilerClassName, arguments, additionalArguments, environment, incrementalCaches, out, messageCollector);
BufferedReader reader = new BufferedReader(new StringReader(stream.toString()));
CompilerOutputParser.parseCompilerMessagesFromReader(messageCollector, reader, collector);
@@ -107,6 +116,7 @@ public class KotlinCompilerRunner {
CommonCompilerArguments arguments,
String additionalArguments,
CompilerEnvironment environment,
Map<String, IncrementalCache> incrementalCaches,
PrintStream out,
MessageCollector messageCollector
) {
@@ -116,8 +126,28 @@ public class KotlinCompilerRunner {
List<String> argumentsList = ArgumentUtils.convertArgumentsToStringList(arguments);
argumentsList.addAll(StringUtil.split(additionalArguments, " "));
String[] argsArray = ArrayUtil.toStringArray(argumentsList);
// trying the daemon first
if (incrementalCaches != null && RmiPackage.isDaemonEnabled()) {
File libPath = CompilerRunnerUtil.getLibPath(environment.getKotlinPaths(), messageCollector);
// TODO: it may be a good idea to cache the compilerId, since making it means calculating digest over jar(s) and if \\
// the lifetime of JPS process is small anyway, we can neglect the probability of changed compiler
CompilerId compilerId = CompilerId.makeCompilerId(new File(libPath, "kotlin-compiler.jar"));
DaemonOptions daemonOptions = RmiPackage.configureDaemonOptions();
DaemonJVMOptions daemonJVMOptions = RmiPackage.configureDaemonLaunchingOptions(true);
// TODO: find proper stream to report daemon connection progress
CompileService daemon = KotlinCompilerClient.Companion.connectToCompileService(compilerId, daemonJVMOptions, daemonOptions, System.out, true, true);
if (daemon != null) {
Integer res = KotlinCompilerClient.Companion.incrementalCompile(daemon, argsArray, incrementalCaches, out);
return res.toString();
}
}
// otherwise fallback to in-process
Object rc = CompilerRunnerUtil.invokeExecMethod(
compilerClassName, ArrayUtil.toStringArray(argumentsList), environment, messageCollector, out
compilerClassName, argsArray, environment, messageCollector, out
);
// exec() returns an ExitCode object, class of which is loaded with a different class loader,
@@ -247,9 +247,10 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
filesToCompile: MultiMap<ModuleBuildTarget, File>, incrementalCaches: Map<ModuleBuildTarget, IncrementalCacheImpl>,
messageCollector: MessageCollectorAdapter, project: JpsProject
): OutputItemsCollectorImpl? {
if (JpsUtils.isJsKotlinModule(chunk.representativeTarget())) {
LOG.debug("Compiling to JS ${filesToCompile.values().size()} files in " + filesToCompile.keySet().map { it.getPresentableName() }.join())
return compileToJs(chunk, commonArguments, environment, messageCollector, project)
return compileToJs(chunk, commonArguments, environment, null, messageCollector, project)
}
if (IncrementalCompilation.ENABLED) {
@@ -279,7 +280,9 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
)
}
return compileToJvm(allCompiledFiles, chunk, commonArguments, context, dirtyFilesHolder, environment, filesToCompile, messageCollector)
return compileToJvm(allCompiledFiles, chunk, commonArguments, context, dirtyFilesHolder, environment,
incrementalCaches.mapKeysTo(HashMap<String, IncrementalCache>(incrementalCaches.size()), { it.getKey().id }),
filesToCompile, messageCollector)
}
private fun createCompileEnvironment(
@@ -436,8 +439,8 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
private fun compileToJs(chunk: ModuleChunk,
commonArguments: CommonCompilerArguments,
environment: CompilerEnvironment,
messageCollector: KotlinBuilder.MessageCollectorAdapter,
project: JpsProject
incrementalCaches: MutableMap<String, IncrementalCache>?,
messageCollector: MessageCollectorAdapter, project: JpsProject
): OutputItemsCollectorImpl? {
val outputItemCollector = OutputItemsCollectorImpl()
@@ -468,7 +471,7 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
val compilerSettings = JpsKotlinCompilerSettings.getCompilerSettings(project)
val k2JsArguments = JpsKotlinCompilerSettings.getK2JsCompilerArguments(project)
runK2JsCompiler(commonArguments, k2JsArguments, compilerSettings, messageCollector, environment, outputItemCollector, sourceFiles, libraryFiles, outputFile)
runK2JsCompiler(commonArguments, k2JsArguments, compilerSettings, messageCollector, environment, incrementalCaches, outputItemCollector, sourceFiles, libraryFiles, outputFile)
return outputItemCollector
}
@@ -491,8 +494,8 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
context: CompileContext,
dirtyFilesHolder: DirtyFilesHolder<JavaSourceRootDescriptor, ModuleBuildTarget>,
environment: CompilerEnvironment,
filesToCompile: MultiMap<ModuleBuildTarget, File>,
messageCollector: KotlinBuilder.MessageCollectorAdapter
incrementalCaches: MutableMap<String, IncrementalCache>?,
filesToCompile: MultiMap<ModuleBuildTarget, File>, messageCollector: MessageCollectorAdapter
): OutputItemsCollectorImpl? {
val outputItemCollector = OutputItemsCollectorImpl()
@@ -535,7 +538,7 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR
+ (if (totalRemovedFiles == 0) "" else " ($totalRemovedFiles removed files)")
+ " in " + filesToCompile.keySet().map { it.getPresentableName() }.join())
runK2JvmCompiler(commonArguments, k2JvmArguments, compilerSettings, messageCollector, environment, moduleFile, outputItemCollector)
runK2JvmCompiler(commonArguments, k2JvmArguments, compilerSettings, messageCollector, environment, incrementalCaches, moduleFile, outputItemCollector)
moduleFile.delete()
return outputItemCollector