Move daemon jar from compiler

This commit is contained in:
Vadim Brilyantov
2019-04-03 19:34:06 +03:00
parent ced973b707
commit 01a05a5495
67 changed files with 441 additions and 759 deletions
+31 -12
View File
@@ -1,28 +1,36 @@
import com.sun.javafx.scene.CameraHelper.project
import org.gradle.internal.impldep.org.junit.experimental.categories.Categories.CategoryFilter.exclude
import org.jetbrains.kotlin.gradle.dsl.Coroutines
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
description = "Kotlin Daemon New"
plugins {
kotlin("jvm")
id("jps-compatible")
}
jvmTarget = "1.8"
val ktorExcludesForDaemon : List<Pair<String, String>> by rootProject.extra
dependencies {
compile(project(":compiler:cli"))
compile(project(":compiler:cli-js"))
compile(project(":compiler:daemon-common"))
compile(project(":compiler:daemon-common-new"))
compile(project(":daemon-common-new"))
compile(project(":compiler:incremental-compilation-impl"))
compile(project(":kotlin-build-common"))
compile(commonDep("org.fusesource.jansi", "jansi"))
compile(commonDep("org.jline", "jline"))
compileOnly(intellijCoreDep()) { includeJars("intellij-core") }
compileOnly(intellijDep()) { includeIntellijCoreJarDependencies(project) }
runtime(project(":kotlin-reflect"))
compileOnly(project(":kotlin-reflect-api"))
compile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-jdk8")) { isTransitive = false }
embedded(project(":daemon-common")) { isTransitive = false }
embedded(project(":daemon-common-new")) { isTransitive = false }
compile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-core")) {
isTransitive = false
}
compile(commonDep("io.ktor", "ktor-network")) {
ktorExcludesForDaemon.forEach { (group, module) ->
exclude(group = group, module = module)
@@ -34,6 +42,17 @@ sourceSets {
"main" { projectDefault() }
"test" {}
}
kotlin {
experimental.coroutines = Coroutines.ENABLE
}
publish()
noDefaultJar()
runtimeJar(task<ShadowJar>("shadowJar")) {
from(mainSourceSet.output)
}
sourcesJar()
javadocJar()
dist()
@@ -32,20 +32,20 @@ val ktorExcludesForDaemon : List<Pair<String, String>> by rootProject.extra
dependencies {
compileOnly(project(":compiler:util"))
compileOnly(project(":compiler:cli-common"))
compileOnly(project(":compiler:daemon-common-new"))
compileOnly(project(":daemon-common-new"))
compileOnly(project(":kotlin-reflect-api"))
compileOnly(project(":kotlin-daemon-client"))
embeddedComponents(project(":kotlin-daemon-client")) { isTransitive = false }
embedded(project(":kotlin-daemon-client")) { isTransitive = false }
compileOnly(project(":js:js.frontend"))
compileOnly(commonDep("net.rubygrapefruit", "native-platform"))
compileOnly(intellijDep()) { includeIntellijCoreJarDependencies(project) }
embeddedComponents(project(":compiler:daemon-common")) { isTransitive = false }
embeddedComponents(commonDep("net.rubygrapefruit", "native-platform"))
embedded(project(":daemon-common")) { isTransitive = false }
embedded(commonDep("net.rubygrapefruit", "native-platform"))
nativePlatformVariants.forEach {
embeddedComponents(commonDep("net.rubygrapefruit", "native-platform", "-$it"))
embedded(commonDep("net.rubygrapefruit", "native-platform", "-$it"))
}
compile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-jdk8")) {
compile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-core")) {
isTransitive = false
}
compile(commonDep("io.ktor", "ktor-network")) {
@@ -66,7 +66,6 @@ noDefaultJar()
runtimeJar(task<ShadowJar>("shadowJar")) {
from(mainSourceSet.output)
fromEmbeddedComponents()
}
sourcesJar()
@@ -74,5 +73,3 @@ sourcesJar()
javadocJar()
dist()
ideaPlugin()
@@ -45,7 +45,6 @@ class KotlinCompilerClient : KotlinCompilerDaemonClient {
val verboseReporting = System.getProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) != null
private val log = Logger.getLogger("KotlinCompilerClient")
private fun String.info(msg: String) = {}()//log.info("[$this] : $msg")
override fun getOrCreateClientFlagFile(daemonOptions: DaemonOptions): File =
// for jps property is passed from IDEA to JPS in KotlinBuildProcessParametersProvider
@@ -64,7 +63,6 @@ class KotlinCompilerClient : KotlinCompilerDaemonClient {
autostart: Boolean,
checkId: Boolean
): CompileServiceAsync? {
log.info("in connectToCompileService")
val flagFile = getOrCreateClientFlagFile(daemonOptions)
return connectToCompileService(
compilerId,
@@ -84,7 +82,6 @@ class KotlinCompilerClient : KotlinCompilerDaemonClient {
reportingTargets: DaemonReportingTargets,
autostart: Boolean
): CompileServiceAsync? {
log.info("connectToCompileService")
return connectAndLease(
compilerId,
clientAliveFlagFile,
@@ -112,14 +109,9 @@ class KotlinCompilerClient : KotlinCompilerDaemonClient {
reportingTargets,
autostart
) { isLastAttempt ->
log.info("connectAndLease")
fun CompileServiceAsync.leaseImpl(): Deferred<CompileServiceSessionAsync?> =
GlobalScope.async {
// the newJVMOptions could be checked here for additional parameters, if needed
log.info("trying registerClient")
println("trying registerClient")
try {
registerClient(clientAliveFlagFile.absolutePath)
} catch (e: Throwable) {
@@ -149,7 +141,6 @@ class KotlinCompilerClient : KotlinCompilerDaemonClient {
service.leaseImpl().await()
} else {
if (!isLastAttempt && autostart) {
log.info("starting daemon...")
if (startDaemon(compilerId, newJVMOptions, daemonOptions, reportingTargets)) {
reportingTargets.report(DaemonReportCategory.DEBUG, "new daemon started, trying to find it")
}
@@ -189,7 +180,6 @@ class KotlinCompilerClient : KotlinCompilerDaemonClient {
port: Int,
profiler: Profiler
): Int = profiler.withMeasure(this) {
log.info("Compile")
val services = BasicCompilerServicesWithResultsFacadeServerServerSide(
messageCollector,
outputsCollector,
@@ -431,24 +421,13 @@ class KotlinCompilerClient : KotlinCompilerDaemonClient {
daemonJVMOptions: DaemonJVMOptions,
report: (DaemonReportCategory, String) -> Unit
): Deferred<Pair<CompileServiceAsync?, DaemonJVMOptions>> = GlobalScope.async {
log.info("tryFindSuitableDaemonOrNewOpts")
registryDir.mkdirs()
val timestampMarker = createTempFile("kotlin-daemon-client-tsmarker", directory = registryDir)
val aliveWithMetadata = try {
log.info("walkDaemonsAsync... : ${registryDir.path}")
walkDaemonsAsync(registryDir, compilerId, timestampMarker, report = report).also {
log.info(
"daemons (${it.size}): ${it.map { "daemon(params : " + it.jvmOptions.jvmParams.joinToString(", ") + ")" }.joinToString(
", ", "[", "]"
)}"
)
}
walkDaemonsAsync(registryDir, compilerId, timestampMarker, report = report)
} finally {
timestampMarker.delete()
}
log.info("daemons : ${aliveWithMetadata.map { it.daemon::class.java.name }}")
log.info("aliveWithMetadata: ${aliveWithMetadata.map { it.daemon::class.java.name }}")
val comparator = compareBy<DaemonWithMetadataAsync, DaemonJVMOptions>(DaemonJVMOptionsMemoryComparator(), { it.jvmOptions })
.thenBy {
when (it.daemon) {
@@ -475,17 +454,13 @@ class KotlinCompilerClient : KotlinCompilerDaemonClient {
daemonOptions: DaemonOptions,
reportingTargets: DaemonReportingTargets
): Boolean {
log.info("in startDaemon() - 0")
val javaExecutable = File(File(System.getProperty("java.home"), "bin"), "java")
log.info("in startDaemon() - 0.1")
val serverHostname = System.getProperty(JAVA_RMI_SERVER_HOSTNAME) ?: error("$JAVA_RMI_SERVER_HOSTNAME is not set!")
log.info("in startDaemon() - 0.2")
val platformSpecificOptions = listOf(
// hide daemon window
"-Djava.awt.headless=true",
"-D$JAVA_RMI_SERVER_HOSTNAME=$serverHostname"
)
log.info("in startDaemon() - 0.3")
val args = listOf(
javaExecutable.absolutePath, "-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator)
) +
@@ -494,13 +469,10 @@ class KotlinCompilerClient : KotlinCompilerDaemonClient {
COMPILER_DAEMON_CLASS_FQN_EXPERIMENTAL +
daemonOptions.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } +
compilerId.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) }
log.info("in startDaemon() - 1")
reportingTargets.report(DaemonReportCategory.DEBUG, "starting the daemon as: " + args.joinToString(" "))
val processBuilder = ProcessBuilder(args)
log.info("in startDaemon() - 2")
processBuilder.redirectErrorStream(true)
// assuming daemon process is deaf and (mostly) silent, so do not handle streams
log.info("daemon = launchProcessWithFallback")
val daemon =
launchProcessWithFallback(processBuilder, reportingTargets, "daemon client")
@@ -548,12 +520,9 @@ class KotlinCompilerClient : KotlinCompilerDaemonClient {
}
} ?: DAEMON_DEFAULT_STARTUP_TIMEOUT_MS
if (daemonOptions.runFilesPath.isNotEmpty()) {
log.info("daemonOptions.runFilesPath.isNotEmpty")
val succeeded = isEchoRead.tryAcquire(daemonStartupTimeout, TimeUnit.MILLISECONDS)
log.info("succeeded : $succeeded")
return when {
!isProcessAlive(daemon) -> {
log.info("!isProcessAlive(daemon)")
reportingTargets.report(
DaemonReportCategory.INFO,
"Daemon terminated unexpectedly with error code: ${daemon.exitValue()}"
@@ -561,14 +530,12 @@ class KotlinCompilerClient : KotlinCompilerDaemonClient {
false
}
!succeeded -> {
log.info("isProcessAlive!")
reportingTargets.report(DaemonReportCategory.INFO, "Unable to get response from daemon in $daemonStartupTimeout ms")
false
}
else -> true
}
} else
log.info("!daemonOptions.runFilesPath.isNotEmpty")
// without startEcho defined waiting for max timeout
Thread.sleep(daemonStartupTimeout)
return true
@@ -23,19 +23,19 @@ val nativePlatformVariants = listOf(
dependencies {
compileOnly(project(":compiler:util"))
compileOnly(project(":compiler:cli-common"))
compileOnly(project(":compiler:daemon-common"))
compileOnly(project(":daemon-common"))
compileOnly(project(":kotlin-reflect-api"))
compileOnly(project(":js:js.frontend"))
compileOnly(commonDep("net.rubygrapefruit", "native-platform"))
compileOnly(intellijDep()) { includeIntellijCoreJarDependencies(project) }
embedded(project(":compiler:daemon-common")) { isTransitive = false }
embedded(project(":daemon-common")) { isTransitive = false }
embedded(commonDep("net.rubygrapefruit", "native-platform"))
nativePlatformVariants.forEach {
embedded(commonDep("net.rubygrapefruit", "native-platform", "-$it"))
}
runtime(project(":kotlin-reflect"))
compile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-jdk8")) {
compile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-core")) {
isTransitive = false
}
}
@@ -25,12 +25,12 @@ dependencies {
compile(project(":core:descriptors.jvm"))
compile(project(":compiler:util"))
compile(project(":compiler:cli-common"))
compileOnly(project(":compiler:daemon-common"))
compileOnly(project(":daemon-common"))
compile(kotlinStdlib())
compileOnly(project(":js:js.frontend"))
compileOnly(intellijCoreDep()) { includeJars("intellij-core") }
compileOnly(intellijDep()) { includeIntellijCoreJarDependencies(project) }
compile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-jdk8")) {
compile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-core")) {
isTransitive = false
}
compile(commonDep("io.ktor", "ktor-network")) {
@@ -99,9 +99,9 @@ private inline fun tryConnectToDaemonByRMI(port: Int, report: (DaemonReportCateg
val daemon = runBlocking {
runWithTimeout(2 * DAEMON_PERIODIC_CHECK_INTERVAL_MS) {
LocateRegistry.getRegistry(
LoopbackNetworkInterface.loopbackInetAddressName,
org.jetbrains.kotlin.daemon.common.LoopbackNetworkInterface.loopbackInetAddressName,
port,
LoopbackNetworkInterface.clientLoopbackSocketFactoryRMI
org.jetbrains.kotlin.daemon.common.LoopbackNetworkInterface.clientLoopbackSocketFactory
)?.lookup(COMPILER_SERVICE_RMI_NAME)
}
}
@@ -134,7 +134,7 @@ private suspend fun tryConnectToDaemonBySockets(
log.info("OK - daemon($port) connected to server!!!")
daemon
} catch (e: Throwable) {
report(DaemonReportCategory.INFO, "kcannot find or connect to socket")
report(DaemonReportCategory.INFO, "cannot find or connect to socket, exception:\n${e.javaClass.name}:${e.message}")
daemon.close()
null
}
@@ -6,7 +6,6 @@
package org.jetbrains.kotlin.daemon.common.experimental
import org.jetbrains.kotlin.daemon.common.IncrementalCompilerServicesFacade
import org.jetbrains.kotlin.daemon.common.SimpleDirtyData
import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Client
import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.DefaultClientRMIWrapper
import java.io.File
@@ -6,8 +6,6 @@
package org.jetbrains.kotlin.daemon.common.experimental
import org.jetbrains.kotlin.daemon.common.IncrementalCompilerServicesFacadeAsync
import org.jetbrains.kotlin.daemon.common.SimpleDirtyData
import org.jetbrains.kotlin.daemon.common.experimental.IncrementalCompilerServicesFacadeServerSide.*
import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Client
import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.DefaultClient
import java.io.File
@@ -7,8 +7,6 @@ package org.jetbrains.kotlin.daemon.common.experimental
import kotlinx.coroutines.runBlocking
import org.jetbrains.kotlin.daemon.common.IncrementalCompilerServicesFacade
import org.jetbrains.kotlin.daemon.common.SimpleDirtyData
import java.io.File
import java.io.Serializable
class IncrementalCompilerServicesFacadeRMIWrapper(val clientSide: IncrementalCompilerServicesFacadeClientSide) :
@@ -6,9 +6,6 @@
package org.jetbrains.kotlin.daemon.common.experimental
import org.jetbrains.kotlin.daemon.common.IncrementalCompilerServicesFacadeAsync
import org.jetbrains.kotlin.daemon.common.SimpleDirtyData
import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Server
import java.io.File
interface IncrementalCompilerServicesFacadeServerSide : IncrementalCompilerServicesFacadeAsync, CompilerServicesFacadeBaseServerSide
@@ -18,58 +18,11 @@ import java.rmi.server.RMIServerSocketFactory
import java.util.*
// copyed from original(org.jetbrains.kotlin.daemon.common.NetworkUtils) TODO
// unique part :
// - AbstractClientLoopbackSocketFactory / ServerLoopbackSocketFactoryRMI / ServerLoopbackSocketFactoryKtor - Ktor-Sockets instead of java sockets
// - findPortAndCreateSocket
// TODO: get rid of copy-paste here
object LoopbackNetworkInterface {
const val IPV4_LOOPBACK_INET_ADDRESS = "127.0.0.1"
const val IPV6_LOOPBACK_INET_ADDRESS = "::1"
// size of the requests queue for daemon services, so far seems that we don't need any big numbers here
// but if we'll start getting "connection refused" errors, that could be the first place to try to fix it
val SERVER_SOCKET_BACKLOG_SIZE by lazy {
System.getProperty(DAEMON_RMI_SOCKET_BACKLOG_SIZE_PROPERTY)?.toIntOrNull() ?: DEFAULT_SERVER_SOCKET_BACKLOG_SIZE
}
val SOCKET_CONNECT_ATTEMPTS by lazy {
System.getProperty(DAEMON_RMI_SOCKET_CONNECT_ATTEMPTS_PROPERTY)?.toIntOrNull() ?: DEFAULT_SOCKET_CONNECT_ATTEMPTS
}
val SOCKET_CONNECT_INTERVAL_MS by lazy {
System.getProperty(DAEMON_RMI_SOCKET_CONNECT_INTERVAL_PROPERTY)?.toLongOrNull() ?: DEFAULT_SOCKET_CONNECT_INTERVAL_MS
}
val serverLoopbackSocketFactoryRMI by lazy { ServerLoopbackSocketFactoryRMI() }
val clientLoopbackSocketFactoryRMI by lazy { ClientLoopbackSocketFactoryRMI() }
object LoopbackNetworkInterfaceKtor {
val serverLoopbackSocketFactoryKtor by lazy { ServerLoopbackSocketFactoryKtor() }
val clientLoopbackSocketFactoryKtor by lazy { ClientLoopbackSocketFactoryKtor() }
// TODO switch to InetAddress.getLoopbackAddress on java 7+
val loopbackInetAddressName by lazy {
try {
if (InetAddress.getByName(null) is Inet6Address) IPV6_LOOPBACK_INET_ADDRESS else IPV4_LOOPBACK_INET_ADDRESS
} catch (e: IOException) {
// getLocalHost may fail for unknown reasons in some situations, the fallback is to assume IPv4 for now
// TODO consider some other ways to detect default to IPv6 addresses in this case
IPV4_LOOPBACK_INET_ADDRESS
}
}
// base socket factories by default don't implement equals properly (see e.g. http://stackoverflow.com/questions/21555710/rmi-and-jmx-socket-factories)
// so implementing it in derived classes using the fact that they are singletons
class ServerLoopbackSocketFactoryRMI : RMIServerSocketFactory, Serializable {
override fun equals(other: Any?): Boolean = other === this || super.equals(other)
override fun hashCode(): Int = super.hashCode()
@Throws(IOException::class)
override fun createServerSocket(port: Int): java.net.ServerSocket =
ServerSocket(port, SERVER_SOCKET_BACKLOG_SIZE, InetAddress.getByName(null))
}
val selectorMgr = ActorSelectorManager(Dispatchers.IO)
class ServerLoopbackSocketFactoryKtor : Serializable {
@@ -83,31 +36,7 @@ object LoopbackNetworkInterface {
.bind(InetSocketAddress(InetAddress.getByName(null), port)) // TODO : NO BACKLOG SIZE CHANGE =(
}
abstract class AbstractClientLoopbackSocketFactory<SocketType> : Serializable {
override fun equals(other: Any?): Boolean = other === this || super.equals(other)
override fun hashCode(): Int = super.hashCode()
abstract protected fun socketCreate(host: String, port: Int): SocketType
@Throws(IOException::class)
fun createSocket(host: String, port: Int): SocketType {
var attemptsLeft = SOCKET_CONNECT_ATTEMPTS
while (true) {
try {
return socketCreate(host, port)
} catch (e: ConnectException) {
if (--attemptsLeft <= 0) throw e
}
Thread.sleep(SOCKET_CONNECT_INTERVAL_MS)
}
}
}
class ClientLoopbackSocketFactoryRMI : AbstractClientLoopbackSocketFactory<java.net.Socket>(), RMIClientSocketFactory {
override fun socketCreate(host: String, port: Int): Socket = Socket(InetAddress.getByName(null), port)
}
class ClientLoopbackSocketFactoryKtor : AbstractClientLoopbackSocketFactory<io.ktor.network.sockets.Socket>() {
class ClientLoopbackSocketFactoryKtor : LoopbackNetworkInterface.AbstractClientLoopbackSocketFactory<io.ktor.network.sockets.Socket>() {
override fun socketCreate(host: String, port: Int): io.ktor.network.sockets.Socket =
runBlocking { aSocket(selectorMgr).tcp().connect(InetSocketAddress(host, port)) }
}
@@ -126,7 +55,7 @@ fun findPortForSocket(attempts: Int, portRangeStart: Int, portRangeEnd: Int): Se
try {
return ServerSocketWrapper(
port,
LoopbackNetworkInterface.serverLoopbackSocketFactoryKtor.createServerSocket(port)
LoopbackNetworkInterfaceKtor.serverLoopbackSocketFactoryKtor.createServerSocket(port)
)
} catch (e: Exception) {
// assuming that the socketPort is already taken
@@ -1,29 +0,0 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.daemon.common.experimental
interface RemoteOutputStreamAsync {
/** closeStream() name is chosen since Clients are AutoClosable now
* and Client-implementations of RemoteOutputStreamAsync have conflict of 'close' name **/
suspend fun closeStream()
suspend fun write(data: ByteArray, offset: Int, length: Int)
suspend fun write(dataByte: Int)
}
interface RemoteInputStreamAsync {
/** closeStream() name is chosen since Clients are AutoClosable now
* and Client-implementations of RemoteInputStreamAsync have conflict of 'close' name **/
suspend fun closeStream()
suspend fun read(length: Int): ByteArray
suspend fun read(): Int
}
@@ -1,12 +0,0 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.daemon.common.experimental
import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Client
interface RemoteOutputStreamAsyncClientSide : RemoteOutputStreamAsync, Client<RemoteOutputStreamAsyncServerSide>
interface RemoteInputStreamClientSide : RemoteInputStreamAsync, Client<RemoteInputStreamServerSide>
@@ -1,43 +0,0 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.daemon.common.experimental
import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.ByteWriteChannelWrapper
import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Server
interface RemoteOutputStreamAsyncServerSide : RemoteOutputStreamAsync, Server<RemoteOutputStreamAsyncServerSide> {
// Query messages:
class CloseMessage : Server.Message<RemoteOutputStreamAsyncServerSide>() {
override suspend fun processImpl(server: RemoteOutputStreamAsyncServerSide, sendReply: (Any?) -> Unit) =
server.closeStream()
}
class WriteMessage(val data: ByteArray, val offset: Int = -1, val length: Int = -1) :
Server.Message<RemoteOutputStreamAsyncServerSide>() {
override suspend fun processImpl(server: RemoteOutputStreamAsyncServerSide, sendReply: (Any?) -> Unit) =
server.write(data, offset, length)
}
class WriteIntMessage(val dataByte: Int) : Server.Message<RemoteOutputStreamAsyncServerSide>() {
override suspend fun processImpl(server: RemoteOutputStreamAsyncServerSide, sendReply: (Any?) -> Unit) =
server.write(dataByte)
}
}
interface RemoteInputStreamServerSide : RemoteInputStreamAsync, Server<RemoteInputStreamServerSide> {
// Query messages:
class CloseMessage : Server.Message<RemoteInputStreamServerSide>() {
override suspend fun processImpl(server: RemoteInputStreamServerSide, sendReply: (Any?) -> Unit) =
server.closeStream()
}
class ReadMessage(val length: Int = -1) : Server.Message<RemoteInputStreamServerSide>() {
override suspend fun processImpl(server: RemoteInputStreamServerSide, sendReply: (Any?) -> Unit) =
sendReply(if (length == -1) server.read() else server.read(length))
}
}
@@ -1,41 +0,0 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.daemon.common.experimental
import org.jetbrains.kotlin.daemon.common.RemoteInputStream
import org.jetbrains.kotlin.daemon.common.RemoteOutputStream
import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Client
import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.DefaultClientRMIWrapper
class RemoteOutputStreamAsyncWrapper(val rmiOutput: RemoteOutputStream) : RemoteOutputStreamAsyncClientSide,
Client<RemoteOutputStreamAsyncServerSide> by DefaultClientRMIWrapper() {
override suspend fun closeStream() =
rmiOutput.close()
override suspend fun write(data: ByteArray, offset: Int, length: Int) =
rmiOutput.write(data, offset, length)
override suspend fun write(dataByte: Int) =
rmiOutput.write(dataByte)
}
class RemoteInputStreamAsyncWrapper(private val rmiInput: RemoteInputStream) : RemoteInputStreamClientSide,
Client<RemoteInputStreamServerSide> by DefaultClientRMIWrapper() {
override suspend fun closeStream() =
rmiInput.close()
override suspend fun read() =
rmiInput.read()
override suspend fun read(length: Int) =
rmiInput.read(length)
}
fun RemoteOutputStream.toClient() = RemoteOutputStreamAsyncWrapper(this)
fun RemoteInputStream.toClient() = RemoteInputStreamAsyncWrapper(this)
@@ -3,7 +3,8 @@ package org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure
import io.ktor.network.sockets.Socket
import kotlinx.coroutines.channels.*
import kotlinx.coroutines.*
import org.jetbrains.kotlin.daemon.common.experimental.LoopbackNetworkInterface
import org.jetbrains.kotlin.daemon.common.LoopbackNetworkInterface
import org.jetbrains.kotlin.daemon.common.experimental.LoopbackNetworkInterfaceKtor
import sun.net.ConnectionResetException
import java.beans.Transient
import java.io.IOException
@@ -198,7 +199,7 @@ abstract class DefaultAuthorizableClient<ServerType : ServerBase>(
try {
socket = LoopbackNetworkInterface.clientLoopbackSocketFactoryKtor.createSocket(
socket = LoopbackNetworkInterfaceKtor.clientLoopbackSocketFactoryKtor.createSocket(
serverHost,
serverPort
)
@@ -10,11 +10,12 @@ dependencies {
compile(project(":core:descriptors.jvm"))
compile(project(":compiler:util"))
compile(project(":compiler:cli-common"))
compile(project(":kotlin-build-common"))
compile(kotlinStdlib())
compileOnly(project(":js:js.frontend"))
compileOnly(intellijCoreDep()) { includeJars("intellij-core") }
compileOnly(intellijDep()) { includeIntellijCoreJarDependencies(project) }
compile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-jdk8")) {
compile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-core")) {
isTransitive = false
}
}
@@ -16,6 +16,7 @@
package org.jetbrains.kotlin.daemon.common
import org.jetbrains.kotlin.incremental.IncrementalModuleInfo
import java.io.File
import java.io.Serializable
import java.util.*
@@ -1,33 +0,0 @@
/*
* Copyright 2000-2018 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.daemon.common
import java.io.File
import java.io.Serializable
data class IncrementalModuleEntry(
private val projectPath: String,
val name: String,
val buildDir: File,
val buildHistoryFile: File
) : Serializable {
companion object {
private const val serialVersionUID = 0L
}
}
class IncrementalModuleInfo(
val projectRoot: File,
val dirToModule: Map<File, IncrementalModuleEntry>,
val nameToModules: Map<String, Set<IncrementalModuleEntry>>,
val jarToClassListFile: Map<File, File>,
// only for js and mpp
val jarToModule: Map<File, IncrementalModuleEntry>
) : Serializable {
companion object {
private const val serialVersionUID = 0L
}
}
@@ -73,25 +73,31 @@ object LoopbackNetworkInterface {
override fun createServerSocket(port: Int): ServerSocket = ServerSocket(port, SERVER_SOCKET_BACKLOG_SIZE, InetAddress.getByName(null))
}
class ClientLoopbackSocketFactory : RMIClientSocketFactory, Serializable {
abstract class AbstractClientLoopbackSocketFactory<SocketType> : Serializable {
override fun equals(other: Any?): Boolean = other === this || super.equals(other)
override fun hashCode(): Int = super.hashCode()
abstract protected fun socketCreate(host: String, port: Int): SocketType
@Throws(IOException::class)
override fun createSocket(host: String, port: Int): Socket {
fun createSocket(host: String, port: Int): SocketType {
var attemptsLeft = SOCKET_CONNECT_ATTEMPTS
while (true) {
try {
return Socket(InetAddress.getByName(null), port)
}
catch (e: ConnectException) {
return socketCreate(host, port)
} catch (e: ConnectException) {
if (--attemptsLeft <= 0) throw e
}
Thread.sleep(SOCKET_CONNECT_INTERVAL_MS)
}
}
}
class ClientLoopbackSocketFactory : AbstractClientLoopbackSocketFactory<java.net.Socket>(), RMIClientSocketFactory {
override fun socketCreate(host: String, port: Int): Socket = Socket(InetAddress.getByName(null), port)
}
}
@@ -93,22 +93,22 @@ inline fun usedMemory(withGC: Boolean): Long {
}
inline fun<R> beginWithMeasureWallTime(perfCounters: PerfCounters) = listOf(System.nanoTime())
inline fun beginMeasureWallTime() = listOf(System.nanoTime())
inline fun<R> endWithMeasureWallTime(perfCounters: PerfCounters, startState: List<Long>) {
inline fun endMeasureWallTime(perfCounters: PerfCounters, startState: List<Long>) {
val (startTime) = startState
perfCounters.addMeasurement(time = System.nanoTime() - startTime) // TODO: add support for time wrapping
}
inline fun beginWithMeasureWallAndThreadTimes(perfCounters: PerfCounters, threadMXBean: ThreadMXBean): List<Long> {
inline fun beginMeasureWallAndThreadTimes(perfCounters: PerfCounters, threadMXBean: ThreadMXBean): List<Long> {
val startTime = System.nanoTime()
val startThreadTime = threadMXBean.threadCpuTime()
val startThreadUserTime = threadMXBean.threadUserTime()
return listOf(startTime, startThreadTime, startThreadUserTime)
}
inline fun endWithMeasureWallAndThreadTimes(perfCounters: PerfCounters, threadMXBean: ThreadMXBean, startState: List<Long>) {
inline fun endMeasureWallAndThreadTimes(perfCounters: PerfCounters, threadMXBean: ThreadMXBean, startState: List<Long>) {
val (startTime, startThreadTime, startThreadUserTime) = startState
// TODO: add support for time wrapping
@@ -117,12 +117,12 @@ inline fun endWithMeasureWallAndThreadTimes(perfCounters: PerfCounters, threadMX
threadUser = threadMXBean.threadUserTime() - startThreadUserTime)
}
inline fun beginWithMeasureWallAndThreadTimes(perfCounters: PerfCounters) =
beginWithMeasureWallAndThreadTimes(perfCounters, ManagementFactory.getThreadMXBean())
inline fun endWithMeasureWallAndThreadTimes(perfCounters: PerfCounters, startState: List<Long>) =
endWithMeasureWallAndThreadTimes(perfCounters, ManagementFactory.getThreadMXBean(), startState)
inline fun beginMeasureWallAndThreadTimes(perfCounters: PerfCounters) =
beginMeasureWallAndThreadTimes(perfCounters, ManagementFactory.getThreadMXBean())
inline fun endMeasureWallAndThreadTimes(perfCounters: PerfCounters, startState: List<Long>) =
endMeasureWallAndThreadTimes(perfCounters, ManagementFactory.getThreadMXBean(), startState)
inline fun beginWithMeasureWallAndThreadTimesAndMemory(perfCounters: PerfCounters, withGC: Boolean = false, threadMXBean: ThreadMXBean): List<Long> {
inline fun beginMeasureWallAndThreadTimesAndMemory(perfCounters: PerfCounters, withGC: Boolean = false, threadMXBean: ThreadMXBean): List<Long> {
val startMem = usedMemory(withGC)
val startTime = System.nanoTime()
val startThreadTime = threadMXBean.threadCpuTime()
@@ -131,7 +131,7 @@ inline fun beginWithMeasureWallAndThreadTimesAndMemory(perfCounters: PerfCounter
return listOf(startMem, startTime, startThreadTime, startThreadUserTime)
}
inline fun endWithMeasureWallAndThreadTimesAndMemory(perfCounters: PerfCounters, withGC: Boolean = false, threadMXBean: ThreadMXBean, startState: List<Long>){
inline fun endMeasureWallAndThreadTimesAndMemory(perfCounters: PerfCounters, withGC: Boolean = false, threadMXBean: ThreadMXBean, startState: List<Long>){
val (startMem, startTime, startThreadTime, startThreadUserTime) = startState
// TODO: add support for time wrapping
@@ -141,11 +141,11 @@ inline fun endWithMeasureWallAndThreadTimesAndMemory(perfCounters: PerfCounters,
memory = usedMemory(withGC) - startMem)
}
inline fun<R> beginWithMeasureWallAndThreadTimesAndMemory(perfCounters: PerfCounters, withGC: Boolean) =
beginWithMeasureWallAndThreadTimesAndMemory(perfCounters, withGC, ManagementFactory.getThreadMXBean())
inline fun<R> beginMeasureWallAndThreadTimesAndMemory(perfCounters: PerfCounters, withGC: Boolean) =
beginMeasureWallAndThreadTimesAndMemory(perfCounters, withGC, ManagementFactory.getThreadMXBean())
inline fun<R> endWithMeasureWallAndThreadTimesAndMemory(perfCounters: PerfCounters, withGC: Boolean, startState: List<Long>) =
endWithMeasureWallAndThreadTimesAndMemory(perfCounters, withGC, ManagementFactory.getThreadMXBean(), startState)
inline fun<R> endMeasureWallAndThreadTimesAndMemory(perfCounters: PerfCounters, withGC: Boolean, startState: List<Long>) =
endMeasureWallAndThreadTimesAndMemory(perfCounters, withGC, ManagementFactory.getThreadMXBean(), startState)
class DummyProfiler : Profiler {
@@ -166,27 +166,27 @@ abstract class TotalProfiler : Profiler {
class WallTotalProfiler : TotalProfiler() {
@Suppress("OVERRIDE_BY_INLINE")
override inline fun beginMeasure(obj: Any?) = beginWithMeasureWallTime(total)
override inline fun beginMeasure(obj: Any?) = beginMeasureWallTime()
@Suppress("OVERRIDE_BY_INLINE")
override inline fun endMeasure(obj: Any?, startState: List<Long>) = endWithMeasureWallTime(total, startState)
override inline fun endMeasure(obj: Any?, startState: List<Long>) = endMeasureWallTime(total, startState)
}
class WallAndThreadTotalProfiler : TotalProfiler() {
@Suppress("OVERRIDE_BY_INLINE")
override inline fun beginMeasure(obj: Any?) = beginWithMeasureWallAndThreadTimes(total, threadMXBean)
override inline fun beginMeasure(obj: Any?) = beginMeasureWallAndThreadTimes(total, threadMXBean)
@Suppress("OVERRIDE_BY_INLINE")
override inline fun endMeasure(obj: Any?, startState: List<Long>) = endWithMeasureWallAndThreadTimes(total, threadMXBean, startState)
override inline fun endMeasure(obj: Any?, startState: List<Long>) = endMeasureWallAndThreadTimes(total, threadMXBean, startState)
}
class WallAndThreadAndMemoryTotalProfiler(val withGC: Boolean) : TotalProfiler() {
@Suppress("OVERRIDE_BY_INLINE")
override inline fun beginMeasure(obj: Any?) =
beginWithMeasureWallAndThreadTimesAndMemory(total, withGC, threadMXBean)
beginMeasureWallAndThreadTimesAndMemory(total, withGC, threadMXBean)
@Suppress("OVERRIDE_BY_INLINE")
override inline fun endMeasure(obj: Any?, startState: List<Long>) =
endWithMeasureWallAndThreadTimesAndMemory(total, withGC, threadMXBean, startState)
endMeasureWallAndThreadTimesAndMemory(total, withGC, threadMXBean, startState)
}
@@ -198,8 +198,8 @@ class WallAndThreadByClassProfiler() : TotalProfiler() {
@Suppress("OVERRIDE_BY_INLINE")
override inline fun beginMeasure(obj: Any?) =
beginWithMeasureWallAndThreadTimes(counters.getOrPut(obj?.javaClass?.name, { SimplePerfCountersWithTotal(total) }), threadMXBean)
beginMeasureWallAndThreadTimes(counters.getOrPut(obj?.javaClass?.name, { SimplePerfCountersWithTotal(total) }), threadMXBean)
@Suppress("OVERRIDE_BY_INLINE")
override inline fun endMeasure(obj: Any?, startState: List<Long>) =
endWithMeasureWallAndThreadTimes(counters.getOrPut(obj?.javaClass?.name, { SimplePerfCountersWithTotal(total) }), threadMXBean, startState)
endMeasureWallAndThreadTimes(counters.getOrPut(obj?.javaClass?.name, { SimplePerfCountersWithTotal(total) }), threadMXBean, startState)
}
@@ -41,10 +41,7 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
import org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler
import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.daemon.common.*
import org.jetbrains.kotlin.daemon.report.CompileServicesFacadeMessageCollector
import org.jetbrains.kotlin.daemon.report.DaemonMessageReporter
import org.jetbrains.kotlin.daemon.report.DaemonMessageReporterPrintStreamAdapter
import org.jetbrains.kotlin.daemon.report.getICReporter
import org.jetbrains.kotlin.daemon.report.*
import org.jetbrains.kotlin.incremental.*
import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
import org.jetbrains.kotlin.incremental.components.LookupTracker
@@ -389,7 +386,6 @@ abstract class CompileServiceImplBase(
val exitCode = checkedCompile(daemonMessageReporter, rpcProfiler) {
body(eventManager, rpcProfiler).code
}
log.fine("got exitCode")
CompileService.CallResult.Good(exitCode)
} finally {
eventManager.fireCompilationFinished()
@@ -413,7 +409,7 @@ abstract class CompileServiceImplBase(
val endMem = if (daemonOptions.reportPerf) usedMemory(withGC = false) else 0L
log.info("Done with result " + res.toString())
log.info("Done with result $res")
if (daemonOptions.reportPerf) {
val pc = profiler.getTotalCounters()
@@ -467,10 +463,13 @@ abstract class CompileServiceImplBase(
// )
// }
fun startDaemonLife() {
fun startDaemonElections() {
timer.schedule(10) {
exceptionLoggingTimerThread { initiateElections() }
}
}
fun configurePeriodicActivities() {
timer.schedule(delay = DAEMON_PERIODIC_CHECK_INTERVAL_MS, period = DAEMON_PERIODIC_CHECK_INTERVAL_MS) {
exceptionLoggingTimerThread { periodicAndAfterSessionCheck() }
}
@@ -516,21 +515,11 @@ abstract class CompileServiceImplBase(
}
}
protected inline fun <R, KotlinJvmReplServiceT> withValidReplImpl(
sessionId: Int,
body: KotlinJvmReplServiceT.() -> R
): CompileService.CallResult<R> =
withValidClientOrSessionProxy(sessionId) { session ->
(session?.data as? KotlinJvmReplServiceT?)?.let {
CompileService.CallResult.Good(it.body())
} ?: CompileService.CallResult.Error("Not a REPL session $sessionId")
}
protected fun execJsIncrementalCompiler(
args: K2JSCompilerArguments,
incrementalCompilationOptions: IncrementalCompilationOptions,
compilerMessageCollector: MessageCollector,
reporter: ICReporter
reporter: RemoteICReporter
): ExitCode {
val allKotlinFiles = arrayListOf<File>()
val freeArgsWithoutKotlinFiles = arrayListOf<String>()
@@ -561,7 +550,7 @@ abstract class CompileServiceImplBase(
return try {
compiler.compile(allKotlinFiles, args, compilerMessageCollector, changedFiles)
} finally {
(reporter as RemoteICReporter).flush()
reporter.flush()
}
}
@@ -569,18 +558,8 @@ abstract class CompileServiceImplBase(
k2jvmArgs: K2JVMCompilerArguments,
incrementalCompilationOptions: IncrementalCompilationOptions,
compilerMessageCollector: MessageCollector,
reporter: ICReporter
reporter: RemoteICReporter
): ExitCode {
val moduleFile = k2jvmArgs.buildFile?.let(::File)
assert(moduleFile?.exists() ?: false) { "Module does not exist ${k2jvmArgs.buildFile}" }
// todo: pass javaSourceRoots and allKotlinFiles using IncrementalCompilationOptions
val parsedModule = run {
val bytesOut = ByteArrayOutputStream()
val printStream = PrintStream(bytesOut)
val mc = PrintingMessageCollector(printStream, MessageRenderer.PLAIN_FULL_PATHS, false)
val parsedModule = ModuleXmlParser.parseModuleScript(k2jvmArgs.buildFile!!, mc)
parsedModule
val allKotlinExtensions = (DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS +
(incrementalCompilationOptions.kotlinScriptExtensions ?: emptyArray())).distinct()
val dotExtensions = allKotlinExtensions.map { ".$it" }
@@ -614,27 +593,22 @@ abstract class CompileServiceImplBase(
}
}
val outputFiles = incrementalCompilationOptions.outputFiles.toMutableList()
incrementalCompilationOptions.classpathFqNamesHistory?.let { outputFiles.add(it) }
val compiler = IncrementalJvmCompilerRunner(
workingDir,
reporter,
buildHistoryFile = incrementalCompilationOptions.multiModuleICSettings.buildHistoryFile,
outputFiles = outputFiles,
outputFiles = incrementalCompilationOptions.outputFiles,
usePreciseJavaTracking = incrementalCompilationOptions.usePreciseJavaTracking,
modulesApiHistory = modulesApiHistory,
kotlinSourceFilesExtensions = allKotlinExtensions,
classpathFqNamesHistory = incrementalCompilationOptions.classpathFqNamesHistory
kotlinSourceFilesExtensions = allKotlinExtensions
)
return try {
compiler.compile(allKotlinFiles, k2jvmArgs, compilerMessageCollector, changedFiles)
} finally {
(reporter as RemoteICReporter).flush()
reporter.flush()
}
}
@JvmName("withValidReplImpl1")
protected inline fun <R, KotlinJvmReplServiceT> withValidReplImpl(
sessionId: Int,
body: KotlinJvmReplServiceT.() -> CompileService.CallResult<R>
@@ -838,7 +812,7 @@ class CompileServiceImpl(
)
val messageCollector = KeepFirstErrorMessageCollector(compilerMessagesStream)
val repl = KotlinJvmReplService(
disposable, port, templateClasspath, templateClassName,
disposable, port, compilerId, templateClasspath, templateClassName,
messageCollector, operationsTracer
)
val sessionId = state.sessions.leaseSession(ClientOrSessionProxy(aliveFlagPath, repl, disposable))
@@ -922,7 +896,7 @@ class CompileServiceImpl(
val disposable = Disposer.newDisposable()
val messageCollector = CompileServicesFacadeMessageCollector(servicesFacade, compilationOptions)
val repl = KotlinJvmReplService(
disposable, port, templateClasspath, templateClassName,
disposable, port, compilerId, templateClasspath, templateClassName,
messageCollector, null
)
val sessionId = state.sessions.leaseSession(ClientOrSessionProxy(aliveFlagPath, repl, disposable))
@@ -55,22 +55,21 @@ class LogStream(name: String) : OutputStream() {
}
}
object KotlinCompileDaemon {
abstract class KotlinCompileDaemonBase {
init {
val logTime: String = SimpleDateFormat("yyyy-MM-dd.HH-mm-ss-SSS").format(Date())
val (logPath: String, fileIsGiven: Boolean) =
System.getProperty(COMPILE_DAEMON_LOG_PATH_PROPERTY)?.trimQuotes()?.let { Pair(it, File(it).isFile) } ?: Pair("%t", false)
val cfg: String =
"handlers = java.util.logging.FileHandler\n" +
"java.util.logging.FileHandler.level = ALL\n" +
"java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter\n" +
"java.util.logging.FileHandler.encoding = UTF-8\n" +
"java.util.logging.FileHandler.limit = ${if (fileIsGiven) 0 else (1 shl 20)}\n" + // if file is provided - disabled, else - 1Mb
"java.util.logging.FileHandler.count = ${if (fileIsGiven) 1 else 3}\n" +
"java.util.logging.FileHandler.append = $fileIsGiven\n" +
"java.util.logging.FileHandler.pattern = ${if (fileIsGiven) logPath else (logPath + File.separator + "$COMPILE_DAEMON_DEFAULT_FILES_PREFIX.$logTime.%u%g.log")}\n" +
"java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s%n\n"
"handlers = java.util.logging.FileHandler\n" +
"java.util.logging.FileHandler.level = ALL\n" +
"java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter\n" +
"java.util.logging.FileHandler.encoding = UTF-8\n" +
"java.util.logging.FileHandler.limit = ${if (fileIsGiven) 0 else (1 shl 20)}\n" + // if file is provided - disabled, else - 1Mb
"java.util.logging.FileHandler.count = ${if (fileIsGiven) 1 else 3}\n" +
"java.util.logging.FileHandler.append = $fileIsGiven\n" +
"java.util.logging.FileHandler.pattern = ${if (fileIsGiven) logPath else (logPath + File.separator + "$COMPILE_DAEMON_DEFAULT_FILES_PREFIX.$logTime.%u%g.log")}\n" +
"java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s%n\n"
LogManager.getLogManager().readConfiguration(cfg.byteInputStream())
}
@@ -78,19 +77,32 @@ object KotlinCompileDaemon {
val log by lazy { Logger.getLogger("daemon") }
private fun loadVersionFromResource(): String? {
(KotlinCompileDaemon::class.java.classLoader as? URLClassLoader)
?.findResource("META-INF/MANIFEST.MF")
?.let {
try {
return Manifest(it.openStream()).mainAttributes.getValue("Implementation-Version") ?: null
}
catch (e: IOException) {}
(KotlinCompileDaemonBase::class.java.classLoader as? URLClassLoader)
?.findResource("META-INF/MANIFEST.MF")
?.let {
try {
return Manifest(it.openStream()).mainAttributes.getValue("Implementation-Version") ?: null
}
catch (e: IOException) {}
}
return null
}
@JvmStatic
fun main(args: Array<String>) {
protected open fun <T> runSynchronized(block: () -> T) = block()
protected abstract fun getCompileServiceAndPort(
compilerSelector: CompilerSelector,
compilerId: CompilerId,
daemonOptions: DaemonOptions,
daemonJVMOptions: DaemonJVMOptions,
timer: Timer
) : Pair<CompileServiceImplBase, Int>
protected open fun runCompileService(compileService: CompileServiceImplBase) : Any? = null
protected open fun awaitServerRun(serverRun: Any?) {}
protected fun mainImpl(args: Array<String>) {
ensureServerHostnameIsSetUp()
val jvmArguments = ManagementFactory.getRuntimeMXBean().inputArguments
@@ -103,84 +115,107 @@ object KotlinCompileDaemon {
val compilerId = CompilerId()
val daemonOptions = DaemonOptions()
runSynchronized {
var serverRun: Any?
try {
val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true,
inheritOtherJvmOptions = true,
inheritAdditionalProperties = true)
try {
val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true,
inheritOtherJvmOptions = true,
inheritAdditionalProperties = true)
val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX)
val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX)
if (filteredArgs.any()) {
val helpLine = "usage: <daemon> <compilerId options> <daemon options>"
log.info(helpLine)
println(helpLine)
throw IllegalArgumentException("Unknown arguments: " + filteredArgs.joinToString(" "))
}
log.info("starting daemon")
// TODO: find minimal set of permissions and restore security management
// note: may be not needed anymore since (hopefully) server is now loopback-only
// if (System.getSecurityManager() == null)
// System.setSecurityManager (RMISecurityManager())
//
// setDaemonPermissions(daemonOptions.port)
val (registry, port) = findPortAndCreateRegistry(COMPILE_DAEMON_FIND_PORT_ATTEMPTS, COMPILE_DAEMON_PORTS_RANGE_START, COMPILE_DAEMON_PORTS_RANGE_END)
val compilerSelector = object : CompilerSelector {
private val jvm by lazy { K2JVMCompiler() }
private val js by lazy { K2JSCompiler() }
private val metadata by lazy { K2MetadataCompiler() }
override fun get(targetPlatform: CompileService.TargetPlatform): CLICompiler<*> = when (targetPlatform) {
CompileService.TargetPlatform.JVM -> jvm
CompileService.TargetPlatform.JS -> js
CompileService.TargetPlatform.METADATA -> metadata
if (filteredArgs.any()) {
val helpLine = "usage: <daemon> <compilerId options> <daemon options>"
log.info(helpLine)
println(helpLine)
throw IllegalArgumentException("Unknown arguments: " + filteredArgs.joinToString(" "))
}
log.info("starting daemon")
// TODO: find minimal set of permissions and restore security management
// note: may be not needed anymore since (hopefully) server is now loopback-only
// if (System.getSecurityManager() == null)
// System.setSecurityManager (RMISecurityManager())
//
// setDaemonPermissions(daemonOptions.port)
val compilerSelector = object : CompilerSelector {
private val jvm by lazy { K2JVMCompiler() }
private val js by lazy { K2JSCompiler() }
private val metadata by lazy { K2MetadataCompiler() }
override fun get(targetPlatform: CompileService.TargetPlatform): CLICompiler<*> = when (targetPlatform) {
CompileService.TargetPlatform.JVM -> jvm
CompileService.TargetPlatform.JS -> js
CompileService.TargetPlatform.METADATA -> metadata
}
}
// timer with a daemon thread, meaning it should not prevent JVM to exit normally
val timer = Timer(true)
val (compilerService, port) = getCompileServiceAndPort(compilerSelector, compilerId, daemonOptions, daemonJVMOptions, timer)
compilerService.startDaemonElections()
compilerService.configurePeriodicActivities()
serverRun = runCompileService(compilerService)
println(COMPILE_DAEMON_IS_READY_MESSAGE)
log.info("daemon is listening on port: $port")
// this supposed to stop redirected streams reader(s) on the client side and prevent some situations with hanging threads, but doesn't work reliably
// TODO: implement more reliable scheme
System.out.close()
System.err.close()
System.setErr(PrintStream(LogStream("stderr")))
System.setOut(PrintStream(LogStream("stdout")))
}
// timer with a daemon thread, meaning it should not prevent JVM to exit normally
val timer = Timer(true)
val compilerService = CompileServiceImpl(registry = registry,
compiler = compilerSelector,
compilerId = compilerId,
daemonOptions = daemonOptions,
daemonJVMOptions = daemonJVMOptions,
port = port,
timer = timer,
onShutdown = {
if (daemonOptions.forceShutdownTimeoutMilliseconds != COMPILE_DAEMON_TIMEOUT_INFINITE_MS) {
// running a watcher thread that ensures that if the daemon is not exited normally (may be due to RMI leftovers), it's forced to exit
timer.schedule(daemonOptions.forceShutdownTimeoutMilliseconds) {
cancel()
log.info("force JVM shutdown")
exitProcess(0)
}
}
else {
timer.cancel()
}
})
compilerService.startDaemonLife()
println(COMPILE_DAEMON_IS_READY_MESSAGE)
log.info("daemon is listening on port: $port")
// this supposed to stop redirected streams reader(s) on the client side and prevent some situations with hanging threads, but doesn't work reliably
// TODO: implement more reliable scheme
System.out.close()
System.err.close()
System.setErr(PrintStream(LogStream("stderr")))
System.setOut(PrintStream(LogStream("stdout")))
}
catch (e: Exception) {
System.err.println("Exception: " + e.message)
e.printStackTrace(System.err)
// repeating it to log for the cases when stderr is not redirected yet
log.log(Level.INFO, "Exception: ", e)
// TODO consider exiting without throwing
throw e
catch (e: Exception) {
System.err.println("Exception: " + e.message)
e.printStackTrace(System.err)
// repeating it to log for the cases when stderr is not redirected yet
log.log(Level.INFO, "Exception: ", e)
// TODO consider exiting without throwing
throw e
}
awaitServerRun(serverRun)
}
}
}
object KotlinCompileDaemon : KotlinCompileDaemonBase() {
@JvmStatic
fun main(args: Array<String>) {
mainImpl(args)
}
override fun getCompileServiceAndPort(
compilerSelector: CompilerSelector,
compilerId: CompilerId,
daemonOptions: DaemonOptions,
daemonJVMOptions: DaemonJVMOptions,
timer: Timer
) = run {
val (registry, port) = findPortAndCreateRegistry(COMPILE_DAEMON_FIND_PORT_ATTEMPTS, COMPILE_DAEMON_PORTS_RANGE_START, COMPILE_DAEMON_PORTS_RANGE_END)
val compilerService = CompileServiceImpl(registry = registry,
compiler = compilerSelector,
compilerId = compilerId,
daemonOptions = daemonOptions,
daemonJVMOptions = daemonJVMOptions,
port = port,
timer = timer,
onShutdown = {
if (daemonOptions.forceShutdownTimeoutMilliseconds != COMPILE_DAEMON_TIMEOUT_INFINITE_MS) {
// running a watcher thread that ensures that if the daemon is not exited normally (may be due to RMI leftovers), it's forced to exit
timer.schedule(daemonOptions.forceShutdownTimeoutMilliseconds) {
cancel()
log.info("force JVM shutdown")
exitProcess(0)
}
}
else {
timer.cancel()
}
})
Pair(compilerService, port)
}
}
@@ -42,7 +42,8 @@ import kotlin.concurrent.write
abstract class KotlinJvmReplServiceBase(
disposable: Disposable,
val compilerId: CompilerId,templateClasspath: List<File>,
val compilerId: CompilerId,
templateClasspath: List<File>,
templateClassName: String,
protected val messageCollector: MessageCollector
) : ReplCompileAction, ReplCheckAction, CreateReplStageStateAction {
@@ -60,27 +61,6 @@ abstract class KotlinJvmReplServiceBase(
configureScripting(compilerId)
}
protected fun makeScriptDefinition(templateClasspath: List<File>, templateClassName: String): KotlinScriptDefinition? {
val classloader = URLClassLoader(templateClasspath.map { it.toURI().toURL() }.toTypedArray(), this::class.java.classLoader)
try {
val cls = classloader.loadClass(templateClassName)
val def = KotlinScriptDefinitionFromAnnotatedTemplate(cls.kotlin, emptyMap())
messageCollector.report(INFO, "New script definition $templateClassName: files pattern = \"${def.scriptFilePattern}\", " +
"resolver = ${def.dependencyResolver.javaClass.name}")
return def
}
catch (ex: ClassNotFoundException) {
messageCollector.report(ERROR, "Cannot find script definition template class $templateClassName")
}
catch (ex: Exception) {
messageCollector.report(ERROR, "Error processing script definition template $templateClassName: ${ex.message}")
}
return null
}
private val scriptDef = makeScriptDefinition(templateClasspath, templateClassName)
protected val replCompiler: ReplCompiler? by lazy {
try {
val projectEnvironment =
@@ -142,12 +122,13 @@ abstract class KotlinJvmReplServiceBase(
open class KotlinJvmReplService(
disposable: Disposable,
val portForServers: Int,
compilerId: CompilerId,
templateClasspath: List<File>,
templateClassName: String,
messageCollector: MessageCollector,
@Deprecated("drop it")
protected val operationsTracer: RemoteOperationsTracer?
) : KotlinJvmReplServiceBase(disposable, templateClasspath, templateClassName, messageCollector) {
) : KotlinJvmReplServiceBase(disposable, compilerId, templateClasspath, templateClassName, messageCollector) {
override fun before(s: String) {
operationsTracer?.before(s)
@@ -331,10 +331,12 @@ class CompileServiceServerSideImpl(
compilationOptions,
servicesFacade,
compilationResults,
hasIncrementalCaches = CompilerCallbackServicesFacadeClientSide::hasIncrementalCaches,
hasIncrementalCaches = { hasIncrementalCaches() },
createMessageCollector = ::CompileServicesFacadeMessageCollector,
createReporter = ::DaemonMessageReporterAsync,
createServices = this::createCompileServices,
createServices = { facade: CompilerCallbackServicesFacadeClientSide, eventMgr, profiler ->
createCompileServices(facade, eventMgr, profiler)
},
getICReporter = ::getICReporterAsync
)
@@ -353,7 +355,7 @@ class CompileServiceServerSideImpl(
val messageCollector =
CompileServicesFacadeMessageCollector(servicesFacade, compilationOptions)
val repl = KotlinJvmReplServiceAsync(
disposable, serverSocketWithPort, templateClasspath, templateClassName,
disposable, serverSocketWithPort, compilerId, templateClasspath, templateClassName,
messageCollector
)
val sessionId = state.sessions.leaseSession(ClientOrSessionProxy(aliveFlagPath, repl, disposable))
@@ -407,7 +409,7 @@ class CompileServiceServerSideImpl(
System.setProperty(KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY, "true")
// TODO UNCOMMENT THIS : this.toRMIServer(daemonOptions, compilerId) // also create RMI server in order to support old clients
rmiServer = this.toRMIServer(daemonOptions, compilerId)
// rmiServer = this.toRMIServer(daemonOptions, compilerId)
KeepAliveServer.runServer()
}
@@ -12,6 +12,7 @@ import org.jetbrains.kotlin.cli.common.environment.setIdeaIoUseFallback
import org.jetbrains.kotlin.cli.js.K2JSCompiler
import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
import org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler
import org.jetbrains.kotlin.daemon.CompileServiceImplBase
import org.jetbrains.kotlin.daemon.CompilerSelector
import org.jetbrains.kotlin.daemon.common.*
import org.jetbrains.kotlin.daemon.common.experimental.findPortForSocket
@@ -29,174 +30,54 @@ import java.util.logging.Level
import java.util.logging.LogManager
import java.util.logging.Logger
import kotlin.concurrent.schedule
import org.jetbrains.kotlin.daemon.KotlinCompileDaemonBase
import org.jetbrains.kotlin.daemon.common.experimental.CompileServiceServerSide
object KotlinCompileDaemon : KotlinCompileDaemonBase() {
override fun <T> runSynchronized(block: () -> T) = runBlocking { block() }
class LogStream(name: String) : OutputStream() {
val log by lazy { Logger.getLogger(name) }
val lineBuf = StringBuilder()
override fun write(byte: Int) {
if (byte.toChar() == '\n') flush()
else lineBuf.append(byte.toChar())
}
override fun flush() {
log.info(lineBuf.toString())
lineBuf.setLength(0)
}
}
object KotlinCompileDaemon {
init {
val logTime: String = SimpleDateFormat("yyyy-MM-dd.HH-mm-ss-SSS").format(Date())
val (logPath: String, fileIsGiven: Boolean) =
System.getProperty(COMPILE_DAEMON_LOG_PATH_PROPERTY)?.trimQuotes()?.let { Pair(it, File(it).isFile) } ?: Pair("%t", false)
val cfg: String =
"handlers = java.util.logging.FileHandler\n" +
"java.util.logging.FileHandler.level = ALL\n" +
"java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter\n" +
"java.util.logging.FileHandler.encoding = UTF-8\n" +
"java.util.logging.FileHandler.limit = ${if (fileIsGiven) 0 else (1 shl 20)}\n" + // if file is provided - disabled, else - 1Mb
"java.util.logging.FileHandler.count = ${if (fileIsGiven) 1 else 3}\n" +
"java.util.logging.FileHandler.append = $fileIsGiven\n" +
"java.util.logging.FileHandler.pattern = ${if (fileIsGiven) logPath else (logPath + File.separator + "$COMPILE_DAEMON_DEFAULT_FILES_PREFIX.$logTime.%u%g.log")}\n" +
"java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s%n\n"
LogManager.getLogManager().readConfiguration(cfg.byteInputStream())
}
val log by lazy { Logger.getLogger("daemon") }
private fun loadVersionFromResource(): String? {
(KotlinCompileDaemon::class.java.classLoader as? URLClassLoader)
?.findResource("META-INF/MANIFEST.MF")
?.let {
try {
return Manifest(it.openStream()).mainAttributes.getValue("Implementation-Version") ?: null
} catch (e: IOException) {
override fun getCompileServiceAndPort(
compilerSelector: CompilerSelector,
compilerId: CompilerId,
daemonOptions: DaemonOptions,
daemonJVMOptions: DaemonJVMOptions,
timer: Timer
) = run {
val port = findPortForSocket(
COMPILE_DAEMON_FIND_PORT_ATTEMPTS,
COMPILE_DAEMON_PORTS_RANGE_START,
COMPILE_DAEMON_PORTS_RANGE_END
)
val compilerService = CompileServiceServerSideImpl(
port,
compilerSelector,
compilerId,
daemonOptions,
daemonJVMOptions,
port.port,
timer,
{
if (daemonOptions.forceShutdownTimeoutMilliseconds != COMPILE_DAEMON_TIMEOUT_INFINITE_MS) {
// running a watcher thread that ensures that if the daemon is not exited normally (may be due to RMI leftovers), it's forced to exit
timer.schedule(daemonOptions.forceShutdownTimeoutMilliseconds) {
cancel()
log.info("force JVM shutdown")
System.exit(0)
}
} else {
timer.cancel()
}
}
return null
})
Pair(compilerService, port.port)
}
override fun runCompileService(compileService: CompileServiceImplBase) = (compileService as CompileServiceServerSide).runServer()
override fun awaitServerRun(serverRun: Any?) {
runBlocking { (serverRun as Deferred<Unit>?)?.await() }
}
@JvmStatic
fun main(args: Array<String>) {
ensureServerHostnameIsSetUp()
val jvmArguments = ManagementFactory.getRuntimeMXBean().inputArguments
log.info("Kotlin compiler daemon version " + (loadVersionFromResource() ?: "<unknown>"))
log.info("daemon JVM args: " + jvmArguments.joinToString(" "))
log.info("daemon args: " + args.joinToString(" "))
setIdeaIoUseFallback()
val compilerId = CompilerId()
val daemonOptions = DaemonOptions()
runBlocking {
var serverRun: Deferred<Unit>?
try {
val daemonJVMOptions = configureDaemonJVMOptions(
inheritMemoryLimits = true,
inheritOtherJvmOptions = true,
inheritAdditionalProperties = true
)
val filteredArgs = args.asIterable()
.filterExtractProps(
compilerId,
daemonOptions,
prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX
)
if (filteredArgs.any()) {
val helpLine = "usage: <daemon> <compilerId options> <daemon options>"
log.info(helpLine)
println(helpLine)
throw IllegalArgumentException("Unknown arguments: " + filteredArgs.joinToString(" "))
}
log.info("starting daemon")
// TODO: find minimal set of permissions and restore security management
// note: may be not needed anymore since (hopefully) server is now loopback-only
// if (System.getSecurityManager() == null)
// System.setSecurityManager (RMISecurityManager())
//
// setDaemonPermissions(daemonOptions.socketPort)
val port = findPortForSocket(
COMPILE_DAEMON_FIND_PORT_ATTEMPTS,
COMPILE_DAEMON_PORTS_RANGE_START,
COMPILE_DAEMON_PORTS_RANGE_END
)
val compilerSelector = object : CompilerSelector {
private val jvm by lazy { K2JVMCompiler() }
private val js by lazy { K2JSCompiler() }
private val metadata by lazy { K2MetadataCompiler() }
override fun get(targetPlatform: CompileService.TargetPlatform): CLICompiler<*> = when (targetPlatform) {
CompileService.TargetPlatform.JVM -> jvm
CompileService.TargetPlatform.JS -> js
CompileService.TargetPlatform.METADATA -> metadata
}
}
// timer with a daemon thread, meaning it should not prevent JVM to exit normally
val timer = Timer(true)
val compilerService = CompileServiceServerSideImpl(
port,
compilerSelector,
compilerId,
daemonOptions,
daemonJVMOptions,
port.port,
timer,
{
if (daemonOptions.forceShutdownTimeoutMilliseconds != COMPILE_DAEMON_TIMEOUT_INFINITE_MS) {
// running a watcher thread that ensures that if the daemon is not exited normally (may be due to RMI leftovers), it's forced to exit
timer.schedule(daemonOptions.forceShutdownTimeoutMilliseconds) {
cancel()
log.info("force JVM shutdown")
System.exit(0)
}
} else {
timer.cancel()
}
})
compilerService.startDaemonLife()
serverRun = compilerService.runServer()
println(COMPILE_DAEMON_IS_READY_MESSAGE)
log.info("daemon is listening on port: ${port.port}")
// this supposed to stop redirected streams reader(s) on the client side and prevent some situations with hanging threads, but doesn't work reliably
// TODO: implement more reliable scheme
System.out.close()
System.err.close()
System.setErr(PrintStream(LogStream("stderr")))
System.setOut(PrintStream(LogStream("stdout")))
} catch (e: Exception) {
log.log(Level.ALL, "Exception: " + e.message)
e.printStackTrace(System.err)
// repeating it to log for the cases when stderr is not redirected yet
log.log(Level.INFO, "Exception: ", e)
// TODO consider exiting without throwing
throw e
}
serverRun.await()
}
mainImpl(args)
}
}
@@ -8,9 +8,9 @@ package org.jetbrains.kotlin.daemon.experimental
import com.intellij.openapi.Disposable
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.cli.common.repl.IReplStageState
import org.jetbrains.kotlin.cli.jvm.repl.GenericReplCompilerState
import org.jetbrains.kotlin.daemon.KotlinJvmReplServiceBase
import org.jetbrains.kotlin.daemon.common.CompileService
import org.jetbrains.kotlin.daemon.common.CompilerId
import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.ServerSocketWrapper
import org.jetbrains.kotlin.daemon.getValidId
import java.io.File
@@ -21,20 +21,17 @@ import kotlin.concurrent.write
open class KotlinJvmReplServiceAsync(
disposable: Disposable,
val portForServers: ServerSocketWrapper,
compilerId: CompilerId,
templateClasspath: List<File>,
templateClassName: String,
messageCollector: MessageCollector
) : KotlinJvmReplServiceBase(disposable, templateClasspath, templateClassName, messageCollector) {
) : KotlinJvmReplServiceBase(disposable, compilerId, templateClasspath, templateClassName, messageCollector) {
protected val states = WeakHashMap<RemoteReplStateFacadeServerSide, Boolean>() // used as (missing) WeakHashSet
suspend fun createRemoteState(port: ServerSocketWrapper = portForServers): RemoteReplStateFacadeServerSide = statesLock.write {
val id = getValidId(stateIdCounter) { id -> states.none { it.key.getId() == id } }
val stateFacade = RemoteReplStateFacadeServerSide(
id,
createState().asState(GenericReplCompilerState::class.java),
port
)
val stateFacade = RemoteReplStateFacadeServerSide(id, createState(), port)
stateFacade.runServer()
states.put(stateFacade, true)
stateFacade
@@ -1,27 +0,0 @@
/*
* Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.daemon.experimental
import kotlinx.coroutines.runBlocking
import org.jetbrains.kotlin.daemon.common.experimental.RemoteOutputStreamAsyncClientSide
import org.jetbrains.kotlin.daemon.common.DummyProfiler
import org.jetbrains.kotlin.daemon.common.Profiler
import org.jetbrains.kotlin.daemon.common.withMeasure
import java.io.OutputStream
class RemoteOutputStreamClient(val remote: RemoteOutputStreamAsyncClientSide, val profiler: Profiler = DummyProfiler()) : OutputStream() {
override fun write(data: ByteArray) = runBlocking {
profiler.withMeasure(this) { remote.write(data, 0, data.size) }
}
override fun write(data: ByteArray, offset: Int, length: Int) = runBlocking {
profiler.withMeasure(this) { remote.write(data, offset, length) }
}
override fun write(byte: Int) = runBlocking {
profiler.withMeasure(this) { remote.write(byte) }
}
}
@@ -7,7 +7,7 @@ package org.jetbrains.kotlin.daemon.experimental
import io.ktor.network.sockets.Socket
import org.jetbrains.kotlin.cli.common.repl.ILineId
import org.jetbrains.kotlin.cli.jvm.repl.GenericReplCompilerState
import org.jetbrains.kotlin.cli.common.repl.IReplStageState
import org.jetbrains.kotlin.daemon.common.COMPILE_DAEMON_FIND_PORT_ATTEMPTS
import org.jetbrains.kotlin.daemon.common.experimental.*
import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.*
@@ -15,7 +15,7 @@ import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.*
@Suppress("UNCHECKED_CAST")
class RemoteReplStateFacadeServerSide(
val _id: Int,
val state: GenericReplCompilerState,
val state: IReplStageState<*>,
override val serverSocketWithPort: ServerSocketWrapper = findPortForSocket(
COMPILE_DAEMON_FIND_PORT_ATTEMPTS,
REPL_SERVER_PORTS_RANGE_START,