diff --git a/build-common/test/org/jetbrains/kotlin/metadata/DebugProtoBuf.java b/build-common/test/org/jetbrains/kotlin/metadata/DebugProtoBuf.java index 390053af263..f36004b80b2 100644 --- a/build-common/test/org/jetbrains/kotlin/metadata/DebugProtoBuf.java +++ b/build-common/test/org/jetbrains/kotlin/metadata/DebugProtoBuf.java @@ -34443,7 +34443,7 @@ public final class DebugProtoBuf { "tbrains.kotlin.metadata.VersionRequireme" + "nt.Level:\005ERROR\022\022\n\nerror_code\030\004 \001(\005\022\025\n\007m" + "essage\030\005 \001(\005B\004\230\265\030\001\022e\n\014version_kind\030\006 \001(\016" + - "2=.org.jetbrains.kotlin.metadata.Version" + + "2=.org.jetbrains.kotlin.metadata.version" + "Requirement.VersionKind:\020LANGUAGE_VERSIO" + "N\"+\n\005Level\022\013\n\007WARNING\020\000\022\t\n\005ERROR\020\001\022\n\n\006HI" + "DDEN\020\002\"J\n\013VersionKind\022\024\n\020LANGUAGE_VERSIO" + diff --git a/build.gradle.kts b/build.gradle.kts index 34a2dfa4c66..14ee819eb76 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -15,6 +15,7 @@ buildscript { repositories.withRedirector(project) { bootstrapKotlinRepo?.let(::maven) maven("https://plugins.gradle.org/m2") + maven("https://dl.bintray.com/kotlin/ktor") } // a workaround for kotlin compiler classpath in kotlin project: sometimes gradle substitutes @@ -95,6 +96,17 @@ val artifactsDir = "$distDir/artifacts" val ideaPluginDir = "$artifactsDir/ideaPlugin/Kotlin" val ideaUltimatePluginDir = "$artifactsDir/ideaUltimatePlugin/Kotlin" +extra["ktorExcludesForDaemon"] = listOf( + "org.jetbrains.kotlin" to "kotlin-reflect", + "org.jetbrains.kotlin" to "kotlin-stdlib", + "org.jetbrains.kotlin" to "kotlin-stdlib-common", + "org.jetbrains.kotlin" to "kotlin-stdlib-jdk8", + "org.jetbrains.kotlin" to "kotlin-stdlib-jdk7", + "org.jetbrains.kotlinx" to "kotlinx-coroutines-jdk8", + "org.jetbrains.kotlinx" to "kotlinx-coroutines-core", + "org.jetbrains.kotlinx" to "kotlinx-coroutines-core-common" +) + // TODO: use "by extra()" syntax where possible extra["distLibDir"] = project.file(distLibDir) extra["libsDir"] = project.file(distLibDir) @@ -160,6 +172,7 @@ extra["versions.jflex"] = "1.7.0" extra["versions.markdown"] = "0.1.25" extra["versions.trove4j"] = "1.0.20181211" extra["versions.kotlin-native-shared"] = "1.0-dev-57" +extra["versions.ktor-network"] = "1.0.1" if (!project.hasProperty("versions.kotlin-native")) { extra["versions.kotlin-native"] = "1.3-dev-9780" @@ -198,6 +211,7 @@ extra["compilerModules"] = arrayOf( ":compiler:frontend.java", ":compiler:cli-common", ":compiler:daemon-common", + ":compiler:daemon-common-new", ":compiler:daemon", ":compiler:ir.tree", ":compiler:ir.psi2ir", @@ -319,6 +333,7 @@ allprojects { maven(intellijRepo) maven(bootstrapKotlinRepo!!.replace("artifacts/content/maven/", "artifacts/content/internal/repo")) maven(kotlinNativeRepo) + maven("https://dl.bintray.com/kotlin/ktor") } configureJvmProject(javaHome!!, jvmTarget!!) diff --git a/compiler/build.gradle.kts b/compiler/build.gradle.kts index 4298eeb822c..f871c6259e8 100644 --- a/compiler/build.gradle.kts +++ b/compiler/build.gradle.kts @@ -30,6 +30,8 @@ fun configureFreeCompilerArg(isEnabled: Boolean, compilerArgument: String) { val antLauncherJar by configurations.creating +val ktorExcludesForDaemon : List> by rootProject.extra + dependencies { testRuntime(intellijDep()) // Should come before compiler, because of "progarded" stuff needed for tests @@ -50,8 +52,14 @@ dependencies { testCompile(project(":compiler:ir.tree")) // used for deepCopyWithSymbols call that is removed by proguard from the compiler TODO: make it more straightforward testCompile(project(":kotlin-scripting-compiler-impl")) testCompile(project(":kotlin-script-util")) - testCompileOnly(projectRuntimeJar(":kotlin-daemon-client")) + testCompileOnly(projectRuntimeJar(":kotlin-daemon-client-new")) testCompileOnly(project(":kotlin-reflect-api")) + testCompile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-jdk8")) { isTransitive = false } + testCompile(commonDep("io.ktor", "ktor-network")) { + ktorExcludesForDaemon.forEach { (group, module) -> + exclude(group = group, module = module) + } + } otherCompilerModules.forEach { testCompileOnly(project(it)) } @@ -59,7 +67,18 @@ dependencies { testCompileOnly(intellijDep()) { includeJars("openapi", "idea", "idea_rt", "util", "asm-all", rootProject = rootProject) } testRuntime(project(":kotlin-reflect")) - testRuntime(project(":kotlin-daemon-client")) + testRuntime(project(":kotlin-daemon-client-new")) + testRuntime(project(":compiler:daemon")) // + + testRuntime(project(":compiler:daemon-common-new")) // + + testRuntime(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-jdk8")) { + isTransitive = false + } + testRuntime(commonDep("io.ktor", "ktor-network")) { + ktorExcludesForDaemon.forEach { (group, module) -> + exclude(group = group, module = module) + } + + } testRuntime(androidDxJar()) testRuntime(files(toolsJar())) diff --git a/compiler/compiler-runner/build.gradle.kts b/compiler/compiler-runner/build.gradle.kts index 77f252b7971..15023c3b2e1 100644 --- a/compiler/compiler-runner/build.gradle.kts +++ b/compiler/compiler-runner/build.gradle.kts @@ -14,10 +14,12 @@ dependencies { compileOnly(project(":kotlin-preloader")) compileOnly(project(":compiler:frontend.java")) compileOnly(project(":compiler:daemon-common")) + compileOnly(project(":compiler:daemon-common-new")) compile(projectRuntimeJar(":kotlin-daemon-client")) compileOnly(project(":compiler:util")) compileOnly(intellijCoreDep()) { includeJars("intellij-core") } runtimeOnly(projectRuntimeJar(":kotlin-compiler-embeddable")) + compile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-jdk8")) { isTransitive = false } } sourceSets { diff --git a/compiler/compiler.pro b/compiler/compiler.pro index 3fe18d569d1..9af95b7ce6e 100644 --- a/compiler/compiler.pro +++ b/compiler/compiler.pro @@ -75,6 +75,7 @@ messages/**) -dontwarn com.intellij.util.io.Decompressor* -dontwarn org.w3c.dom.Location -dontwarn org.w3c.dom.Window +-dontwarn org.slf4j.** #-libraryjars '' diff --git a/compiler/compiler.pro.183 b/compiler/compiler.pro.183 index 765db257657..10c2dbd5a00 100644 --- a/compiler/compiler.pro.183 +++ b/compiler/compiler.pro.183 @@ -51,6 +51,7 @@ messages/**) -dontwarn org.jetbrains.annotations.Mutable -dontwarn com.intellij.util.io.TarUtil -dontwarn com.intellij.util.io.Compressor$Tar +-dontwarn org.slf4j.** # Annotations from intellijCore/annotations.jar that not presented in org.jetbrains.annotations -dontwarn org.jetbrains.annotations.Async* diff --git a/compiler/compiler.pro.as34 b/compiler/compiler.pro.as34 index c8589f42d1d..6c12794156f 100644 --- a/compiler/compiler.pro.as34 +++ b/compiler/compiler.pro.as34 @@ -52,6 +52,7 @@ messages/**) -dontwarn org.jetbrains.annotations.Mutable -dontwarn com.intellij.util.io.TarUtil -dontwarn com.intellij.util.io.Compressor$Tar +-dontwarn org.slf4j.** # Annotations from intellijCore/annotations.jar that not presented in org.jetbrains.annotations -dontwarn org.jetbrains.annotations.Async* diff --git a/compiler/daemon/build.gradle.kts b/compiler/daemon/build.gradle.kts index 23e13529fe9..74aae8690c2 100644 --- a/compiler/daemon/build.gradle.kts +++ b/compiler/daemon/build.gradle.kts @@ -1,22 +1,39 @@ +import com.sun.javafx.scene.CameraHelper.project +import org.gradle.internal.impldep.org.junit.experimental.categories.Categories.CategoryFilter.exclude +import org.jetbrains.kotlin.gradle.dsl.Coroutines plugins { kotlin("jvm") id("jps-compatible") } +val ktorExcludesForDaemon : List> by rootProject.extra + dependencies { compile(project(":compiler:cli")) compile(project(":compiler:cli-js")) compile(project(":compiler:daemon-common")) + compile(project(":compiler:daemon-common-new")) compile(project(":compiler:incremental-compilation-impl")) compile(project(":kotlin-build-common")) compile(commonDep("org.fusesource.jansi", "jansi")) compile(commonDep("org.jline", "jline")) compileOnly(intellijCoreDep()) { includeJars("intellij-core") } compileOnly(intellijDep()) { includeIntellijCoreJarDependencies(project) } + runtime(project(":kotlin-reflect")) + compileOnly(project(":kotlin-reflect-api")) + compile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-jdk8")) { isTransitive = false } + compile(commonDep("io.ktor", "ktor-network")) { + ktorExcludesForDaemon.forEach { (group, module) -> + exclude(group = group, module = module) + } + } } sourceSets { "main" { projectDefault() } "test" {} } +kotlin { + experimental.coroutines = Coroutines.ENABLE +} \ No newline at end of file diff --git a/compiler/daemon/daemon-client-new/build.gradle.kts b/compiler/daemon/daemon-client-new/build.gradle.kts new file mode 100644 index 00000000000..ca5bd7323e9 --- /dev/null +++ b/compiler/daemon/daemon-client-new/build.gradle.kts @@ -0,0 +1,78 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +description = "Kotlin Daemon Client New" + +plugins { + kotlin("jvm") + id("jps-compatible") +} + +jvmTarget = "1.8" + +val nativePlatformVariants = listOf( + "windows-amd64", + "windows-i386", + "osx-amd64", + "osx-i386", + "linux-amd64", + "linux-i386", + "freebsd-amd64-libcpp", + "freebsd-amd64-libstdcpp", + "freebsd-i386-libcpp", + "freebsd-i386-libstdcpp" +) + +val ktorExcludesForDaemon : List> by rootProject.extra + +dependencies { + compileOnly(project(":compiler:util")) + compileOnly(project(":compiler:cli-common")) + compileOnly(project(":compiler:daemon-common-new")) + compileOnly(project(":kotlin-reflect-api")) + compileOnly(project(":kotlin-daemon-client")) + embeddedComponents(project(":kotlin-daemon-client")) { isTransitive = false } + compileOnly(project(":js:js.frontend")) + compileOnly(commonDep("net.rubygrapefruit", "native-platform")) + compileOnly(intellijDep()) { includeIntellijCoreJarDependencies(project) } + + embeddedComponents(project(":compiler:daemon-common")) { isTransitive = false } + embeddedComponents(commonDep("net.rubygrapefruit", "native-platform")) + nativePlatformVariants.forEach { + embeddedComponents(commonDep("net.rubygrapefruit", "native-platform", "-$it")) + } + compile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-jdk8")) { + isTransitive = false + } + compile(commonDep("io.ktor", "ktor-network")) { + ktorExcludesForDaemon.forEach { (group, module) -> + exclude(group = group, module = module) + } + } +} + +sourceSets { + "main" { projectDefault() } + "test" {} +} + +publish() + +noDefaultJar() + +runtimeJar(task("shadowJar")) { + from(mainSourceSet.output) + fromEmbeddedComponents() +} + +sourcesJar() + +javadocJar() + +dist() + +ideaPlugin() diff --git a/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/BasicCompilerServicesWithResultsFacadeAsync.kt b/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/BasicCompilerServicesWithResultsFacadeAsync.kt new file mode 100644 index 00000000000..0743e044a10 --- /dev/null +++ b/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/BasicCompilerServicesWithResultsFacadeAsync.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.client.experimental + +import io.ktor.network.sockets.Socket +import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.daemon.client.reportFromDaemon +import org.jetbrains.kotlin.daemon.common.experimental.* +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Server +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.ServerSocketWrapper +import java.io.File +import java.io.Serializable + +open class BasicCompilerServicesWithResultsFacadeServerServerSide( + val messageCollector: MessageCollector, + val outputsCollector: ((File, List) -> Unit)? = null, + override val serverSocketWithPort: ServerSocketWrapper = findCallbackServerSocket() +) : CompilerServicesFacadeBaseServerSide { + + override val clients = hashMapOf() + + override suspend fun report(category: Int, severity: Int, message: String?, attachment: Serializable?) { + messageCollector.reportFromDaemon(outputsCollector, category, severity, message, attachment) + } + + val clientSide: CompilerServicesFacadeBaseClientSide + get() = CompilerServicesFacadeBaseClientSideImpl(serverSocketWithPort.port) + +} diff --git a/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/CompilerCallbackServicesFacadeAsync.kt b/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/CompilerCallbackServicesFacadeAsync.kt new file mode 100644 index 00000000000..540046bcd6d --- /dev/null +++ b/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/CompilerCallbackServicesFacadeAsync.kt @@ -0,0 +1,104 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.client.experimental + +import io.ktor.network.sockets.Socket +import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.daemon.client.reportFromDaemon +import org.jetbrains.kotlin.daemon.common.experimental.* +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Server +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.ServerSocketWrapper + +import org.jetbrains.kotlin.incremental.components.LookupInfo +import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents +import org.jetbrains.kotlin.load.kotlin.incremental.components.JvmPackagePartProto +import org.jetbrains.kotlin.modules.TargetId +import org.jetbrains.kotlin.progress.experimental.CompilationCanceledStatus +import org.jetbrains.kotlin.utils.isProcessCanceledException +import java.io.Serializable + +open class CompilerCallbackServicesFacadeServerServerSide( + val incrementalCompilationComponents: IncrementalCompilationComponents? = null, + val lookupTracker: LookupTracker? = null, + val compilationCanceledStatus: CompilationCanceledStatus? = null, + val messageCollector: MessageCollector? = null, + override val serverSocketWithPort: ServerSocketWrapper = findCallbackServerSocket() +) : CompilerCallbackServicesFacadeServerSide { + + override suspend fun report(category: Int, severity: Int, message: String?, attachment: Serializable?) { + messageCollector?.reportFromDaemon(null, category, severity, message, attachment) + } + + override val clients = hashMapOf() + + val clientSide : CompilerServicesFacadeBaseClientSide + get() = CompilerCallbackServicesFacadeClientSideImpl(serverSocketWithPort.port) + + override suspend fun hasIncrementalCaches(): Boolean = incrementalCompilationComponents != null + + override suspend fun hasLookupTracker(): Boolean = lookupTracker != null + + override suspend fun hasCompilationCanceledStatus(): Boolean = compilationCanceledStatus != null + + // TODO: consider replacing NPE with other reporting, although NPE here means most probably incorrect usage + + override suspend fun incrementalCache_getObsoletePackageParts(target: TargetId): Collection = + incrementalCompilationComponents!!.getIncrementalCache(target).getObsoletePackageParts() + + override suspend fun incrementalCache_getObsoleteMultifileClassFacades(target: TargetId): Collection = + incrementalCompilationComponents!!.getIncrementalCache(target).getObsoleteMultifileClasses() + + override suspend fun incrementalCache_getMultifileFacadeParts(target: TargetId, internalName: String): Collection? = + incrementalCompilationComponents!!.getIncrementalCache(target).getStableMultifileFacadeParts(internalName) + + override suspend fun incrementalCache_getPackagePartData(target: TargetId, partInternalName: String): JvmPackagePartProto? = + incrementalCompilationComponents!!.getIncrementalCache(target).getPackagePartData(partInternalName) + + override suspend fun incrementalCache_getModuleMappingData(target: TargetId): ByteArray? = + incrementalCompilationComponents!!.getIncrementalCache(target).getModuleMappingData() + + // todo: remove (the method it called was relevant only for old IC) + override suspend fun incrementalCache_registerInline(target: TargetId, fromPath: String, jvmSignature: String, toPath: String) { + } + + override suspend fun incrementalCache_getClassFilePath(target: TargetId, internalClassName: String): String = + incrementalCompilationComponents!!.getIncrementalCache(target).getClassFilePath(internalClassName) + + override suspend fun incrementalCache_close(target: TargetId) { + incrementalCompilationComponents!!.getIncrementalCache(target).close() + } + + override suspend fun lookupTracker_requiresPosition() = lookupTracker!!.requiresPosition + + override fun lookupTracker_record(lookups: Collection) { + val lookupTracker = lookupTracker!! + + for (it in lookups) { + lookupTracker.record(it.filePath, it.position, it.scopeFqName, it.scopeKind, it.name) + } + } + + private val lookupTracker_isDoNothing: Boolean = lookupTracker === LookupTracker.DO_NOTHING + + override suspend fun lookupTracker_isDoNothing(): Boolean = lookupTracker_isDoNothing + + override suspend fun compilationCanceledStatus_checkCanceled(): Void? { + try { + compilationCanceledStatus?.checkCanceled() + return null + } + catch (e: Exception) { + // avoid passing exceptions that may have different serialVersionUID on across rmi border + // removing dependency from openapi (this is obsolete part anyway, and will be removed soon) + if (e.isProcessCanceledException()) + throw Exception("-TODO- RmiFriendlyCompilationCanceledException()") //RmiFriendlyCompilationCanceledException() + else throw e + } + } + +} + diff --git a/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/KotlinCompilerClient.kt b/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/KotlinCompilerClient.kt new file mode 100644 index 00000000000..09d0a218077 --- /dev/null +++ b/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/KotlinCompilerClient.kt @@ -0,0 +1,612 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.client.experimental + +import io.ktor.network.sockets.Socket +import kotlinx.coroutines.* +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity +import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.daemon.client.CompileServiceSessionAsync +import org.jetbrains.kotlin.daemon.client.KotlinCompilerDaemonClient +import org.jetbrains.kotlin.daemon.client.DaemonReportMessage +import org.jetbrains.kotlin.daemon.client.DaemonReportingTargets +import org.jetbrains.kotlin.daemon.common.* +import org.jetbrains.kotlin.daemon.common.Profiler +import org.jetbrains.kotlin.daemon.common.experimental.* +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Server +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.ServerSocketWrapper +import org.jetbrains.kotlin.daemon.common.* +import java.io.File +import java.io.Serializable +import java.net.SocketException +import java.nio.channels.ClosedChannelException +import java.rmi.ConnectException +import java.rmi.ConnectIOException +import java.rmi.UnmarshalException +import java.util.concurrent.Semaphore +import java.util.concurrent.TimeUnit +import java.util.logging.Logger +import kotlin.concurrent.thread +import org.jetbrains.kotlin.daemon.client.launchProcessWithFallback + + +class KotlinCompilerClient : KotlinCompilerDaemonClient { + + init { + println("experimental KotlinCompilerClient is being instantiated") + } + + val DAEMON_DEFAULT_STARTUP_TIMEOUT_MS = 10000L + val DAEMON_CONNECT_CYCLE_ATTEMPTS = 3 + + val verboseReporting = System.getProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) != null + + private val log = Logger.getLogger("KotlinCompilerClient") + private fun String.info(msg: String) = {}()//log.info("[$this] : $msg") + + override fun getOrCreateClientFlagFile(daemonOptions: DaemonOptions): File = + // for jps property is passed from IDEA to JPS in KotlinBuildProcessParametersProvider + System.getProperty(COMPILE_DAEMON_CLIENT_ALIVE_PATH_PROPERTY) + ?.let(String::trimQuotes) + ?.takeUnless(String::isBlank) + ?.let(::File) + ?.takeIf(File::exists) + ?: makeAutodeletingFlagFile(baseDir = File(daemonOptions.runFilesPathOrDefault)) + + override suspend fun connectToCompileService( + compilerId: CompilerId, + daemonJVMOptions: DaemonJVMOptions, + daemonOptions: DaemonOptions, + reportingTargets: DaemonReportingTargets, + autostart: Boolean, + checkId: Boolean + ): CompileServiceAsync? { + log.info("in connectToCompileService") + val flagFile = getOrCreateClientFlagFile(daemonOptions) + return connectToCompileService( + compilerId, + flagFile, + daemonJVMOptions, + daemonOptions, + reportingTargets, + autostart + ) + } + + override suspend fun connectToCompileService( + compilerId: CompilerId, + clientAliveFlagFile: File, + daemonJVMOptions: DaemonJVMOptions, + daemonOptions: DaemonOptions, + reportingTargets: DaemonReportingTargets, + autostart: Boolean + ): CompileServiceAsync? { + log.info("connectToCompileService") + return connectAndLease( + compilerId, + clientAliveFlagFile, + daemonJVMOptions, + daemonOptions, + reportingTargets, + autostart, + leaseSession = false, + sessionAliveFlagFile = null + )?.compileService + } + + + override suspend fun connectAndLease( + compilerId: CompilerId, + clientAliveFlagFile: File, + daemonJVMOptions: DaemonJVMOptions, + daemonOptions: DaemonOptions, + reportingTargets: DaemonReportingTargets, + autostart: Boolean, + leaseSession: Boolean, + sessionAliveFlagFile: File? + ): CompileServiceSessionAsync? { + return connectLoop( + reportingTargets, + autostart + ) { isLastAttempt -> + + log.info("connectAndLease") + + fun CompileServiceAsync.leaseImpl(): Deferred = + GlobalScope.async { + // the newJVMOptions could be checked here for additional parameters, if needed + log.info("trying registerClient") + println("trying registerClient") + try { + registerClient(clientAliveFlagFile.absolutePath) + } catch (e: Throwable) { + return@async null + } + reportingTargets.report(DaemonReportCategory.DEBUG, "connected to the daemon") + if (!leaseSession) + CompileServiceSessionAsync(this@leaseImpl, CompileService.NO_SESSION) + else + try { + leaseCompileSession(sessionAliveFlagFile?.absolutePath) + } catch (e: Throwable) { + return@async null + } + .takeUnless { it is CompileService.CallResult.Dying } + ?.let { + CompileServiceSessionAsync(this@leaseImpl, it.get()) + } + } + + ensureServerHostnameIsSetUp() + val (service, newJVMOptions) = + tryFindSuitableDaemonOrNewOpts(File(daemonOptions.runFilesPath), compilerId, daemonJVMOptions) { cat, msg -> + GlobalScope.async { reportingTargets.report(cat, msg) } + }.await() + if (service != null) { + service.leaseImpl().await() + } else { + if (!isLastAttempt && autostart) { + log.info("starting daemon...") + if (startDaemon(compilerId, newJVMOptions, daemonOptions, reportingTargets)) { + reportingTargets.report(DaemonReportCategory.DEBUG, "new daemon started, trying to find it") + } + } + null + } + } + } + + override suspend fun shutdownCompileService(compilerId: CompilerId, daemonOptions: DaemonOptions) { + connectToCompileService( + compilerId, + DaemonJVMOptions(), + daemonOptions, + DaemonReportingTargets(out = System.out), + autostart = false, + checkId = false + )?.shutdown() + } + + override suspend fun leaseCompileSession(compilerService: CompileServiceAsync, aliveFlagPath: String?): Int = + compilerService.leaseCompileSession(aliveFlagPath).get() + + override suspend fun releaseCompileSession(compilerService: CompileServiceAsync, sessionId: Int) { + compilerService.releaseCompileSession(sessionId) + } + + override suspend fun compile( + compilerService: CompileServiceAsync, + sessionId: Int, + targetPlatform: CompileService.TargetPlatform, + args: Array, + messageCollector: MessageCollector, + outputsCollector: ((File, List) -> Unit)?, + compilerMode: CompilerMode, + reportSeverity: ReportSeverity, + port: Int, + profiler: Profiler + ): Int = profiler.withMeasure(this) { + log.info("Compile") + val services = BasicCompilerServicesWithResultsFacadeServerServerSide( + messageCollector, + outputsCollector, + findCallbackServerSocket() + ) + runBlocking { + services.runServer() + compilerService.compile( + sessionId, + args, + CompilationOptions( + compilerMode, + targetPlatform, + arrayOf( + ReportCategory.COMPILER_MESSAGE.code, + ReportCategory.DAEMON_MESSAGE.code, + ReportCategory.EXCEPTION.code, + ReportCategory.OUTPUT_MESSAGE.code + ), + reportSeverity.code, + emptyArray() + ), + services.clientSide, + createCompResults().clientSide + ).get() + } + } + + val COMPILE_DAEMON_CLIENT_OPTIONS_PROPERTY: String = "kotlin.daemon.client.options" + + data class ClientOptions( + var stop: Boolean = false + ) : OptionsGroup { + override val mappers: List> + get() = listOf(BoolPropMapper(this, ClientOptions::stop)) + } + + private fun configureClientOptions(opts: ClientOptions): ClientOptions { + System.getProperty(COMPILE_DAEMON_CLIENT_OPTIONS_PROPERTY)?.let { + val unrecognized = it.trimQuotes().split(",").filterExtractProps(opts.mappers, "") + if (unrecognized.any()) + throw IllegalArgumentException( + "Unrecognized client options passed via property $COMPILE_DAEMON_OPTIONS_PROPERTY: " + unrecognized.joinToString(" ") + + "\nSupported options: " + opts.mappers.joinToString(", ", transform = { it.names.first() }) + ) + } + return opts + } + + private fun configureClientOptions(): ClientOptions = + configureClientOptions(ClientOptions()) + + override fun main(vararg args: String) { + runBlocking { + val compilerId = CompilerId() + val daemonOptions = configureDaemonOptions() + val daemonLaunchingOptions = configureDaemonJVMOptions( + inheritMemoryLimits = true, + inheritOtherJvmOptions = false, + inheritAdditionalProperties = true + ) + + val clientOptions = configureClientOptions() + val filteredArgs = args.asIterable().filterExtractProps( + compilerId, + daemonOptions, + daemonLaunchingOptions, + clientOptions, + prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX + ) + + if (!clientOptions.stop) { + if (compilerId.compilerClasspath.none()) { + // attempt to find compiler to use + System.err.println("compiler wasn't explicitly specified, attempt to find appropriate jar") + detectCompilerClasspath() + ?.let { compilerId.compilerClasspath = it } + } + if (compilerId.compilerClasspath.none()) + throw IllegalArgumentException("Cannot find compiler jar") + else + log.info("desired compiler classpath: " + compilerId.compilerClasspath.joinToString(File.pathSeparator)) + } + + val daemon = connectToCompileService( + compilerId, + daemonLaunchingOptions, + daemonOptions, + DaemonReportingTargets(out = System.out), + autostart = !clientOptions.stop, + checkId = !clientOptions.stop + ) + + if (daemon == null) { + if (clientOptions.stop) { + System.err.println("No daemon found to shut down") + } else throw Exception("Unable to connect to daemon") + } else when { + clientOptions.stop -> { + log.info("Shutdown the daemon") + daemon.shutdown() + log.info("Daemon shut down successfully") + } + filteredArgs.none() -> { + // so far used only in tests + log.info( + "Warning: empty arguments list, only daemon check is performed: checkCompilerId() returns ${ + daemon.checkCompilerId( + compilerId + )}" + ) + } + else -> { + log.info("Executing daemon compilation with args: " + filteredArgs.joinToString(" ")) + val servicesFacade = + CompilerCallbackServicesFacadeServerServerSide() + val serverRun = servicesFacade.runServer() + try { + val memBefore = daemon.getUsedMemory().get() / 1024 + val startTime = System.nanoTime() + + val compResults = createCompResults() + val compResultsServerRun = compResults.runServer() + val res = daemon.compile( + CompileService.NO_SESSION, + filteredArgs.toList().toTypedArray(), + CompilationOptions( + CompilerMode.NON_INCREMENTAL_COMPILER, + CompileService.TargetPlatform.JVM, + arrayOf(), // TODO ??? + ReportSeverity.INFO.code, // TODO ??? + arrayOf() // TODO ??? + ), + servicesFacade.clientSide, + compResults.clientSide + ) + + val endTime = System.nanoTime() + log.info("Compilation ${if (res.isGood) "succeeded" else "failed"}, result code: ${res.get()}") + val memAfter = daemon.getUsedMemory().get() / 1024 + log.info("Compilation time: " + TimeUnit.NANOSECONDS.toMillis(endTime - startTime) + " ms") + log.info("Used memory $memAfter (${"%+d".format(memAfter - memBefore)} kb)") + } finally { + // TODO ?? + } + } + } + } + } + + override fun createCompResults(): CompilationResultsServerSide = object : CompilationResultsServerSide { + + override val clients = hashMapOf() + + override val serverSocketWithPort: ServerSocketWrapper + get() = resultsPort + + private val resultsPort = findPortForSocket( + COMPILE_DAEMON_FIND_PORT_ATTEMPTS, + RESULTS_SERVER_PORTS_RANGE_START, + RESULTS_SERVER_PORTS_RANGE_END + ) + + private val resultsMap = hashMapOf>() + + override val clientSide: CompilationResultsClientSide + get() = CompilationResultsClientSideImpl(resultsPort.port) + + override suspend fun add(compilationResultCategory: Int, value: Serializable) { + synchronized(this) { + resultsMap.putIfAbsent(compilationResultCategory, mutableListOf()) + resultsMap[compilationResultCategory]!!.add(value) + // TODO logger? + } + } + + } + + private fun detectCompilerClasspath(): List? = + System.getProperty("java.class.path") + ?.split(File.pathSeparator) + ?.map { File(it).parentFile } + ?.distinct() + ?.mapNotNull { + it?.walk() + ?.firstOrNull { it.name.equals(COMPILER_JAR_NAME, ignoreCase = true) } + } + ?.firstOrNull() + ?.let { listOf(it.absolutePath) } + + // --- Implementation --------------------------------------- + + @Synchronized + private inline fun connectLoop(reportingTargets: DaemonReportingTargets, autostart: Boolean, body: (Boolean) -> R?): R? { + try { + var attempts = 1 + while (true) { + val (res, err) = try { + body(attempts >= DAEMON_CONNECT_CYCLE_ATTEMPTS) to null + } catch (e: SocketException) { + null to e + } catch (e: ConnectException) { + null to e + } catch (e: ConnectIOException) { + null to e + } catch (e: UnmarshalException) { + null to e + } catch (e: RuntimeException) { + null to e + } catch (e: ClosedChannelException) { + null to e + } + + if (res != null) return res + + if (err != null) { + GlobalScope.async { + reportingTargets.report( + DaemonReportCategory.INFO, + (if (attempts >= DAEMON_CONNECT_CYCLE_ATTEMPTS || !autostart) "no more retries on: " else "retrying($attempts) on: ") + + err.toString() + ) + } + } + + if (attempts++ > DAEMON_CONNECT_CYCLE_ATTEMPTS || !autostart) { + return null + } + } + } catch (e: Throwable) { + GlobalScope.async { reportingTargets.report(DaemonReportCategory.EXCEPTION, e.toString()) } + } + return null + } + + private fun tryFindSuitableDaemonOrNewOpts( + registryDir: File, + compilerId: CompilerId, + daemonJVMOptions: DaemonJVMOptions, + report: (DaemonReportCategory, String) -> Unit + ): Deferred> = GlobalScope.async { + log.info("tryFindSuitableDaemonOrNewOpts") + + registryDir.mkdirs() + val timestampMarker = createTempFile("kotlin-daemon-client-tsmarker", directory = registryDir) + val aliveWithMetadata = try { + log.info("walkDaemonsAsync... : ${registryDir.path}") + walkDaemonsAsync(registryDir, compilerId, timestampMarker, report = report).also { + log.info( + "daemons (${it.size}): ${it.map { "daemon(params : " + it.jvmOptions.jvmParams.joinToString(", ") + ")" }.joinToString( + ", ", "[", "]" + )}" + ) + } + } finally { + timestampMarker.delete() + } + log.info("daemons : ${aliveWithMetadata.map { it.daemon::class.java.name }}") + log.info("aliveWithMetadata: ${aliveWithMetadata.map { it.daemon::class.java.name }}") + val comparator = compareBy(DaemonJVMOptionsMemoryComparator(), { it.jvmOptions }) + .thenBy { + when (it.daemon) { + is CompileServiceAsyncWrapper -> 0 + else -> 1 + } + } + .thenBy(FileAgeComparator()) { it.runFile } + val optsCopy = daemonJVMOptions.copy() + // if required options fit into fattest running daemon - return the daemon and required options with memory params set to actual ones in the daemon + aliveWithMetadata.maxWith(comparator) + ?.takeIf { daemonJVMOptions memorywiseFitsInto it.jvmOptions } + ?.let { + Pair(it.daemon, optsCopy.updateMemoryUpperBounds(it.jvmOptions)) + } + // else combine all options from running daemon to get fattest option for a new daemon to runServer + ?: Pair(null, aliveWithMetadata.fold(optsCopy, { opts, d -> opts.updateMemoryUpperBounds(d.jvmOptions) })) + } + + + private suspend fun startDaemon( + compilerId: CompilerId, + daemonJVMOptions: DaemonJVMOptions, + daemonOptions: DaemonOptions, + reportingTargets: DaemonReportingTargets + ): Boolean { + log.info("in startDaemon() - 0") + val javaExecutable = File(File(System.getProperty("java.home"), "bin"), "java") + log.info("in startDaemon() - 0.1") + val serverHostname = System.getProperty(JAVA_RMI_SERVER_HOSTNAME) ?: error("$JAVA_RMI_SERVER_HOSTNAME is not set!") + log.info("in startDaemon() - 0.2") + val platformSpecificOptions = listOf( + // hide daemon window + "-Djava.awt.headless=true", + "-D$JAVA_RMI_SERVER_HOSTNAME=$serverHostname" + ) + log.info("in startDaemon() - 0.3") + val args = listOf( + javaExecutable.absolutePath, "-cp", compilerId.compilerClasspath.joinToString(File.pathSeparator) + ) + + platformSpecificOptions + + daemonJVMOptions.mappers.flatMap { it.toArgs("-") } + + COMPILER_DAEMON_CLASS_FQN_EXPERIMENTAL + + daemonOptions.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } + + compilerId.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } + log.info("in startDaemon() - 1") + reportingTargets.report(DaemonReportCategory.DEBUG, "starting the daemon as: " + args.joinToString(" ")) + val processBuilder = ProcessBuilder(args) + log.info("in startDaemon() - 2") + processBuilder.redirectErrorStream(true) + // assuming daemon process is deaf and (mostly) silent, so do not handle streams + log.info("daemon = launchProcessWithFallback") + val daemon = + launchProcessWithFallback(processBuilder, reportingTargets, "daemon client") + + val isEchoRead = Semaphore(1) + isEchoRead.acquire() + + val stdoutThread = + thread { + try { + daemon.inputStream + .reader() + .forEachLine { + log.info("daemon_process_report : $it") + if (it == COMPILE_DAEMON_IS_READY_MESSAGE) { + GlobalScope.async { + reportingTargets.report( + DaemonReportCategory.DEBUG, + "Received the message signalling that the daemon is ready" + ) + } + isEchoRead.release() + //TODO return@forEachLine + } else { + GlobalScope.async { reportingTargets.report(DaemonReportCategory.INFO, it, "daemon") } + } + } + } finally { + daemon.inputStream.close() + daemon.outputStream.close() + daemon.errorStream.close() + isEchoRead.release() + } + } + try { + // trying to wait for process + val daemonStartupTimeout = System.getProperty(COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY)?.let { + try { + it.toLong() + } catch (e: Exception) { + reportingTargets.report( + DaemonReportCategory.INFO, + "unable to interpret $COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY property ('$it'); using default timeout $DAEMON_DEFAULT_STARTUP_TIMEOUT_MS ms" + ) + null + } + } ?: DAEMON_DEFAULT_STARTUP_TIMEOUT_MS + if (daemonOptions.runFilesPath.isNotEmpty()) { + log.info("daemonOptions.runFilesPath.isNotEmpty") + val succeeded = isEchoRead.tryAcquire(daemonStartupTimeout, TimeUnit.MILLISECONDS) + log.info("succeeded : $succeeded") + return when { + !isProcessAlive(daemon) -> { + log.info("!isProcessAlive(daemon)") + reportingTargets.report( + DaemonReportCategory.INFO, + "Daemon terminated unexpectedly with error code: ${daemon.exitValue()}" + ) + false + } + !succeeded -> { + log.info("isProcessAlive!") + reportingTargets.report(DaemonReportCategory.INFO, "Unable to get response from daemon in $daemonStartupTimeout ms") + false + } + else -> true + } + } else + log.info("!daemonOptions.runFilesPath.isNotEmpty") + // without startEcho defined waiting for max timeout + Thread.sleep(daemonStartupTimeout) + return true + } finally { + // assuming that all important output is already done, the rest should be routed to the log by the daemon itself + if (stdoutThread.isAlive) { + // TODO: find better method to stop the thread, but seems it will require asynchronous consuming of the stream + stdoutThread.stop() + } + reportingTargets.out?.flush() + } + } +} + +internal suspend fun DaemonReportingTargets.report(category: DaemonReportCategory, message: String, source: String? = null) { + val sourceMessage: String by lazy { source?.let { "[$it] $message" } ?: message } + out?.println("${category.name}: $sourceMessage") + messages?.add(DaemonReportMessage(category, sourceMessage)) + messageCollector?.let { + when (category) { + DaemonReportCategory.DEBUG -> it.report(CompilerMessageSeverity.LOGGING, sourceMessage) + DaemonReportCategory.INFO -> it.report(CompilerMessageSeverity.INFO, sourceMessage) + DaemonReportCategory.EXCEPTION -> it.report(CompilerMessageSeverity.EXCEPTION, sourceMessage) + } + } + compilerServices?.let { + when (category) { + DaemonReportCategory.DEBUG -> it.report(ReportCategory.DAEMON_MESSAGE, ReportSeverity.DEBUG, message, source) + DaemonReportCategory.INFO -> it.report(ReportCategory.DAEMON_MESSAGE, ReportSeverity.INFO, message, source) + DaemonReportCategory.EXCEPTION -> it.report(ReportCategory.EXCEPTION, ReportSeverity.ERROR, message, source) + } + } +} + +internal fun isProcessAlive(process: Process) = + try { + process.exitValue() + false + } catch (e: IllegalThreadStateException) { + true + } diff --git a/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/KotlinRemoteReplCompilerClientAsync.kt b/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/KotlinRemoteReplCompilerClientAsync.kt new file mode 100644 index 00000000000..12b10c6c3b2 --- /dev/null +++ b/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/KotlinRemoteReplCompilerClientAsync.kt @@ -0,0 +1,79 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.client.experimental + +import kotlinx.coroutines.runBlocking +import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.cli.common.repl.* +import org.jetbrains.kotlin.daemon.client.KotlinRemoteReplCompilerClient +import org.jetbrains.kotlin.daemon.common.* +import org.jetbrains.kotlin.daemon.common.CompileServiceClientSide +import org.jetbrains.kotlin.daemon.common.experimental.findCallbackServerSocket +import org.jetbrains.kotlin.daemon.common.ReportCategory +import org.jetbrains.kotlin.daemon.common.ReportSeverity +import java.io.File +import java.util.concurrent.locks.ReentrantReadWriteLock +import org.jetbrains.kotlin.daemon.client.RemoteReplCompilerState + +// TODO: reduce number of ports used then SOCKET_ANY_FREE_PORT is passed (same problem with other calls) + +open class KotlinRemoteReplCompilerClientAsync( + protected val compileService: CompileServiceAsync, + clientAliveFlagFile: File?, + targetPlatform: CompileService.TargetPlatform, + args: Array, + messageCollector: MessageCollector, + templateClasspath: List, + templateClassName: String +) : ReplCompiler { + val services = BasicCompilerServicesWithResultsFacadeServerServerSide( + messageCollector, + null, + findCallbackServerSocket() + ) + + val sessionId = runBlocking { + compileService.leaseReplSession( + clientAliveFlagFile?.absolutePath, + args, + CompilationOptions( + CompilerMode.NON_INCREMENTAL_COMPILER, + targetPlatform, + arrayOf(ReportCategory.COMPILER_MESSAGE.code, ReportCategory.DAEMON_MESSAGE.code, ReportCategory.EXCEPTION.code, ReportCategory.OUTPUT_MESSAGE.code), + ReportSeverity.INFO.code, + emptyArray()), + services.clientSide, + templateClasspath, + templateClassName + ).get() + } + + // dispose should be called at the end of the repl lifetime to free daemon repl session and appropriate resources + open fun dispose() { + runBlocking { + try { + compileService.releaseReplSession(sessionId) + } catch (ex: java.lang.Exception) { + // assuming that communication failed and daemon most likely is already down + } + } + } + + override fun createState(lock: ReentrantReadWriteLock): IReplStageState<*> = + RemoteReplCompilerStateAsync( + runBlocking { compileService.replCreateState(sessionId).get() }, + lock + ) + + override fun check(state: IReplStageState<*>, codeLine: ReplCodeLine): ReplCheckResult = runBlocking { + compileService.replCheck(sessionId, state.asState(RemoteReplCompilerStateAsync::class.java).replStateFacade.getId(), codeLine).get() + } + + override fun compile(state: IReplStageState<*>, codeLine: ReplCodeLine): ReplCompileResult = runBlocking { + compileService.replCompile(sessionId, state.asState(RemoteReplCompilerStateAsync::class.java).replStateFacade.getId(), codeLine).get() + } + +} diff --git a/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/ProcessWithFallbackAsync.kt b/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/ProcessWithFallbackAsync.kt new file mode 100644 index 00000000000..98873e2f9cf --- /dev/null +++ b/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/ProcessWithFallbackAsync.kt @@ -0,0 +1,49 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.client.experimental + +import org.jetbrains.kotlin.daemon.client.DaemonReportingTargets +import org.jetbrains.kotlin.daemon.client.NativePlatformLauncherWrapper +import org.jetbrains.kotlin.daemon.common.DaemonReportCategory +import java.io.IOException + +suspend fun launchProcessWithFallback( + processBuilder: ProcessBuilder, + reportingTargets: DaemonReportingTargets, + reportingSource: String = "process launcher" +): Process = + try { + // A separate class to delay classloading until this point, where we can catch class loading errors in case then the native lib is not in the classpath + NativePlatformLauncherWrapper().launch(processBuilder) + } catch (e: UnsatisfiedLinkError) { + reportingTargets.report( + DaemonReportCategory.DEBUG, + "Could not start process with native process launcher, falling back to ProcessBuilder#start ($e)", + reportingSource + ) + null + } catch (e: IOException) { + reportingTargets.report( + DaemonReportCategory.DEBUG, + "Could not start process with native process launcher, falling back to ProcessBuilder#start (${e.cause})", + reportingSource + ) + null + } catch (e: NoClassDefFoundError) { + reportingTargets.report( + DaemonReportCategory.DEBUG, + "net.rubygrapefruit.platform library is not in the classpath, falling back to ProcessBuilder#start ($e)", + reportingSource + ) + null + } catch (e: ClassNotFoundException) { + reportingTargets.report( + DaemonReportCategory.DEBUG, + "net.rubygrapefruit.platform library is not in the classpath, falling back to ProcessBuilder#start ($e)", + reportingSource + ) + null + } ?: processBuilder.start() diff --git a/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/ReplCompilerStateAsync.kt b/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/ReplCompilerStateAsync.kt new file mode 100644 index 00000000000..a5ab5b46ca2 --- /dev/null +++ b/compiler/daemon/daemon-client-new/src/org/jetbrains/kotlin/daemon/client/experimental/ReplCompilerStateAsync.kt @@ -0,0 +1,64 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.client.experimental + +import kotlinx.coroutines.runBlocking +import org.jetbrains.kotlin.cli.common.repl.* +import org.jetbrains.kotlin.daemon.client.RemoteReplCompilerStateHistory +import org.jetbrains.kotlin.daemon.common.ReplStateFacadeAsync +import org.jetbrains.kotlin.daemon.common.experimental.ReplStateFacadeClientSide +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.locks.ReentrantReadWriteLock +import org.jetbrains.kotlin.daemon.client.RemoteReplCompilerState +import org.jetbrains.kotlin.daemon.common.toRMI + + +class RemoteReplCompilerStateHistoryAsync(private val state: RemoteReplCompilerStateAsync) : IReplStageHistory, + AbstractList>() { + override val size: Int + get() = runBlocking { state.replStateFacade.getHistorySize() } + + override fun get(index: Int): ReplHistoryRecord = runBlocking { + ReplHistoryRecord(state.replStateFacade.historyGet(index), Unit) + } + + override fun push(id: ILineId, item: Unit) { + throw NotImplementedError("push to remote history is not supported") + } + + override fun pop(): ReplHistoryRecord? { + throw NotImplementedError("pop from remote history is not supported") + } + + override fun reset(): Iterable = runBlocking { + state.replStateFacade.historyReset().apply { + currentGeneration.incrementAndGet() + } + } + + override fun resetTo(id: ILineId): Iterable = runBlocking { + state.replStateFacade.historyResetTo(id).apply { + currentGeneration.incrementAndGet() + } + } + + val currentGeneration = AtomicInteger(REPL_CODE_LINE_FIRST_GEN) + + override val lock: ReentrantReadWriteLock get() = state.lock +} + +class RemoteReplCompilerStateAsync( + internal val replStateFacade: ReplStateFacadeAsync, + override val lock: ReentrantReadWriteLock = ReentrantReadWriteLock() +) : IReplStageState { + + override val currentGeneration: Int get() = (history as RemoteReplCompilerStateHistory).currentGeneration.get() + + override val history: IReplStageHistory = + RemoteReplCompilerStateHistoryAsync(this) + + fun toRMI() = RemoteReplCompilerState(replStateFacade.toRMI(), lock) +} \ No newline at end of file diff --git a/compiler/daemon/daemon-client/build.gradle.kts b/compiler/daemon/daemon-client/build.gradle.kts index 3bc66359eee..293453f43e1 100644 --- a/compiler/daemon/daemon-client/build.gradle.kts +++ b/compiler/daemon/daemon-client/build.gradle.kts @@ -34,6 +34,10 @@ dependencies { nativePlatformVariants.forEach { embedded(commonDep("net.rubygrapefruit", "native-platform", "-$it")) } + runtime(project(":kotlin-reflect")) + compile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-jdk8")) { + isTransitive = false + } } sourceSets { diff --git a/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/KotlinCompilerDaemonClient.kt b/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/KotlinCompilerDaemonClient.kt new file mode 100644 index 00000000000..a149936d039 --- /dev/null +++ b/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/KotlinCompilerDaemonClient.kt @@ -0,0 +1,105 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.client + +import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.daemon.client.KotlinCompilerDaemonClient.Companion.instantiate +import org.jetbrains.kotlin.daemon.common.* +import org.jetbrains.kotlin.daemon.common.ReportSeverity +import org.jetbrains.kotlin.daemon.common.DummyProfiler +import org.jetbrains.kotlin.daemon.common.Profiler +import java.io.File + +data class CompileServiceSessionAsync(val compileService: CompileServiceAsync, val sessionId: Int) + +fun CompileServiceSession.toAsync() = CompileServiceSessionAsync(this.compileService.toClient(), this.sessionId) +fun CompileServiceSessionAsync.toRMI() = CompileServiceSession(this.compileService.toRMI(), this.sessionId) + +interface KotlinCompilerDaemonClient { + suspend fun connectToCompileService( + compilerId: CompilerId, + daemonJVMOptions: DaemonJVMOptions, + daemonOptions: DaemonOptions, + reportingTargets: DaemonReportingTargets, + autostart: Boolean, + checkId: Boolean + ): CompileServiceAsync? + + suspend fun connectToCompileService( + compilerId: CompilerId, + clientAliveFlagFile: File, + daemonJVMOptions: DaemonJVMOptions, + daemonOptions: DaemonOptions, + reportingTargets: DaemonReportingTargets, + autostart: Boolean + ): CompileServiceAsync? + + suspend fun connectAndLease( + compilerId: CompilerId, + clientAliveFlagFile: File, + daemonJVMOptions: DaemonJVMOptions, + daemonOptions: DaemonOptions, + reportingTargets: DaemonReportingTargets, + autostart: Boolean, + leaseSession: Boolean, + sessionAliveFlagFile: File? = null + ): CompileServiceSessionAsync? + + suspend fun shutdownCompileService(compilerId: CompilerId, daemonOptions: DaemonOptions) + + suspend fun leaseCompileSession(compilerService: CompileServiceAsync, aliveFlagPath: String?): Int + suspend fun releaseCompileSession(compilerService: CompileServiceAsync, sessionId: Int): Unit + suspend fun compile( + compilerService: CompileServiceAsync, + sessionId: Int, + targetPlatform: CompileService.TargetPlatform, + args: Array, + messageCollector: MessageCollector, + outputsCollector: ((File, List) -> Unit)? = null, + compilerMode: CompilerMode = CompilerMode.NON_INCREMENTAL_COMPILER, + reportSeverity: ReportSeverity = ReportSeverity.INFO, + port: Int = SOCKET_ANY_FREE_PORT, + profiler: Profiler = DummyProfiler() + ): Int + + fun getOrCreateClientFlagFile(daemonOptions: DaemonOptions): File + + fun createCompResults(): CompilationResultsAsync + + fun main(vararg args: String) + + companion object { + fun instantiate(daemonProtocolVariant: DaemonProtocolVariant): KotlinCompilerDaemonClient = + KotlinCompilerDaemonClient::class.java + .classLoader + .loadClass( + when(daemonProtocolVariant) { + DaemonProtocolVariant.RMI -> "org.jetbrains.kotlin.daemon.client.impls.KotlinCompilerClientImpl" + DaemonProtocolVariant.SOCKETS -> "org.jetbrains.kotlin.daemon.client.experimental.KotlinCompilerClient" + } + ) + .newInstance() as KotlinCompilerDaemonClient + } + +} + +object KotlinCompilerClientInstance { + + const val RMI_FLAG = "-old" + const val SOCKETS_FLAG = "-new_with_sockets" + + @JvmStatic + fun main(vararg args: String) { + val clientInstance: KotlinCompilerDaemonClient? = when (args.last()) { + SOCKETS_FLAG -> + instantiate(DaemonProtocolVariant.SOCKETS) + else -> + instantiate(DaemonProtocolVariant.RMI) + } + clientInstance?.main(*args.sliceArray(0..args.lastIndex)) + } + +} \ No newline at end of file diff --git a/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/NativePlatformUtil.kt b/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/NativePlatformUtil.kt index 317c470b342..8b8d2142e48 100644 --- a/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/NativePlatformUtil.kt +++ b/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/NativePlatformUtil.kt @@ -19,40 +19,40 @@ package org.jetbrains.kotlin.daemon.client import org.jetbrains.kotlin.daemon.common.DaemonReportCategory import java.io.IOException -private class NativePlatformLauncherWrapper { +class NativePlatformLauncherWrapper { private val nativeLauncher: net.rubygrapefruit.platform.ProcessLauncher by lazy { net.rubygrapefruit.platform.Native.get(net.rubygrapefruit.platform.ProcessLauncher::class.java) } fun launch(processBuilder: ProcessBuilder): Process = - try { - nativeLauncher.start(processBuilder) - } - catch (e: net.rubygrapefruit.platform.NativeException) { - throw IOException(e) - } + try { + nativeLauncher.start(processBuilder) + } + catch (e: net.rubygrapefruit.platform.NativeException) { + throw IOException(e) + } } fun launchProcessWithFallback(processBuilder: ProcessBuilder, reportingTargets: DaemonReportingTargets, reportingSource: String = "process launcher"): Process = - try { - // A separate class to delay classloading until this point, where we can catch class loading errors in case then the native lib is not in the classpath - NativePlatformLauncherWrapper().launch(processBuilder) - } - catch (e: UnsatisfiedLinkError) { - reportingTargets.report(DaemonReportCategory.DEBUG, "Could not start process with native process launcher, falling back to ProcessBuilder#start ($e)", reportingSource) - null - } - catch (e: IOException) { - reportingTargets.report(DaemonReportCategory.DEBUG, "Could not start process with native process launcher, falling back to ProcessBuilder#start (${e.cause})", reportingSource) - null - } - catch (e: NoClassDefFoundError) { - reportingTargets.report(DaemonReportCategory.DEBUG, "net.rubygrapefruit.platform library is not in the classpath, falling back to ProcessBuilder#start ($e)", reportingSource) - null - } - catch (e: ClassNotFoundException) { - reportingTargets.report(DaemonReportCategory.DEBUG, "net.rubygrapefruit.platform library is not in the classpath, falling back to ProcessBuilder#start ($e)", reportingSource) - null - } + try { + // A separate class to delay classloading until this point, where we can catch class loading errors in case then the native lib is not in the classpath + NativePlatformLauncherWrapper().launch(processBuilder) + } + catch (e: UnsatisfiedLinkError) { + reportingTargets.report(DaemonReportCategory.DEBUG, "Could not start process with native process launcher, falling back to ProcessBuilder#start ($e)", reportingSource) + null + } + catch (e: IOException) { + reportingTargets.report(DaemonReportCategory.DEBUG, "Could not start process with native process launcher, falling back to ProcessBuilder#start (${e.cause})", reportingSource) + null + } + catch (e: NoClassDefFoundError) { + reportingTargets.report(DaemonReportCategory.DEBUG, "net.rubygrapefruit.platform library is not in the classpath, falling back to ProcessBuilder#start ($e)", reportingSource) + null + } + catch (e: ClassNotFoundException) { + reportingTargets.report(DaemonReportCategory.DEBUG, "net.rubygrapefruit.platform library is not in the classpath, falling back to ProcessBuilder#start ($e)", reportingSource) + null + } ?: processBuilder.start() diff --git a/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/RemoteReplCompilerState.kt b/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/RemoteReplCompilerState.kt index 3129b755dff..65c7aa6ce2a 100644 --- a/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/RemoteReplCompilerState.kt +++ b/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/RemoteReplCompilerState.kt @@ -50,7 +50,10 @@ class RemoteReplCompilerStateHistory(private val state: RemoteReplCompilerState) override val lock: ReentrantReadWriteLock get() = state.lock } -class RemoteReplCompilerState(internal val replStateFacade: ReplStateFacade, override val lock: ReentrantReadWriteLock = ReentrantReadWriteLock()) : IReplStageState { +class RemoteReplCompilerState( + internal val replStateFacade: ReplStateFacade, + override val lock: ReentrantReadWriteLock = ReentrantReadWriteLock() +) : IReplStageState { override val currentGeneration: Int get() = (history as RemoteReplCompilerStateHistory).currentGeneration.get() diff --git a/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/impls/KotlinCompilerClientImpl.kt b/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/impls/KotlinCompilerClientImpl.kt new file mode 100644 index 00000000000..73feef5db77 --- /dev/null +++ b/compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/impls/KotlinCompilerClientImpl.kt @@ -0,0 +1,125 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.client.impls + +import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.daemon.client.* +import org.jetbrains.kotlin.daemon.common.* +import org.jetbrains.kotlin.daemon.common.ReportSeverity +import org.jetbrains.kotlin.daemon.common.Profiler +import java.io.File + +object KotlinCompilerClientImpl : KotlinCompilerDaemonClient { + + val oldClient = KotlinCompilerClient + + override suspend fun connectToCompileService( + compilerId: CompilerId, + daemonJVMOptions: DaemonJVMOptions, + daemonOptions: DaemonOptions, + reportingTargets: DaemonReportingTargets, + autostart: Boolean, + checkId: Boolean + ): CompileServiceAsync? = oldClient.connectToCompileService( + compilerId, + daemonJVMOptions, + daemonOptions, + reportingTargets, + autostart, + checkId + )?.toClient() + + override suspend fun connectToCompileService( + compilerId: CompilerId, + clientAliveFlagFile: File, + daemonJVMOptions: DaemonJVMOptions, + daemonOptions: DaemonOptions, + reportingTargets: DaemonReportingTargets, + autostart: Boolean + ): CompileServiceAsync? = oldClient.connectToCompileService( + compilerId, + clientAliveFlagFile, + daemonJVMOptions, + daemonOptions, + reportingTargets, + autostart + )?.toClient() + + override suspend fun connectAndLease( + compilerId: CompilerId, + clientAliveFlagFile: File, + daemonJVMOptions: DaemonJVMOptions, + daemonOptions: DaemonOptions, + reportingTargets: DaemonReportingTargets, + autostart: Boolean, + leaseSession: Boolean, + sessionAliveFlagFile: File? + ): CompileServiceSessionAsync? = oldClient.connectAndLease( + compilerId, + clientAliveFlagFile, + daemonJVMOptions, + daemonOptions, + reportingTargets, + autostart, + leaseSession, + sessionAliveFlagFile + )?.toAsync() + + override suspend fun shutdownCompileService( + compilerId: CompilerId, + daemonOptions: DaemonOptions + ) = oldClient.shutdownCompileService( + compilerId, + daemonOptions + ) + + override suspend fun leaseCompileSession( + compilerService: CompileServiceAsync, + aliveFlagPath: String? + ) = oldClient.leaseCompileSession( + compilerService.toRMI(), + aliveFlagPath + ) + + override suspend fun releaseCompileSession( + compilerService: CompileServiceAsync, + sessionId: Int + ) = oldClient.releaseCompileSession( + compilerService.toRMI(), + sessionId + ) + + override suspend fun compile( + compilerService: CompileServiceAsync, + sessionId: Int, + targetPlatform: CompileService.TargetPlatform, + args: Array, + messageCollector: MessageCollector, + outputsCollector: ((File, List) -> Unit)?, + compilerMode: CompilerMode, + reportSeverity: ReportSeverity, + port: Int, + profiler: Profiler + ): Int = oldClient.compile( + compilerService.toRMI(), + sessionId, + targetPlatform, + args, + messageCollector, + outputsCollector, + compilerMode, + reportSeverity, + port, + profiler + ) + + override fun getOrCreateClientFlagFile(daemonOptions: DaemonOptions): File = oldClient.getOrCreateClientFlagFile(daemonOptions) + + override fun createCompResults() = TODO("not implemented") + + override fun main(vararg args: String) = oldClient.main(*args) + +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/build.gradle.kts b/compiler/daemon/daemon-common-new/build.gradle.kts new file mode 100644 index 00000000000..295af070405 --- /dev/null +++ b/compiler/daemon/daemon-common-new/build.gradle.kts @@ -0,0 +1,46 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +import com.sun.javafx.scene.CameraHelper.project + +plugins { + java + kotlin("jvm") + id("jps-compatible") +} + +jvmTarget = "1.6" + +val ktorExcludesForDaemon : List> by rootProject.extra + +dependencies { + compile(project(":core:descriptors")) + compile(project(":core:descriptors.jvm")) + compile(project(":compiler:util")) + compile(project(":compiler:cli-common")) + compileOnly(project(":compiler:daemon-common")) + compile(kotlinStdlib()) + compileOnly(project(":js:js.frontend")) + compileOnly(intellijCoreDep()) { includeJars("intellij-core") } + compileOnly(intellijDep()) { includeIntellijCoreJarDependencies(project) } + compile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-jdk8")) { + isTransitive = false + } + compile(commonDep("io.ktor", "ktor-network")) { + ktorExcludesForDaemon.forEach { (group, module) -> + exclude(group = group, module = module) + } + } +} + +sourceSets { + "main" { projectDefault() } + "test" {} +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/AdditionalDaemonParams.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/AdditionalDaemonParams.kt new file mode 100644 index 00000000000..f35d71c3c4b --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/AdditionalDaemonParams.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import java.util.concurrent.TimeUnit + +val RMI_WRAPPER_PORTS_RANGE_START: Int = 13001 +val RMI_WRAPPER_PORTS_RANGE_END: Int = 14000 + +val REPL_SERVER_PORTS_RANGE_START: Int = 14001 +val REPL_SERVER_PORTS_RANGE_END: Int = 15000 + + +val CALLBACK_SERVER_PORTS_RANGE_START: Int = 15001 +val CALLBACK_SERVER_PORTS_RANGE_END: Int = 16000 + +val RESULTS_SERVER_PORTS_RANGE_START: Int = 16001 +val RESULTS_SERVER_PORTS_RANGE_END: Int = 17000 + +val COMPILER_DAEMON_CLASS_FQN_EXPERIMENTAL: String = "org.jetbrains.kotlin.daemon.experimental.KotlinCompileDaemon" + +val FIRST_HANDSHAKE_BYTE_TOKEN = byteArrayOf(1, 2, 3, 4) +val AUTH_TIMEOUT_IN_MILLISECONDS = 200L + +val DAEMON_PERIODIC_CHECK_INTERVAL_MS = 1000L +val DAEMON_PERIODIC_SELDOM_CHECK_INTERVAL_MS = 60000L + +val KEEPALIVE_PERIOD = 2000L +val KEEPALIVE_PERIOD_SERVER = 4000L \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/ClientUtils.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/ClientUtils.kt new file mode 100644 index 00000000000..40d025953a9 --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/ClientUtils.kt @@ -0,0 +1,153 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + + +import kotlinx.coroutines.* +import org.jetbrains.kotlin.daemon.common.* +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.runWithTimeout +import org.jetbrains.kotlin.daemon.common.DaemonReportCategory +import org.jetbrains.kotlin.daemon.common.makePortFromRunFilenameExtractor +import java.io.File +import java.rmi.registry.LocateRegistry +import java.util.logging.Logger + +/* +1) walkDaemonsAsync = walkDaemons + some async calls inside (also some used classes changed *** -> ***Async) +2) tryConnectToDaemonBySockets / tryConnectToDaemonByRMI + +*/ + +internal val MAX_PORT_NUMBER = 0xffff + +private const val ORPHANED_RUN_FILE_AGE_THRESHOLD_MS = 1000000L + +data class DaemonWithMetadataAsync(val daemon: CompileServiceAsync, val runFile: File, val jvmOptions: DaemonJVMOptions) + +val log = Logger.getLogger("client utils") + +// TODO: replace mapNotNull in walkDaemonsAsync with this method. +private suspend fun List.mapNotNullAsync(transform: suspend (T) -> R?): List = + this + .map { GlobalScope.async { transform(it) } } + .mapNotNull { it.await() } // await for completion of the last action + + +// TODO: write metadata into discovery file to speed up selection +// TODO: consider using compiler jar signature (checksum) as a CompilerID (plus java version, plus ???) instead of classpath checksum +// would allow to use same compiler from taken different locations +// reqs: check that plugins (or anything els) should not be part of the CP +suspend fun walkDaemonsAsync( + registryDir: File, + compilerId: CompilerId, + fileToCompareTimestamp: File, + filter: (File, Int) -> Boolean = { _, _ -> true }, + report: (DaemonReportCategory, String) -> Unit = { _, _ -> }, + useRMI: Boolean = true, + useSockets: Boolean = true +): List { // TODO: replace with Deferred> and use mapNotNullAsync to speed this up + val classPathDigest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest().toHexString() + val portExtractor = makePortFromRunFilenameExtractor(classPathDigest) + return registryDir.walk().toList() // list, since walk returns Sequence and Sequence.map{...} is not inline => coroutines dont work + .map { Pair(it, portExtractor(it.name)) } + .filter { (file, port) -> port != null && filter(file, port) } + .mapNotNull { (file, port) -> + // all actions process concurrently + assert(port!! in 1..(MAX_PORT_NUMBER - 1)) + val relativeAge = fileToCompareTimestamp.lastModified() - file.lastModified() + val daemon = tryConnectToDaemonAsync(port, report, file, useRMI, useSockets) + // cleaning orphaned file; note: daemon should shut itself down if it detects that the runServer file is deleted + if (daemon == null) { + if (relativeAge - ORPHANED_RUN_FILE_AGE_THRESHOLD_MS <= 0) { + report( + DaemonReportCategory.DEBUG, + "found fresh runServer file '${file.absolutePath}' ($relativeAge ms old), but no daemon, ignoring it" + ) + } else { + report( + DaemonReportCategory.DEBUG, + "found seemingly orphaned runServer file '${file.absolutePath}' ($relativeAge ms old), deleting it" + ) + if (!file.delete()) { + report( + DaemonReportCategory.INFO, + "WARNING: unable to delete seemingly orphaned file '${file.absolutePath}', cleanup recommended" + ) + } + } + } + try { + daemon?.let { + DaemonWithMetadataAsync(it, file, it.getDaemonJVMOptions().get()) + } + } catch (e: Exception) { + report( + DaemonReportCategory.INFO, + "ERROR: unable to retrieve daemon JVM options, assuming daemon is dead: ${e.message}" + ) + null + } + } +} + +private inline fun tryConnectToDaemonByRMI(port: Int, report: (DaemonReportCategory, String) -> Unit): CompileServiceAsync? { + try { + log.info("tryConnectToDaemonByRMI(port = $port)") + val daemon = runBlocking { + runWithTimeout(2 * DAEMON_PERIODIC_CHECK_INTERVAL_MS) { + LocateRegistry.getRegistry( + LoopbackNetworkInterface.loopbackInetAddressName, + port, + LoopbackNetworkInterface.clientLoopbackSocketFactoryRMI + )?.lookup(COMPILER_SERVICE_RMI_NAME) + } + } + when (daemon) { + null -> report(DaemonReportCategory.INFO, "daemon not found") + is CompileService -> return daemon.toClient() + else -> report(DaemonReportCategory.INFO, "Unable to cast compiler service, actual class received: ${daemon::class.java.name}") + } + } catch (e: Throwable) { + report(DaemonReportCategory.INFO, "cannot connect to registry: " + (e.cause?.message ?: e.message ?: "unknown error")) + } + return null +} + +private suspend fun tryConnectToDaemonBySockets( + port: Int, + file: File, + report: (DaemonReportCategory, String) -> Unit +): CompileServiceClientSide? { + return CompileServiceClientSideImpl( + port, + LoopbackNetworkInterface.loopbackInetAddressName, + file + ).let { daemon -> + try { + log.info("tryConnectToDaemonBySockets(port = $port)") + log.info("daemon($port) = $daemon") + log.info("daemon($port) connecting to server...") + daemon.connectToServer() + log.info("OK - daemon($port) connected to server!!!") + daemon + } catch (e: Throwable) { + report(DaemonReportCategory.INFO, "kcannot find or connect to socket") + daemon.close() + null + } + } +} + +private suspend fun tryConnectToDaemonAsync( + port: Int, + report: (DaemonReportCategory, String) -> Unit, + file: File, + useRMI: Boolean = true, + useSockets: Boolean = true +): CompileServiceAsync? = + useSockets.takeIf { it }?.let { tryConnectToDaemonBySockets(port, file, report) } + ?: (useRMI.takeIf { it }?.let { tryConnectToDaemonByRMI(port, report) }) + diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompilationResultsAsyncImpls.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompilationResultsAsyncImpls.kt new file mode 100644 index 00000000000..6cd91888451 --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompilationResultsAsyncImpls.kt @@ -0,0 +1,42 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import org.jetbrains.kotlin.daemon.common.CompilationResultsAsync +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Client +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.DefaultClient +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Server +import java.io.Serializable + +interface CompilationResultsServerSide : CompilationResultsAsync, Server { + class AddMessage( + val compilationResultCategory: Int, + val value: Serializable + ) : Server.Message() { + override suspend fun processImpl(server: CompilationResultsServerSide, sendReply: (Any?) -> Unit) { + server.add(compilationResultCategory, value) + } + } +} + +interface CompilationResultsClientSide : CompilationResultsAsync, Client + +class CompilationResultsClientSideImpl(val socketPort: Int) : CompilationResultsClientSide, + Client by DefaultClient(socketPort) { + + override val clientSide: CompilationResultsAsync + get() = this + + override suspend fun add(compilationResultCategory: Int, value: Serializable) { + sendMessage(CompilationResultsServerSide.AddMessage(compilationResultCategory, value)) + } + + // TODO: consider connecting to server in init-block +} + +enum class CompilationResultCategory(val code: Int) { + IC_COMPILE_ITERATION(0) +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompileServiceClientSide.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompileServiceClientSide.kt new file mode 100644 index 00000000000..e74b68230b8 --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompileServiceClientSide.kt @@ -0,0 +1,15 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +@file:Suppress("EXPERIMENTAL_FEATURE_WARNING") + +package org.jetbrains.kotlin.daemon.common + +import org.jetbrains.kotlin.daemon.common.experimental.CompileServiceServerSide +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.* + +interface CompileServiceClientSide : CompileServiceAsync, Client { + override val serverPort: Int +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompileServiceClientSideImpl.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompileServiceClientSideImpl.kt new file mode 100644 index 00000000000..e13d12550ac --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompileServiceClientSideImpl.kt @@ -0,0 +1,393 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import kotlinx.coroutines.* +import org.jetbrains.kotlin.cli.common.repl.ReplCheckResult +import org.jetbrains.kotlin.cli.common.repl.ReplCodeLine +import org.jetbrains.kotlin.cli.common.repl.ReplCompileResult +import org.jetbrains.kotlin.daemon.common.* +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.* +import org.jetbrains.kotlin.daemon.common.CompileService +import org.jetbrains.kotlin.daemon.common.CompilerServicesFacadeBase +import java.io.File +import java.util.logging.Logger + +class CompileServiceClientSideImpl( + override val serverPort: Int, + val serverHost: String, + val serverFile: File +) : CompileServiceClientSide, + Client by object : DefaultAuthorizableClient( + serverPort, + serverHost + ) { + + private fun nowMillieconds() = System.currentTimeMillis() + + @Volatile + private var lastUsedMilliSeconds: Long = nowMillieconds() + + private fun millisecondsSinceLastUsed() = nowMillieconds() - lastUsedMilliSeconds + + private fun keepAliveSuccess() = millisecondsSinceLastUsed() < KEEPALIVE_PERIOD + + override suspend fun authorizeOnServer(serverOutputChannel: ByteWriteChannelWrapper): Boolean = + runWithTimeout { + val signature = serverFile.inputStream().use(::readTokenKeyPairAndSign) + sendSignature(serverOutputChannel, signature) + true + } ?: false + + override suspend fun clientHandshake(input: ByteReadChannelWrapper, output: ByteWriteChannelWrapper, log: Logger): Boolean { + return trySendHandshakeMessage(output, log) && tryAcquireHandshakeMessage(input, log) + } + + override fun startKeepAlives() { + val keepAliveMessage = Server.KeepAliveMessage() + GlobalScope.async(newSingleThreadContext("keepAliveThread")) { + delay(KEEPALIVE_PERIOD * 4) + while (true) { + delay(KEEPALIVE_PERIOD) + while (keepAliveSuccess()) { + delay(KEEPALIVE_PERIOD - millisecondsSinceLastUsed()) + } + runWithTimeout(timeout = KEEPALIVE_PERIOD / 2) { + val id = sendMessage(keepAliveMessage) + readMessage>(id) + } ?: if (!keepAliveSuccess()) readActor.send(StopAllRequests()).also { + } + } + } + } + + override fun delayKeepAlives() { + lastUsedMilliSeconds = nowMillieconds() + } + + } { + override suspend fun classesFqNamesByFiles(sessionId: Int, sourceFiles: Set): CompileService.CallResult> { + val id = sendMessage(ClassesFqNamesByFilesMessage(sessionId, sourceFiles)) + return readMessage(id) + } + + override suspend fun compile( + sessionId: Int, + compilerArguments: Array, + compilationOptions: CompilationOptions, + servicesFacade: CompilerServicesFacadeBaseAsync, + compilationResults: CompilationResultsAsync? + ): CompileService.CallResult { + val id = sendMessage(CompileMessage( + sessionId, + compilerArguments, + compilationOptions, + servicesFacade, + compilationResults + )) + return readMessage(id) + } + + override suspend fun leaseReplSession( + aliveFlagPath: String?, + compilerArguments: Array, + compilationOptions: CompilationOptions, + servicesFacade: CompilerServicesFacadeBaseAsync, + templateClasspath: List, + templateClassName: String + ): CompileService.CallResult { + val id = sendMessage( + LeaseReplSessionMessage( + aliveFlagPath, + compilerArguments, + compilationOptions, + servicesFacade, + templateClasspath, + templateClassName + ) + ) + return readMessage(id) + } + + // CompileService methods: + + override suspend fun checkCompilerId(expectedCompilerId: CompilerId): Boolean { + val id = sendMessage( + CheckCompilerIdMessage( + expectedCompilerId + ) + ) + return readMessage(id) + } + + override suspend fun getUsedMemory(): CompileService.CallResult { + val id = sendMessage(GetUsedMemoryMessage()) + return readMessage(id) + } + + + override suspend fun getDaemonOptions(): CompileService.CallResult { + val id = sendMessage(GetDaemonOptionsMessage()) + return readMessage(id) + } + + override suspend fun getDaemonInfo(): CompileService.CallResult { + val id = sendMessage(GetDaemonInfoMessage()) + return readMessage(id) + } + + override suspend fun getDaemonJVMOptions(): CompileService.CallResult { + val id = sendMessage(GetDaemonJVMOptionsMessage()) + val res = readMessage>(id) + return res + } + + override suspend fun registerClient(aliveFlagPath: String?): CompileService.CallResult { + val id = sendMessage(RegisterClientMessage(aliveFlagPath)) + return readMessage(id) + } + + override suspend fun getClients(): CompileService.CallResult> { + val id = sendMessage(GetClientsMessage()) + return readMessage(id) + } + + override suspend fun leaseCompileSession(aliveFlagPath: String?): CompileService.CallResult { + val id = sendMessage( + LeaseCompileSessionMessage( + aliveFlagPath + ) + ) + return readMessage(id) + } + + override suspend fun releaseCompileSession(sessionId: Int): CompileService.CallResult { + val id = sendMessage( + ReleaseCompileSessionMessage( + sessionId + ) + ) + return readMessage(id) + } + + override suspend fun shutdown(): CompileService.CallResult { + val id = sendMessage(ShutdownMessage()) + val res = readMessage>(id) + return res + } + + override suspend fun scheduleShutdown(graceful: Boolean): CompileService.CallResult { + val id = sendMessage(ScheduleShutdownMessage(graceful)) + return readMessage(id) + } + + override suspend fun clearJarCache() { + val id = sendMessage(ClearJarCacheMessage()) + } + + override suspend fun releaseReplSession(sessionId: Int): CompileService.CallResult { + val id = sendMessage(ReleaseReplSessionMessage(sessionId)) + return readMessage(id) + } + + override suspend fun replCreateState(sessionId: Int): CompileService.CallResult { + val id = sendMessage(ReplCreateStateMessage(sessionId)) + return readMessage(id) + } + + override suspend fun replCheck( + sessionId: Int, + replStateId: Int, + codeLine: ReplCodeLine + ): CompileService.CallResult { + val id = sendMessage( + ReplCheckMessage( + sessionId, + replStateId, + codeLine + ) + ) + return readMessage(id) + } + + override suspend fun replCompile( + sessionId: Int, + replStateId: Int, + codeLine: ReplCodeLine + ): CompileService.CallResult { + val id = sendMessage( + ReplCompileMessage( + sessionId, + replStateId, + codeLine + ) + ) + return readMessage(id) + } + + // Query messages: + + class CheckCompilerIdMessage(val expectedCompilerId: CompilerId) : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.checkCompilerId(expectedCompilerId)) + } + + class GetUsedMemoryMessage : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.getUsedMemory()) + } + + class GetDaemonOptionsMessage : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.getDaemonOptions()) + } + + class GetDaemonJVMOptionsMessage : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.getDaemonJVMOptions()) + } + + class GetDaemonInfoMessage : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.getDaemonInfo()) + } + + class RegisterClientMessage(val aliveFlagPath: String?) : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.registerClient(aliveFlagPath)) + } + + + class GetClientsMessage : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.getClients()) + } + + class LeaseCompileSessionMessage(val aliveFlagPath: String?) : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.leaseCompileSession(aliveFlagPath)) + } + + class ReleaseCompileSessionMessage(val sessionId: Int) : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.releaseCompileSession(sessionId)) + } + + class ShutdownMessage : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.shutdown()) + } + + class ScheduleShutdownMessage(val graceful: Boolean) : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.scheduleShutdown(graceful)) + } + + class CompileMessage( + val sessionId: Int, + val compilerArguments: Array, + val compilationOptions: CompilationOptions, + val servicesFacade: CompilerServicesFacadeBaseAsync, + val compilationResults: CompilationResultsAsync? + ) : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply( + server.compile( + sessionId, + compilerArguments, + compilationOptions, + servicesFacade, + compilationResults + ) + ) + } + + class ClassesFqNamesByFilesMessage( + val sessionId: Int, + val sourceFiles: Set + ) : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply( + server.classesFqNamesByFiles(sessionId, sourceFiles) + ) + } + + class ClearJarCacheMessage : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + server.clearJarCache() + } + + class LeaseReplSessionMessage( + val aliveFlagPath: String?, + val compilerArguments: Array, + val compilationOptions: CompilationOptions, + val servicesFacade: CompilerServicesFacadeBaseAsync, + val templateClasspath: List, + val templateClassName: String + ) : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply( + server.leaseReplSession( + aliveFlagPath, + compilerArguments, + compilationOptions, + servicesFacade, + templateClasspath, + templateClassName + ) + ) + } + + class ReleaseReplSessionMessage(val sessionId: Int) : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.releaseReplSession(sessionId)) + } + + class LeaseReplSession_Short_Message( + val aliveFlagPath: String?, + val compilerArguments: Array, + val compilationOptions: CompilationOptions, + val servicesFacade: CompilerServicesFacadeBase, + val templateClasspath: List, + val templateClassName: String + ) : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply( + server.leaseReplSession( + aliveFlagPath, + compilerArguments, + compilationOptions, + servicesFacade.toClient(), + templateClasspath, + templateClassName + ) + ) + } + + class ReplCreateStateMessage(val sessionId: Int) : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.replCreateState(sessionId)) + } + + class ReplCheckMessage( + val sessionId: Int, + val replStateId: Int, + val codeLine: ReplCodeLine + ) : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.replCheck(sessionId, replStateId, codeLine)) + } + + class ReplCompileMessage( + val sessionId: Int, + val replStateId: Int, + val codeLine: ReplCodeLine + ) : Server.Message() { + override suspend fun processImpl(server: CompileServiceServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.replCompile(sessionId, replStateId, codeLine)) + } + +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompileServiceRMIWrapper.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompileServiceRMIWrapper.kt new file mode 100644 index 00000000000..612e3d1ec3c --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompileServiceRMIWrapper.kt @@ -0,0 +1,234 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import kotlinx.coroutines.runBlocking +import org.jetbrains.kotlin.cli.common.repl.ReplCodeLine +import org.jetbrains.kotlin.daemon.common.* +import org.jetbrains.kotlin.daemon.common.LoopbackNetworkInterface +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Client +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.DefaultClientRMIWrapper +import org.jetbrains.kotlin.daemon.common.* +import java.io.File +import java.io.Serializable +import java.rmi.NoSuchObjectException +import java.rmi.server.UnicastRemoteObject +import java.util.* +import java.util.logging.Logger + +class CompileServiceRMIWrapper(val server: CompileServiceServerSide, daemonOptions: DaemonOptions, compilerId: CompilerId) : + CompileService { + + override fun classesFqNamesByFiles(sessionId: Int, sourceFiles: Set) = runBlocking { + server.classesFqNamesByFiles(sessionId, sourceFiles) + } + + val log = Logger.getLogger("CompileServiceRMIWrapper") + + private fun deprecated(): Nothing = TODO("NEVER USE DEPRECATED METHODS, PLEASE!") // prints this todo message + + override fun checkCompilerId(expectedCompilerId: CompilerId) = runBlocking { + server.checkCompilerId(expectedCompilerId) + } + + override fun getUsedMemory() = runBlocking { + server.getUsedMemory() + } + + override fun getDaemonOptions() = runBlocking { + server.getDaemonOptions() + } + + override fun getDaemonInfo() = runBlocking { + server.getDaemonInfo() + } + + override fun getDaemonJVMOptions() = runBlocking { + server.getDaemonJVMOptions() + } + + override fun registerClient(aliveFlagPath: String?) = runBlocking { + server.registerClient(aliveFlagPath) + } + + override fun getClients() = runBlocking { + server.getClients() + } + + override fun leaseCompileSession(aliveFlagPath: String?) = runBlocking { + server.leaseCompileSession(aliveFlagPath) + } + + override fun releaseCompileSession(sessionId: Int) = runBlocking { + server.releaseCompileSession(sessionId) + } + + override fun shutdown() = runBlocking { + server.shutdown() + } + + override fun scheduleShutdown(graceful: Boolean) = runBlocking { + server.scheduleShutdown(graceful) + } + + override fun remoteCompile( + sessionId: Int, + targetPlatform: CompileService.TargetPlatform, + args: Array, + servicesFacade: CompilerCallbackServicesFacade, + compilerOutputStream: RemoteOutputStream, + outputFormat: CompileService.OutputFormat, + serviceOutputStream: RemoteOutputStream, + operationsTracer: RemoteOperationsTracer? + ) = deprecated() + + override fun remoteIncrementalCompile( + sessionId: Int, + targetPlatform: CompileService.TargetPlatform, + args: Array, + servicesFacade: CompilerCallbackServicesFacade, + compilerOutputStream: RemoteOutputStream, + compilerOutputFormat: CompileService.OutputFormat, + serviceOutputStream: RemoteOutputStream, + operationsTracer: RemoteOperationsTracer? + ) = deprecated() + + override fun compile( + sessionId: Int, + compilerArguments: Array, + compilationOptions: CompilationOptions, + servicesFacade: CompilerServicesFacadeBase, + compilationResults: CompilationResults? + ) = runBlocking { + server.compile( + sessionId, + compilerArguments, + compilationOptions, + servicesFacade.toClient(), + compilationResults?.toClient() ?: object : CompilationResultsClientSide, + Client by DefaultClientRMIWrapper() { + + override val clientSide: CompilationResultsAsync + get() = this + + override suspend fun add(compilationResultCategory: Int, value: Serializable) {} + } + ) + } + + override fun clearJarCache() = runBlocking { + server.clearJarCache() + } + + override fun leaseReplSession( + aliveFlagPath: String?, + targetPlatform: CompileService.TargetPlatform, + servicesFacade: CompilerCallbackServicesFacade, + templateClasspath: List, + templateClassName: String, + scriptArgs: Array?, + scriptArgsTypes: Array>?, + compilerMessagesOutputStream: RemoteOutputStream, + evalOutputStream: RemoteOutputStream?, + evalErrorStream: RemoteOutputStream?, + evalInputStream: RemoteInputStream?, + operationsTracer: RemoteOperationsTracer? + ) = deprecated() + + override fun releaseReplSession(sessionId: Int) = runBlocking { + server.releaseReplSession(sessionId) + } + + override fun remoteReplLineCheck(sessionId: Int, codeLine: ReplCodeLine) = deprecated() + + override fun remoteReplLineCompile( + sessionId: Int, + codeLine: ReplCodeLine, + history: List? + ) = deprecated() + + override fun remoteReplLineEval( + sessionId: Int, + codeLine: ReplCodeLine, + history: List? + ) = deprecated() + + override fun leaseReplSession( + aliveFlagPath: String?, + compilerArguments: Array, + compilationOptions: CompilationOptions, + servicesFacade: CompilerServicesFacadeBase, + templateClasspath: List, + templateClassName: String + ) = runBlocking { + server.leaseReplSession( + aliveFlagPath, + compilerArguments, + compilationOptions, + servicesFacade.toClient(), + templateClasspath, + templateClassName + ) + } + + override fun replCreateState(sessionId: Int) = runBlocking { + server.replCreateState(sessionId).toRMI() + } + + override fun replCheck(sessionId: Int, replStateId: Int, codeLine: ReplCodeLine) = runBlocking { + server.replCheck(sessionId, replStateId, codeLine) + } + + override fun replCompile(sessionId: Int, replStateId: Int, codeLine: ReplCodeLine) = runBlocking { + server.replCompile(sessionId, replStateId, codeLine) + } + + init { + try { + // cleanup for the case of incorrect restart and many other situations + UnicastRemoteObject.unexportObject(this, false) + } catch (e: NoSuchObjectException) { + // ignoring if object already exported + } + + val (registry, port) = findPortAndCreateRegistry( + COMPILE_DAEMON_FIND_PORT_ATTEMPTS, + RMI_WRAPPER_PORTS_RANGE_START, + RMI_WRAPPER_PORTS_RANGE_END + ) + + val stub = UnicastRemoteObject.exportObject( + this, + port, + LoopbackNetworkInterface.clientLoopbackSocketFactory, + LoopbackNetworkInterface.serverLoopbackSocketFactory + ) as CompileService + + registry.rebind(COMPILER_SERVICE_RMI_NAME, stub) + + // create file : + val runFileDir = File(daemonOptions.runFilesPathOrDefault) + runFileDir.mkdirs() + val runFile = File( + runFileDir, + makeRunFilenameString( + timestamp = "%tFT% { + override val serverPort: Int +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompilerCallbackServicesFacadeClientSide.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompilerCallbackServicesFacadeClientSide.kt new file mode 100644 index 00000000000..5f8904cf274 --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompilerCallbackServicesFacadeClientSide.kt @@ -0,0 +1,104 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import org.jetbrains.kotlin.daemon.common.CompilerCallbackServicesFacadeAsync +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Client +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.DefaultClient +import org.jetbrains.kotlin.incremental.components.LookupInfo +import org.jetbrains.kotlin.load.kotlin.incremental.components.JvmPackagePartProto +import org.jetbrains.kotlin.modules.TargetId +import java.io.Serializable + + +interface CompilerCallbackServicesFacadeClientSide : CompilerCallbackServicesFacadeAsync, Client, CompilerServicesFacadeBaseClientSide + +@Suppress("UNCHECKED_CAST") +class CompilerCallbackServicesFacadeClientSideImpl(serverPort: Int) : CompilerCallbackServicesFacadeClientSide, + Client by DefaultClient(serverPort) { + + override suspend fun hasIncrementalCaches(): Boolean { + val id = sendMessage(CompilerCallbackServicesFacadeServerSide.HasIncrementalCachesMessage()) + return readMessage(id) + } + + override suspend fun hasLookupTracker(): Boolean { + val id = sendMessage(CompilerCallbackServicesFacadeServerSide.HasLookupTrackerMessage()) + return readMessage(id) + } + + override suspend fun hasCompilationCanceledStatus(): Boolean { + val id = sendMessage(CompilerCallbackServicesFacadeServerSide.HasCompilationCanceledStatusMessage()) + return readMessage(id) + } + + override suspend fun incrementalCache_getObsoletePackageParts(target: TargetId): Collection { + val id = sendMessage(CompilerCallbackServicesFacadeServerSide.IncrementalCache_getObsoletePackagePartsMessage(target)) + return readMessage(id) + } + + override suspend fun incrementalCache_getObsoleteMultifileClassFacades(target: TargetId): Collection { + val id = sendMessage(CompilerCallbackServicesFacadeServerSide.IncrementalCache_getObsoleteMultifileClassFacadesMessage(target)) + return readMessage(id) + } + + override suspend fun incrementalCache_getPackagePartData(target: TargetId, partInternalName: String): JvmPackagePartProto? { + val id = sendMessage(CompilerCallbackServicesFacadeServerSide.IncrementalCache_getPackagePartDataMessage(target, partInternalName)) + return readMessage(id) + } + + override suspend fun incrementalCache_getModuleMappingData(target: TargetId): ByteArray? { + val id = sendMessage(CompilerCallbackServicesFacadeServerSide.IncrementalCache_getModuleMappingDataMessage(target)) + return readMessage(id) + } + + override suspend fun incrementalCache_registerInline(target: TargetId, fromPath: String, jvmSignature: String, toPath: String) { + sendNoReplyMessage( + CompilerCallbackServicesFacadeServerSide.IncrementalCache_registerInlineMessage( + target, + fromPath, + jvmSignature, + toPath + ) + ) + } + + override suspend fun incrementalCache_getClassFilePath(target: TargetId, internalClassName: String): String { + val id = sendMessage(CompilerCallbackServicesFacadeServerSide.IncrementalCache_getClassFilePathMessage(target, internalClassName)) + return readMessage(id) + } + + override suspend fun incrementalCache_close(target: TargetId) = + sendNoReplyMessage(CompilerCallbackServicesFacadeServerSide.IncrementalCache_closeMessage(target)) + + override suspend fun incrementalCache_getMultifileFacadeParts(target: TargetId, internalName: String): Collection? { + val id = sendMessage(CompilerCallbackServicesFacadeServerSide.IncrementalCache_getMultifileFacadePartsMessage(target, internalName)) + return readMessage(id) + } + + override suspend fun lookupTracker_requiresPosition(): Boolean { + val id = sendMessage(CompilerCallbackServicesFacadeServerSide.LookupTracker_requiresPositionMessage()) + return readMessage(id) + } + + override fun lookupTracker_record(lookups: Collection) = + sendNoReplyMessage(CompilerCallbackServicesFacadeServerSide.LookupTracker_recordMessage(lookups)) + + override suspend fun lookupTracker_isDoNothing(): Boolean { + val id = sendMessage(CompilerCallbackServicesFacadeServerSide.LookupTracker_isDoNothingMessage()) + return readMessage(id) + } + + override suspend fun compilationCanceledStatus_checkCanceled(): Void? { + sendNoReplyMessage(CompilerCallbackServicesFacadeServerSide.CompilationCanceledStatus_checkCanceledMessage()) + return null + } + + override suspend fun report(category: Int, severity: Int, message: String?, attachment: Serializable?) { + sendNoReplyMessage(CompilerServicesFacadeBaseServerSide.ReportMessage(category, severity, message, attachment)) + } + +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompilerCallbackServicesFacadeServerSide.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompilerCallbackServicesFacadeServerSide.kt new file mode 100644 index 00000000000..3c2ef557cf8 --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompilerCallbackServicesFacadeServerSide.kt @@ -0,0 +1,104 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import org.jetbrains.kotlin.daemon.common.CompilerCallbackServicesFacadeAsync +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Server.Message +import org.jetbrains.kotlin.incremental.components.LookupInfo +import org.jetbrains.kotlin.modules.TargetId + +interface CompilerCallbackServicesFacadeServerSide : CompilerCallbackServicesFacadeAsync, CompilerServicesFacadeBaseServerSide { + + class HasIncrementalCachesMessage : Message() { + override suspend fun processImpl(server: CompilerCallbackServicesFacadeServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.hasIncrementalCaches()) + } + + class HasLookupTrackerMessage : Message() { + override suspend fun processImpl(server: CompilerCallbackServicesFacadeServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.hasLookupTracker()) + } + + class HasCompilationCanceledStatusMessage : Message() { + override suspend fun processImpl(server: CompilerCallbackServicesFacadeServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.hasCompilationCanceledStatus()) + } + + // ---------------------------------------------------- + // IncrementalCache + class IncrementalCache_getObsoletePackagePartsMessage(val target: TargetId) : Message() { + override suspend fun processImpl(server: CompilerCallbackServicesFacadeServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.incrementalCache_getObsoletePackageParts(target)) + } + + class IncrementalCache_getObsoleteMultifileClassFacadesMessage(val target: TargetId) : Message() { + override suspend fun processImpl(server: CompilerCallbackServicesFacadeServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.incrementalCache_getObsoleteMultifileClassFacades(target)) + } + + class IncrementalCache_getPackagePartDataMessage(val target: TargetId, val partInternalName: String) : Message() { + override suspend fun processImpl(server: CompilerCallbackServicesFacadeServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.incrementalCache_getPackagePartData(target, partInternalName)) + } + + class IncrementalCache_getModuleMappingDataMessage(val target: TargetId) : Message() { + override suspend fun processImpl(server: CompilerCallbackServicesFacadeServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.incrementalCache_getModuleMappingData(target)) + } + + class IncrementalCache_registerInlineMessage( + val target: TargetId, + val fromPath: String, + val jvmSignature: String, + val toPath: String + ) : Message() { + override suspend fun processImpl(server: CompilerCallbackServicesFacadeServerSide, sendReply: (Any?) -> Unit) = + server.incrementalCache_registerInline(target, fromPath, jvmSignature, toPath) + } + + class IncrementalCache_getClassFilePathMessage(val target: TargetId, val internalClassName: String) : Message() { + override suspend fun processImpl(server: CompilerCallbackServicesFacadeServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.incrementalCache_getClassFilePath(target, internalClassName)) + } + + class IncrementalCache_closeMessage(val target: TargetId) : Message() { + override suspend fun processImpl(server: CompilerCallbackServicesFacadeServerSide, sendReply: (Any?) -> Unit) = + server.incrementalCache_close(target) + } + + class IncrementalCache_getMultifileFacadePartsMessage(val target: TargetId, val internalName: String) : Message() { + override suspend fun processImpl(server: CompilerCallbackServicesFacadeServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.incrementalCache_getMultifileFacadeParts(target, internalName)) + } + + // ---------------------------------------------------- + // LookupTracker + + class LookupTracker_requiresPositionMessage : Message() { + override suspend fun processImpl(server: CompilerCallbackServicesFacadeServerSide, sendReply: (Any?) -> Unit) { + server.lookupTracker_requiresPosition() + } + } + + class LookupTracker_recordMessage(val lookups: Collection) : Message() { + override suspend fun processImpl(server: CompilerCallbackServicesFacadeServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.lookupTracker_record(lookups)) + } + + class LookupTracker_isDoNothingMessage : Message() { + override suspend fun processImpl(server: CompilerCallbackServicesFacadeServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.lookupTracker_isDoNothing()) + } + + // ---------------------------------------------------- + // CompilationCanceledStatus + class CompilationCanceledStatus_checkCanceledMessage : Message() { + override suspend fun processImpl(server: CompilerCallbackServicesFacadeServerSide, sendReply: (Any?) -> Unit) { + server.compilationCanceledStatus_checkCanceled() + } + } + +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompilerServicesFacadeBaseClientSide.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompilerServicesFacadeBaseClientSide.kt new file mode 100644 index 00000000000..7f25269eee7 --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompilerServicesFacadeBaseClientSide.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import org.jetbrains.kotlin.daemon.common.CompilerServicesFacadeBaseAsync +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Client +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.DefaultClient +import java.io.Serializable + +interface CompilerServicesFacadeBaseClientSide : CompilerServicesFacadeBaseAsync, Client + +class CompilerServicesFacadeBaseClientSideImpl(val serverPort: Int) : + CompilerServicesFacadeBaseClientSide, + Client by DefaultClient(serverPort) { + + // TODO: consider invoking connectToServer() in init block + + override suspend fun report(category: Int, severity: Int, message: String?, attachment: Serializable?) { + log.info("client $serverPort - fun report") + sendNoReplyMessage( + CompilerServicesFacadeBaseServerSide.ReportMessage( + category, severity, message, attachment + ) + ) + } + +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompilerServicesFacadeBaseServerSide.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompilerServicesFacadeBaseServerSide.kt new file mode 100644 index 00000000000..19cd88c54af --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/CompilerServicesFacadeBaseServerSide.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import org.jetbrains.kotlin.daemon.common.CompilerServicesFacadeBaseAsync +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Server +import java.io.Serializable + +interface CompilerServicesFacadeBaseServerSide : CompilerServicesFacadeBaseAsync, Server { + + class ReportMessage( + val category: Int, + val severity: Int, + val message: String?, + val attachment: Serializable? + ) : Server.Message() { + + override suspend fun processImpl(server: CompilerServicesFacadeBaseServerSide, sendReply: (Any?) -> Unit) { + server.report(category, severity, message, attachment) + } + + } +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/IncrementalCompilerServicesFacadeAsyncWrapper.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/IncrementalCompilerServicesFacadeAsyncWrapper.kt new file mode 100644 index 00000000000..8316354cddf --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/IncrementalCompilerServicesFacadeAsyncWrapper.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import org.jetbrains.kotlin.daemon.common.IncrementalCompilerServicesFacade +import org.jetbrains.kotlin.daemon.common.SimpleDirtyData +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Client +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.DefaultClientRMIWrapper +import java.io.File +import java.io.Serializable + +class IncrementalCompilerServicesFacadeAsyncWrapper( + val rmiImpl: IncrementalCompilerServicesFacade +) : IncrementalCompilerServicesFacadeClientSide, Client by DefaultClientRMIWrapper() { + + override suspend fun report(category: Int, severity: Int, message: String?, attachment: Serializable?) = + rmiImpl.report(category, severity, message, attachment) + +} + +fun IncrementalCompilerServicesFacade.toClient() = IncrementalCompilerServicesFacadeAsyncWrapper(this) diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/IncrementalCompilerServicesFacadeClientSide.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/IncrementalCompilerServicesFacadeClientSide.kt new file mode 100644 index 00000000000..ee926b54fa1 --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/IncrementalCompilerServicesFacadeClientSide.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import org.jetbrains.kotlin.daemon.common.IncrementalCompilerServicesFacadeAsync +import org.jetbrains.kotlin.daemon.common.SimpleDirtyData +import org.jetbrains.kotlin.daemon.common.experimental.IncrementalCompilerServicesFacadeServerSide.* +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Client +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.DefaultClient +import java.io.File +import java.io.Serializable + +interface IncrementalCompilerServicesFacadeClientSide : IncrementalCompilerServicesFacadeAsync, CompilerServicesFacadeBaseClientSide + +class IncrementalCompilerServicesFacadeClientSideImpl(val serverPort: Int) : + IncrementalCompilerServicesFacadeClientSide, + Client by DefaultClient(serverPort) { + + override suspend fun report(category: Int, severity: Int, message: String?, attachment: Serializable?) { + sendNoReplyMessage(CompilerServicesFacadeBaseServerSide.ReportMessage(category, severity, message, attachment)) + } + +} diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/IncrementalCompilerServicesFacadeRMIWrapper.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/IncrementalCompilerServicesFacadeRMIWrapper.kt new file mode 100644 index 00000000000..03c97682912 --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/IncrementalCompilerServicesFacadeRMIWrapper.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import kotlinx.coroutines.runBlocking +import org.jetbrains.kotlin.daemon.common.IncrementalCompilerServicesFacade +import org.jetbrains.kotlin.daemon.common.SimpleDirtyData +import java.io.File +import java.io.Serializable + +class IncrementalCompilerServicesFacadeRMIWrapper(val clientSide: IncrementalCompilerServicesFacadeClientSide) : + IncrementalCompilerServicesFacade, Serializable { + + override fun report(category: Int, severity: Int, message: String?, attachment: Serializable?) = runBlocking { + clientSide.report(category, severity, message, attachment) + } + + // TODO: consider connecting to server right here in init-block +} + +fun IncrementalCompilerServicesFacadeClientSide.toRMI() = + if (this is IncrementalCompilerServicesFacadeAsyncWrapper) this.rmiImpl + else IncrementalCompilerServicesFacadeRMIWrapper(this) \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/IncrementalCompilerServicesFacadeServerSide.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/IncrementalCompilerServicesFacadeServerSide.kt new file mode 100644 index 00000000000..71b7f72f5fb --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/IncrementalCompilerServicesFacadeServerSide.kt @@ -0,0 +1,14 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import org.jetbrains.kotlin.daemon.common.IncrementalCompilerServicesFacadeAsync +import org.jetbrains.kotlin.daemon.common.SimpleDirtyData +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Server +import java.io.File + + +interface IncrementalCompilerServicesFacadeServerSide : IncrementalCompilerServicesFacadeAsync, CompilerServicesFacadeBaseServerSide \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/NetworkUtils.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/NetworkUtils.kt new file mode 100644 index 00000000000..4ffc1d95dd7 --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/NetworkUtils.kt @@ -0,0 +1,137 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import io.ktor.network.selector.ActorSelectorManager +import io.ktor.network.sockets.aSocket +import kotlinx.coroutines.* +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.ServerSocketWrapper +import org.jetbrains.kotlin.daemon.common.* +import java.io.IOException +import java.io.Serializable +import java.net.* +import java.rmi.server.RMIClientSocketFactory +import java.rmi.server.RMIServerSocketFactory +import java.util.* + + +// copyed from original(org.jetbrains.kotlin.daemon.common.NetworkUtils) TODO +// unique part : +// - AbstractClientLoopbackSocketFactory / ServerLoopbackSocketFactoryRMI / ServerLoopbackSocketFactoryKtor - Ktor-Sockets instead of java sockets +// - findPortAndCreateSocket +// TODO: get rid of copy-paste here + +object LoopbackNetworkInterface { + + const val IPV4_LOOPBACK_INET_ADDRESS = "127.0.0.1" + const val IPV6_LOOPBACK_INET_ADDRESS = "::1" + + // size of the requests queue for daemon services, so far seems that we don't need any big numbers here + // but if we'll start getting "connection refused" errors, that could be the first place to try to fix it + val SERVER_SOCKET_BACKLOG_SIZE by lazy { + System.getProperty(DAEMON_RMI_SOCKET_BACKLOG_SIZE_PROPERTY)?.toIntOrNull() ?: DEFAULT_SERVER_SOCKET_BACKLOG_SIZE + } + val SOCKET_CONNECT_ATTEMPTS by lazy { + System.getProperty(DAEMON_RMI_SOCKET_CONNECT_ATTEMPTS_PROPERTY)?.toIntOrNull() ?: DEFAULT_SOCKET_CONNECT_ATTEMPTS + } + val SOCKET_CONNECT_INTERVAL_MS by lazy { + System.getProperty(DAEMON_RMI_SOCKET_CONNECT_INTERVAL_PROPERTY)?.toLongOrNull() ?: DEFAULT_SOCKET_CONNECT_INTERVAL_MS + } + + val serverLoopbackSocketFactoryRMI by lazy { ServerLoopbackSocketFactoryRMI() } + val clientLoopbackSocketFactoryRMI by lazy { ClientLoopbackSocketFactoryRMI() } + + val serverLoopbackSocketFactoryKtor by lazy { ServerLoopbackSocketFactoryKtor() } + val clientLoopbackSocketFactoryKtor by lazy { ClientLoopbackSocketFactoryKtor() } + + // TODO switch to InetAddress.getLoopbackAddress on java 7+ + val loopbackInetAddressName by lazy { + try { + if (InetAddress.getByName(null) is Inet6Address) IPV6_LOOPBACK_INET_ADDRESS else IPV4_LOOPBACK_INET_ADDRESS + } catch (e: IOException) { + // getLocalHost may fail for unknown reasons in some situations, the fallback is to assume IPv4 for now + // TODO consider some other ways to detect default to IPv6 addresses in this case + IPV4_LOOPBACK_INET_ADDRESS + } + } + + // base socket factories by default don't implement equals properly (see e.g. http://stackoverflow.com/questions/21555710/rmi-and-jmx-socket-factories) + // so implementing it in derived classes using the fact that they are singletons + + class ServerLoopbackSocketFactoryRMI : RMIServerSocketFactory, Serializable { + override fun equals(other: Any?): Boolean = other === this || super.equals(other) + override fun hashCode(): Int = super.hashCode() + + @Throws(IOException::class) + override fun createServerSocket(port: Int): java.net.ServerSocket = + ServerSocket(port, SERVER_SOCKET_BACKLOG_SIZE, InetAddress.getByName(null)) + } + + val selectorMgr = ActorSelectorManager(Dispatchers.IO) + + class ServerLoopbackSocketFactoryKtor : Serializable { + override fun equals(other: Any?): Boolean = other === this || super.equals(other) + override fun hashCode(): Int = super.hashCode() + + @Throws(IOException::class) + fun createServerSocket(port: Int) = + aSocket(selectorMgr) + .tcp() + .bind(InetSocketAddress(InetAddress.getByName(null), port)) // TODO : NO BACKLOG SIZE CHANGE =( + } + + abstract class AbstractClientLoopbackSocketFactory : Serializable { + override fun equals(other: Any?): Boolean = other === this || super.equals(other) + override fun hashCode(): Int = super.hashCode() + + abstract protected fun socketCreate(host: String, port: Int): SocketType + + @Throws(IOException::class) + fun createSocket(host: String, port: Int): SocketType { + var attemptsLeft = SOCKET_CONNECT_ATTEMPTS + while (true) { + try { + return socketCreate(host, port) + } catch (e: ConnectException) { + if (--attemptsLeft <= 0) throw e + } + Thread.sleep(SOCKET_CONNECT_INTERVAL_MS) + } + } + } + + class ClientLoopbackSocketFactoryRMI : AbstractClientLoopbackSocketFactory(), RMIClientSocketFactory { + override fun socketCreate(host: String, port: Int): Socket = Socket(InetAddress.getByName(null), port) + } + + class ClientLoopbackSocketFactoryKtor : AbstractClientLoopbackSocketFactory() { + override fun socketCreate(host: String, port: Int): io.ktor.network.sockets.Socket = + runBlocking { aSocket(selectorMgr).tcp().connect(InetSocketAddress(host, port)) } + } + +} + + +private val portSelectionRng = Random() + +fun findPortForSocket(attempts: Int, portRangeStart: Int, portRangeEnd: Int): ServerSocketWrapper { + var i = 0 + var lastException: Exception? = null + + while (i++ < attempts) { + val port = portSelectionRng.nextInt(portRangeEnd - portRangeStart) + portRangeStart + try { + return ServerSocketWrapper( + port, + LoopbackNetworkInterface.serverLoopbackSocketFactoryKtor.createServerSocket(port) + ) + } catch (e: Exception) { + // assuming that the socketPort is already taken + lastException = e + } + } + throw IllegalStateException("Cannot find free socketPort in $attempts attempts", lastException) +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/RemoteStreamAsync.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/RemoteStreamAsync.kt new file mode 100644 index 00000000000..923c5c9174a --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/RemoteStreamAsync.kt @@ -0,0 +1,29 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +interface RemoteOutputStreamAsync { + + /** closeStream() name is chosen since Clients are AutoClosable now + * and Client-implementations of RemoteOutputStreamAsync have conflict of 'close' name **/ + suspend fun closeStream() + + suspend fun write(data: ByteArray, offset: Int, length: Int) + + suspend fun write(dataByte: Int) +} + +interface RemoteInputStreamAsync { + + /** closeStream() name is chosen since Clients are AutoClosable now + * and Client-implementations of RemoteInputStreamAsync have conflict of 'close' name **/ + suspend fun closeStream() + + suspend fun read(length: Int): ByteArray + + suspend fun read(): Int + +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/RemoteStreamAsyncClientSide.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/RemoteStreamAsyncClientSide.kt new file mode 100644 index 00000000000..b29fa395b4c --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/RemoteStreamAsyncClientSide.kt @@ -0,0 +1,12 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Client + +interface RemoteOutputStreamAsyncClientSide : RemoteOutputStreamAsync, Client + +interface RemoteInputStreamClientSide : RemoteInputStreamAsync, Client diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/RemoteStreamAsyncServerSide.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/RemoteStreamAsyncServerSide.kt new file mode 100644 index 00000000000..0c890955a82 --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/RemoteStreamAsyncServerSide.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.ByteWriteChannelWrapper +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Server + + +interface RemoteOutputStreamAsyncServerSide : RemoteOutputStreamAsync, Server { + // Query messages: + class CloseMessage : Server.Message() { + override suspend fun processImpl(server: RemoteOutputStreamAsyncServerSide, sendReply: (Any?) -> Unit) = + server.closeStream() + } + + class WriteMessage(val data: ByteArray, val offset: Int = -1, val length: Int = -1) : + Server.Message() { + override suspend fun processImpl(server: RemoteOutputStreamAsyncServerSide, sendReply: (Any?) -> Unit) = + server.write(data, offset, length) + } + + class WriteIntMessage(val dataByte: Int) : Server.Message() { + override suspend fun processImpl(server: RemoteOutputStreamAsyncServerSide, sendReply: (Any?) -> Unit) = + server.write(dataByte) + } +} + + +interface RemoteInputStreamServerSide : RemoteInputStreamAsync, Server { + // Query messages: + class CloseMessage : Server.Message() { + override suspend fun processImpl(server: RemoteInputStreamServerSide, sendReply: (Any?) -> Unit) = + server.closeStream() + } + + class ReadMessage(val length: Int = -1) : Server.Message() { + override suspend fun processImpl(server: RemoteInputStreamServerSide, sendReply: (Any?) -> Unit) = + sendReply(if (length == -1) server.read() else server.read(length)) + } +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/RemoteStreamAsyncWrapper.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/RemoteStreamAsyncWrapper.kt new file mode 100644 index 00000000000..e313b26f33d --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/RemoteStreamAsyncWrapper.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import org.jetbrains.kotlin.daemon.common.RemoteInputStream +import org.jetbrains.kotlin.daemon.common.RemoteOutputStream +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Client +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.DefaultClientRMIWrapper + +class RemoteOutputStreamAsyncWrapper(val rmiOutput: RemoteOutputStream) : RemoteOutputStreamAsyncClientSide, + Client by DefaultClientRMIWrapper() { + + override suspend fun closeStream() = + rmiOutput.close() + + override suspend fun write(data: ByteArray, offset: Int, length: Int) = + rmiOutput.write(data, offset, length) + + override suspend fun write(dataByte: Int) = + rmiOutput.write(dataByte) + +} + +class RemoteInputStreamAsyncWrapper(private val rmiInput: RemoteInputStream) : RemoteInputStreamClientSide, + Client by DefaultClientRMIWrapper() { + + override suspend fun closeStream() = + rmiInput.close() + + override suspend fun read() = + rmiInput.read() + + override suspend fun read(length: Int) = + rmiInput.read(length) +} + +fun RemoteOutputStream.toClient() = RemoteOutputStreamAsyncWrapper(this) +fun RemoteInputStream.toClient() = RemoteInputStreamAsyncWrapper(this) \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/ReplStateFacadeClientSide.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/ReplStateFacadeClientSide.kt new file mode 100644 index 00000000000..b44742f34ea --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/ReplStateFacadeClientSide.kt @@ -0,0 +1,11 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import org.jetbrains.kotlin.daemon.common.ReplStateFacadeAsync +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Client + +interface ReplStateFacadeClientSide: ReplStateFacadeAsync, Client \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/ReplStateFacadeServerSide.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/ReplStateFacadeServerSide.kt new file mode 100644 index 00000000000..ba2d9e25da0 --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/ReplStateFacadeServerSide.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import org.jetbrains.kotlin.cli.common.repl.ILineId +import org.jetbrains.kotlin.daemon.common.ReplStateFacadeAsync +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.ByteWriteChannelWrapper +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Server + +interface ReplStateFacadeServerSide: ReplStateFacadeAsync, Server { + + // Query messages: + class GetIdMessage : Server.Message() { + override suspend fun processImpl(server: ReplStateFacadeServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.getId()) + } + + class GetHistorySizeMessage : Server.Message() { + override suspend fun processImpl(server: ReplStateFacadeServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.getHistorySize()) + } + + class HistoryGetMessage(val index: Int) : Server.Message() { + override suspend fun processImpl(server: ReplStateFacadeServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.historyGet(index)) + } + + class HistoryResetMessage : Server.Message() { + override suspend fun processImpl(server: ReplStateFacadeServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.historyReset()) + } + + class HistoryResetToMessage(val id: ILineId) : Server.Message() { + override suspend fun processImpl(server: ReplStateFacadeServerSide, sendReply: (Any?) -> Unit) = + sendReply(server.historyResetTo(id)) + } +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/SecurityUtils.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/SecurityUtils.kt new file mode 100644 index 00000000000..a162a88c414 --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/SecurityUtils.kt @@ -0,0 +1,77 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +@file:Suppress("CAST_NEVER_SUCCEEDS") + +package org.jetbrains.kotlin.daemon.common.experimental + +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.ByteReadChannelWrapper +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.ByteWriteChannelWrapper +import java.io.FileInputStream +import java.io.FileOutputStream +import java.io.ObjectInputStream +import java.io.ObjectOutputStream +import java.security.* +import java.security.spec.InvalidKeySpecException +import java.security.spec.X509EncodedKeySpec + + +const val SECURITY_TOKEN_SIZE = 128 + +private val secureRandom = SecureRandom.getInstance("SHA1PRNG", "SUN"); +private val pairGenerator = KeyPairGenerator.getInstance("DSA", "SUN") + +private fun generateSecurityToken(): ByteArray { + val tokenBuffer = ByteArray(SECURITY_TOKEN_SIZE) + secureRandom.nextBytes(tokenBuffer) + return tokenBuffer +} + +data class SecurityData(val privateKey: PrivateKey, val publicKey: PublicKey, val token: ByteArray) +fun generateKeysAndToken() = pairGenerator.generateKeyPair().let { + SecurityData(it.private, it.public, generateSecurityToken()) +} + +private fun FileInputStream.readBytesFixedLength(n: Int): ByteArray { + val buffer = ByteArray(n) + var bytesRead = 0 + while (bytesRead != n) { + bytesRead += this.read(buffer, bytesRead, n - bytesRead) + } + return buffer +} + +// server part : +fun sendTokenKeyPair(output: FileOutputStream, token: ByteArray, privateKey: PrivateKey) { + output.write(token) + ObjectOutputStream(output).use { + it.writeObject(privateKey) + } +} + +private fun instantiateDsa() = Signature.getInstance("SHA1withDSA", "SUN") + +suspend fun getSignatureAndVerify(input: ByteReadChannelWrapper, expectedToken: ByteArray, publicKey: PublicKey): Boolean { + val signature = input.nextBytes() + val dsa = instantiateDsa() + dsa.initVerify(publicKey) + dsa.update(expectedToken, 0, SECURITY_TOKEN_SIZE) + val verified = dsa.verify(signature) + log.fine("verified : $verified") + return verified +} + + +// client part : +fun readTokenKeyPairAndSign(input: FileInputStream): ByteArray { + val token = input.readBytesFixedLength(SECURITY_TOKEN_SIZE) + val privateKey = ObjectInputStream(input).use(ObjectInputStream::readObject) as PrivateKey + val dsa = instantiateDsa() + dsa.initSign(privateKey) + dsa.update(token, 0, SECURITY_TOKEN_SIZE) + return dsa.sign() +} + +suspend fun sendSignature(output: ByteWriteChannelWrapper, signature: ByteArray) = output.writeBytesAndLength(signature.size, signature) \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/SocketFind.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/SocketFind.kt new file mode 100644 index 00000000000..c99ce4a8ced --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/SocketFind.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common.experimental + +import org.jetbrains.kotlin.daemon.common.COMPILE_DAEMON_FIND_PORT_ATTEMPTS +import org.jetbrains.kotlin.daemon.common.experimental.* + +fun findCallbackServerSocket() = findPortForSocket( + COMPILE_DAEMON_FIND_PORT_ATTEMPTS, + CALLBACK_SERVER_PORTS_RANGE_START, + CALLBACK_SERVER_PORTS_RANGE_END +) + +fun findReplServerSocket() = findPortForSocket( + COMPILE_DAEMON_FIND_PORT_ATTEMPTS, + REPL_SERVER_PORTS_RANGE_START, + REPL_SERVER_PORTS_RANGE_END +) \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/socketInfrastructure/Client.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/socketInfrastructure/Client.kt new file mode 100644 index 00000000000..be61accbe71 --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/socketInfrastructure/Client.kt @@ -0,0 +1,260 @@ +package org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure + +import io.ktor.network.sockets.Socket +import kotlinx.coroutines.channels.* +import kotlinx.coroutines.* +import org.jetbrains.kotlin.daemon.common.experimental.LoopbackNetworkInterface +import sun.net.ConnectionResetException +import java.beans.Transient +import java.io.IOException +import java.io.ObjectInputStream +import java.io.ObjectOutputStream +import java.io.Serializable +import java.util.logging.Logger +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.Server.* + +interface Client : Serializable, AutoCloseable { + + @Throws(Exception::class) + suspend fun connectToServer() + + suspend fun sendMessage(msg: AnyMessage): Int // returns message unique id + fun sendNoReplyMessage(msg: AnyMessage) + suspend fun readMessage(id: Int): T + +} + +@Suppress("UNCHECKED_CAST") +abstract class DefaultAuthorizableClient( + val serverPort: Int, + val serverHost: String = LoopbackNetworkInterface.loopbackInetAddressName +) : Client { + + val log: Logger + @Transient get() = Logger.getLogger("default client($serverPort)")//.also { it.setUseParentHandlers(false); } + + @kotlin.jvm.Transient + lateinit var input: ByteReadChannelWrapper + + @kotlin.jvm.Transient + lateinit var output: ByteWriteChannelWrapper + + @kotlin.jvm.Transient + private var socket: Socket? = null + + abstract suspend fun authorizeOnServer(serverOutputChannel: ByteWriteChannelWrapper): Boolean + abstract suspend fun clientHandshake(input: ByteReadChannelWrapper, output: ByteWriteChannelWrapper, log: Logger): Boolean + + /* + The purpose of ths function is the following : a client starts sending keep-alive requests to the server. + It sends a request every K seconds and awaits a response after K/2 seconds. + In case of not receiving the response from the server we stop waiting for other responses. + This resolves the case when the server is down but the client is still waiting for some calculations. + Keep-alives run in a separate thread on client and on server, hence we can assume that the calculations on server + shouldn't delay keep-alive responses too much + */ + abstract fun startKeepAlives() + + /* + `delayKeepAlives` resolves another issue. Imagine that a server and a client have too many short requests/responses, + say, 1000 requests/responses and 0.001 seconds of latency. Then we can miss keep-alive response because of a socket workload, + or, say, if a scheduler decides not to schedule on a keep-alive thread. + In this case we say that any response can stand for a keep-alive message, as well. delayKeepAlives() serves this purpose. + */ + abstract fun delayKeepAlives() + + override fun close() { + socket?.close() + } + + class MessageReply(val messageId: Int, val reply: T?) : Serializable + + protected interface ReadActorQuery + protected data class ExpectReplyQuery(val messageId: Int, val result: CompletableDeferred>) : ReadActorQuery + protected class ReceiveReplyQuery(val reply: MessageReply<*>) : ReadActorQuery + + protected interface WriteActorQuery + protected data class SendNoreplyMessageQuery(val message: AnyMessage<*>) : WriteActorQuery + protected data class SendMessageQuery(val message: AnyMessage<*>, val messageId: CompletableDeferred) : WriteActorQuery + + protected class StopAllRequests : ReadActorQuery, WriteActorQuery + + @kotlin.jvm.Transient + protected lateinit var readActor: SendChannel + + @kotlin.jvm.Transient + private lateinit var writeActor: SendChannel + + override suspend fun sendMessage(msg: AnyMessage): Int { + val id = CompletableDeferred() + writeActor.send(SendMessageQuery(msg, id)) + val idVal = id.await() + if (idVal is IOException) { + throw idVal + } + return idVal as Int + } + + override fun sendNoReplyMessage(msg: AnyMessage) { + writeActor.offer(SendNoreplyMessageQuery(msg)) + } + + override suspend fun readMessage(id: Int): T { + val result = CompletableDeferred>() + try { + readActor.send(ExpectReplyQuery(id, result)) + } catch (e: ClosedSendChannelException) { + throw IOException("failed to read message (channel was closed)") + } + val actualResult = result.await().reply + if (actualResult is IOException) { + throw actualResult + } + return actualResult as T + } + + override suspend fun connectToServer() { + + writeActor = GlobalScope.actor(capacity = Channel.UNLIMITED) { + var firstFreeMessageId = 0 + consumeEach { query -> + when (query) { + is SendMessageQuery -> { + val id = firstFreeMessageId++ + try { + output.writeObject(query.message.withId(id)) + query.messageId.complete(id) + } catch (e: IOException) { + query.messageId.complete(e) + } + } + is SendNoreplyMessageQuery -> { + output.writeObject(query.message.withId(-1)) + } + is StopAllRequests -> { + channel.close() + } + } + } + } + + class NextObjectQuery + + val nextObjectQuery = NextObjectQuery() + val objectReaderActor = GlobalScope.actor(capacity = Channel.UNLIMITED) { + consumeEach { + try { + val reply = input.nextObject() + when (reply) { + is ServerDownMessage<*> -> throw IOException("connection closed by server") + !is MessageReply<*> -> throw IOException("contrafact message (expected MessageReply<*>)") + else -> readActor.send(ReceiveReplyQuery(reply)) + } + } catch (e: IOException) { + readActor.send(StopAllRequests()) + } + } + } + + readActor = GlobalScope.actor(capacity = Channel.UNLIMITED) { + val receivedMessages = hashMapOf>() + val expectedMessages = hashMapOf() + + fun broadcastIOException(e: IOException) { + channel.close() + expectedMessages.forEach { id, deferred -> + deferred.result.complete(MessageReply(id, e)) + } + expectedMessages.clear() + receivedMessages.clear() + } + + consumeEach { query -> + when (query) { + is ExpectReplyQuery -> { + receivedMessages[query.messageId]?.also { reply -> + query.result.complete(reply) + } ?: expectedMessages.put(query.messageId, query).also { + objectReaderActor.send(nextObjectQuery) + } + } + is ReceiveReplyQuery -> { + val reply = query.reply + expectedMessages[reply.messageId]?.also { expectedMsg -> + expectedMsg.result.complete(reply) + } ?: receivedMessages.put(reply.messageId, reply).also { + objectReaderActor.send(nextObjectQuery) + } + delayKeepAlives() + } + is StopAllRequests -> { + broadcastIOException(IOException("KeepAlive failed")) + writeActor.send(StopAllRequests()) + } + } + } + } + + + + try { + socket = LoopbackNetworkInterface.clientLoopbackSocketFactoryKtor.createSocket( + serverHost, + serverPort + ) + } catch (e: Throwable) { + close() + throw e + } + socket?.openIO(log)?.also { + input = it.input + output = it.output + if (!clientHandshake(input, output, log)) { + throw ConnectionResetException("failed to establish connection with server (handshake failed)") + } + if (!authorizeOnServer(output)) { + throw ConnectionResetException("failed to establish connection with server (authorization failed)") + } + } + + startKeepAlives() + + } + + @Throws(ClassNotFoundException::class, IOException::class) + private fun readObject(aInputStream: ObjectInputStream) { + aInputStream.defaultReadObject() + println("connecting...") + runBlocking { connectToServer() } + println("connectED") + } + + @Throws(IOException::class) + private fun writeObject(aOutputStream: ObjectOutputStream) { + aOutputStream.defaultWriteObject() + } + +} + +class DefaultClient( + serverPort: Int, + serverHost: String = LoopbackNetworkInterface.loopbackInetAddressName +) : DefaultAuthorizableClient(serverPort, serverHost) { + override suspend fun clientHandshake(input: ByteReadChannelWrapper, output: ByteWriteChannelWrapper, log: Logger) = true + override suspend fun authorizeOnServer(output: ByteWriteChannelWrapper): Boolean = true + override fun startKeepAlives() {} + override fun delayKeepAlives() {} +} + +class DefaultClientRMIWrapper : Client { + + override suspend fun connectToServer() {} + override suspend fun sendMessage(msg: AnyMessage) = + throw UnsupportedOperationException("sendMessage is not supported for RMI wrappers") + + override fun sendNoReplyMessage(msg: AnyMessage) = + throw UnsupportedOperationException("sendMessage is not supported for RMI wrappers") + + override suspend fun readMessage(id: Int) = throw UnsupportedOperationException("readMessage is not supported for RMI wrappers") + override fun close() {} +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/socketInfrastructure/Server.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/socketInfrastructure/Server.kt new file mode 100644 index 00000000000..8f23176183c --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/socketInfrastructure/Server.kt @@ -0,0 +1,208 @@ +package org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure + +import io.ktor.network.sockets.ServerSocket +import io.ktor.network.sockets.Socket +import kotlinx.coroutines.* +import org.jetbrains.kotlin.daemon.common.experimental.* +import java.io.Serializable +import java.util.concurrent.TimeUnit +import java.util.logging.Logger + +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +data class ServerSocketWrapper(val port: Int, val socket: ServerSocket) + +interface ServerBase + +@Suppress("UNCHECKED_CAST") +interface Server : ServerBase { + + val serverSocketWithPort: ServerSocketWrapper + val serverPort: Int + get() = serverSocketWithPort.port + + private val log: Logger + get() = Logger.getLogger("default server($serverPort)") + + enum class State { + WORKING, CLOSED, ERROR, DOWNING, UNVERIFIED + } + + fun processMessage(msg: AnyMessage, output: ByteWriteChannelWrapper): State = + when (msg) { + is Message -> State.WORKING.also { + msg.process(this as T, output) + } + is EndConnectionMessage -> { + State.CLOSED + } + is ServerDownMessage -> State.CLOSED + else -> State.ERROR + } + + // TODO: replace GlobalScope here and below with smth. more explicit + fun attachClient(client: Socket): Deferred = GlobalScope.async { + val (input, output) = client.openIO(log) + if (!serverHandshake(input, output, log)) { + return@async State.UNVERIFIED + } + if (!checkClientCanReadFile(input)) { + return@async State.UNVERIFIED + } + clients[client] = ClientInfo(client, input, output) + var finalState = State.WORKING + val keepAliveAcknowledgement = KeepAliveAcknowledgement() + loop@ + while (true) { + val message = input.nextObject() + when (message) { + is ServerDownMessage<*> -> { + shutdownClient(client) + break@loop + } + is KeepAliveMessage<*> -> State.WORKING.also { + output.writeObject( + DefaultAuthorizableClient.MessageReply( + message.messageId!!, + keepAliveAcknowledgement + ) + ) + } + !is AnyMessage<*> -> { + finalState = State.ERROR + break@loop + } + else -> { + val state = processMessage(message as AnyMessage, output) + when (state) { + State.WORKING -> continue@loop + State.ERROR -> { + finalState = State.ERROR + break@loop + } + else -> { + finalState = state + break@loop + } + } + } + } + } + finalState + } + + abstract class AnyMessage : Serializable { + var messageId: Int? = null + fun withId(id: Int): AnyMessage { + messageId = id + return this + } + } + + abstract class Message : AnyMessage() { + fun process(server: ServerType, output: ByteWriteChannelWrapper) = GlobalScope.async { + log.fine("$server starts processing ${this@Message}") + processImpl(server, { + log.fine("$server finished processing ${this@Message}, sending output") + GlobalScope.async { + log.fine("$server starts sending ${this@Message} to output") + output.writeObject(DefaultAuthorizableClient.MessageReply(messageId ?: -1, it)) + log.fine("$server finished sending ${this@Message} to output") + } + }) + } + + abstract suspend fun processImpl(server: ServerType, sendReply: (Any?) -> Unit) + } + + class EndConnectionMessage : AnyMessage() + + class KeepAliveAcknowledgement : AnyMessage() + + class KeepAliveMessage : AnyMessage() + + class ServerDownMessage : AnyMessage() + + data class ClientInfo(val socket: Socket, val input: ByteReadChannelWrapper, val output: ByteWriteChannelWrapper) + + val clients: HashMap + + private fun dealWithClient(client: Socket) = GlobalScope.async { + val state = attachClient(client).await() + when (state) { + State.CLOSED, State.UNVERIFIED -> shutdownClient(client) + State.DOWNING -> shutdownServer() + else -> shutdownClient(client) + } + } + + fun runServer(): Deferred { + val serverSocket = serverSocketWithPort.socket + return GlobalScope.async { + serverSocket.use { + while (true) { + dealWithClient(serverSocket.accept()) + } + } + } + } + + fun shutdownServer() { + clients.forEach { socket, info -> + runBlockingWithTimeout { + info.output.writeObject(ServerDownMessage()) + info.output.close() + } + socket.close() + } + clients.clear() + serverSocketWithPort.socket.close() + } + + private fun shutdownClient(client: Socket) { + clients.remove(client) + client.close() + } + + /* + This function writes some message in the server file, and awaits the confirmation from the client that it has read the message + correctly. The purpose here is to check whether the client can actually access file system and read file contents. + */ + suspend fun checkClientCanReadFile(clientInputChannel: ByteReadChannelWrapper): Boolean = true + + suspend fun serverHandshake(input: ByteReadChannelWrapper, output: ByteWriteChannelWrapper, log: Logger) = true + +} + +fun runBlockingWithTimeout(timeout: Long = AUTH_TIMEOUT_IN_MILLISECONDS, block: suspend () -> T) = + runBlocking { runWithTimeout(timeout = timeout) { block() } } + +//@Throws(TimeoutException::class) +suspend fun runWithTimeout( + timeout: Long = AUTH_TIMEOUT_IN_MILLISECONDS, + unit: TimeUnit = TimeUnit.MILLISECONDS, + block: suspend CoroutineScope.() -> T +): T? = withTimeoutOrNull(unit.toMillis(timeout)) { block() } + +//@Throws(ConnectionResetException::class) +suspend fun tryAcquireHandshakeMessage(input: ByteReadChannelWrapper, log: Logger): Boolean { + val bytes = runWithTimeout { + input.nextBytes() + } ?: return false + if (bytes.zip(FIRST_HANDSHAKE_BYTE_TOKEN).any { it.first != it.second }) { + return false + } + return true +} + + +//@Throws(ConnectionResetException::class) +suspend fun trySendHandshakeMessage(output: ByteWriteChannelWrapper, log: Logger): Boolean { + runWithTimeout { + output.writeBytesAndLength(FIRST_HANDSHAKE_BYTE_TOKEN.size, FIRST_HANDSHAKE_BYTE_TOKEN) + } ?: return false + return true +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/socketInfrastructure/StreamWrappers.kt b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/socketInfrastructure/StreamWrappers.kt new file mode 100644 index 00000000000..a15ab3ae61f --- /dev/null +++ b/compiler/daemon/daemon-common-new/src/org/jetbrains/kotlin/daemon/common/experimental/socketInfrastructure/StreamWrappers.kt @@ -0,0 +1,217 @@ +package org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure + +import io.ktor.network.sockets.Socket +import io.ktor.network.sockets.openReadChannel +import io.ktor.network.sockets.openWriteChannel +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.actor +import kotlinx.coroutines.channels.consumeEach +import kotlinx.coroutines.io.* +import kotlinx.io.core.readBytes +import java.io.* +import java.nio.ByteBuffer +import java.util.logging.Logger + +private val DEFAULT_BYTE_ARRAY = byteArrayOf(0, 0, 0, 0) + +class ByteReadChannelWrapper(readChannel: ByteReadChannel, private val log: Logger) { + + private interface ReadQuery + + private open class BytesQuery(val bytes: CompletableDeferred) : ReadQuery + + private class SerObjectQuery(val obj: CompletableDeferred) : ReadQuery + + suspend fun readLength(readChannel: ByteReadChannel) = + if (readChannel.isClosedForRead) + null + else + try { + readChannel.readPacket(4).readBytes() + } catch (e: Exception) { + log.fine("failed to read message length, ${e.message}") + null + } + + suspend fun readPacket(length: Int, readChannel: ByteReadChannel) = + try { + readChannel.readPacket( + length + ).readBytes() + } catch (e: Exception) { + log.fine("failed to read packet (${e.message})") + null + } + + // TODO : replace GlobalScope with something more explicit here and below. + private val readActor = GlobalScope.actor(capacity = Channel.UNLIMITED) { + consumeEach { message -> + if (!readChannel.isClosedForRead) { + readLength(readChannel)?.let { messageLength -> + when (message) { + is BytesQuery -> message.bytes.complete( + readChannel.readPacket( + getLength(messageLength) + ).readBytes() + ) + + is SerObjectQuery -> message.obj.complete( + getObject( + getLength(messageLength), + { len -> readPacket(len, readChannel) } + ) + ) + + else -> { + } + } + } + } else { + log.fine("read chanel closed " + log.name) + } + } + } + + private fun getLength(packet: ByteArray): Int { + val (b1, b2, b3, b4) = packet.map(Byte::toInt) + return (0xFF and b1 shl 24 or (0xFF and b2 shl 16) or + (0xFF and b3 shl 8) or (0xFF and b4)).also { log.fine(" $it") } + } + + /** first reads length token (4 bytes) and then -- reads length bytes. + * after deafault timeout returns DEFAULT_BYTE_ARRAY */ + suspend fun nextBytes(): ByteArray = runWithTimeout { + val expectedBytes = CompletableDeferred() + readActor.send(BytesQuery(expectedBytes)) + expectedBytes.await() + } ?: DEFAULT_BYTE_ARRAY + + private suspend fun getObject(length: Int, readPacket: suspend (Int) -> ByteArray?): Any? = + if (length >= 0) { + readPacket(length)?.let { bytes -> + ObjectInputStream( + ByteArrayInputStream(bytes) + ).use { + it.readObject() + } + } + } else { // optimize for long strings! + readPacket(-length)?.let { bytes -> + String( + ByteArrayInputStream( + bytes + ).readBytes() + ) + } + } + + /** first reads length token (4 bytes), then reads length bytes and returns deserialized object */ + suspend fun nextObject(): Any? { + val obj = CompletableDeferred() + readActor.send(SerObjectQuery(obj)) + val result = obj.await() + if (result is Server.ServerDownMessage<*>) { + throw IOException("connection closed by server") + } + return result + } + +} + +class ByteWriteChannelWrapper(writeChannel: ByteWriteChannel, private val log: Logger) { + + private interface WriteActorQuery + + private open class ByteData(val bytes: ByteArray) : WriteActorQuery { + open fun toByteArray(): ByteArray = bytes + } + + private class ObjectWithLength(val lengthBytes: ByteArray, bytes: ByteArray) : ByteData(bytes) { + override fun toByteArray() = lengthBytes + bytes + } + + private class CloseMessage : WriteActorQuery + + private suspend fun tryWrite(b: ByteArray, writeChannel: ByteWriteChannel) { + if (!writeChannel.isClosedForWrite) { + try { + writeChannel.writeFully(b) + } catch (e: Exception) { + log.fine("failed to print message, ${e.message}") + } + } else { + log.fine("closed chanel (write)") + } + } + + private val writeActor = GlobalScope.actor(capacity = Channel.UNLIMITED) { + consumeEach { message -> + if (!writeChannel.isClosedForWrite) { + when (message) { + is CloseMessage -> { + log.fine("${log.name} closing chanel...") + writeChannel.close() + } + is ByteData -> { + tryWrite(message.toByteArray(), writeChannel) + if (!writeChannel.isClosedForWrite) { + try { + writeChannel.flush() + } catch (e: Exception) { + log.fine("failed to flush byte write chanel") + } + } + } + } + } else { + log.fine("${log.name} write chanel closed") + } + } + } + + suspend fun writeBytesAndLength(length: Int, bytes: ByteArray) { + writeActor.send( + ObjectWithLength( + getLengthBytes(length), + bytes + ) + ) + } + + private suspend fun writeObjectImpl(obj: Any?) = + ByteArrayOutputStream().use { bos -> + ObjectOutputStream(bos).use { objOut -> + objOut.writeObject(obj) + objOut.flush() + val bytes = bos.toByteArray() + writeBytesAndLength(bytes.size, bytes) + } + } + + private suspend fun writeString(s: String) = writeBytesAndLength(-s.length, s.toByteArray()) + + fun getLengthBytes(length: Int) = + ByteBuffer + .allocate(4) + .putInt(length) + .array() + + suspend fun writeObject(obj: Any?) { + if (obj is String) writeString(obj) + else writeObjectImpl(obj) + } + + suspend fun close() = writeActor.send(CloseMessage()) + +} + +fun ByteReadChannel.toWrapper(log: Logger) = ByteReadChannelWrapper(this, log) +fun ByteWriteChannel.toWrapper(log: Logger) = ByteWriteChannelWrapper(this, log) + +fun Socket.openAndWrapReadChannel(log: Logger) = this.openReadChannel().toWrapper(log) +fun Socket.openAndWrapWriteChannel(log: Logger) = this.openWriteChannel().toWrapper(log) + +data class IOPair(val input: ByteReadChannelWrapper, val output: ByteWriteChannelWrapper) + +fun Socket.openIO(log: Logger) = IOPair(this.openAndWrapReadChannel(log), this.openAndWrapWriteChannel(log)) \ No newline at end of file diff --git a/compiler/daemon/daemon-common/build.gradle.kts b/compiler/daemon/daemon-common/build.gradle.kts index 18103fa3d60..81d88d0757e 100644 --- a/compiler/daemon/daemon-common/build.gradle.kts +++ b/compiler/daemon/daemon-common/build.gradle.kts @@ -1,3 +1,4 @@ +import com.sun.javafx.scene.CameraHelper.project plugins { kotlin("jvm") @@ -13,6 +14,9 @@ dependencies { compileOnly(project(":js:js.frontend")) compileOnly(intellijCoreDep()) { includeJars("intellij-core") } compileOnly(intellijDep()) { includeIntellijCoreJarDependencies(project) } + compile(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-jdk8")) { + isTransitive = false + } } sourceSets { diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationResults.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationResults.kt index 92bdec7ab7e..60876993244 100644 --- a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationResults.kt +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationResults.kt @@ -29,4 +29,9 @@ enum class CompilationResultCategory(val code: Int) { IC_COMPILE_ITERATION(0), BUILD_REPORT_LINES(1), VERBOSE_BUILD_REPORT_LINES(2), +} + +interface CompilationResultsAsync { + suspend fun add(compilationResultCategory: Int, value: Serializable) + val clientSide: CompilationResultsAsync } \ No newline at end of file diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationResultsWrappers.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationResultsWrappers.kt new file mode 100644 index 00000000000..6f24e9dd31a --- /dev/null +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationResultsWrappers.kt @@ -0,0 +1,36 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common + +import kotlinx.coroutines.runBlocking +import java.io.Serializable + +class CompilationResultsAsyncWrapper(val rmiImpl: CompilationResults) : CompilationResultsAsync { + + override val clientSide: CompilationResultsAsync + get() = this + + override suspend fun add(compilationResultCategory: Int, value: Serializable) { + rmiImpl.add(compilationResultCategory, value) + } + +} + +class CompilationResultsRMIWrapper(val clientSide: CompilationResultsAsync) : CompilationResults, Serializable { + + override fun add(compilationResultCategory: Int, value: Serializable) = runBlocking { + clientSide.add(compilationResultCategory, value) + } + +} + +fun CompilationResults.toClient() = + if (this is CompilationResultsRMIWrapper) this.clientSide + else CompilationResultsAsyncWrapper(this) + +fun CompilationResultsAsync.toRMI() = + if (this is CompilationResultsAsyncWrapper) this.rmiImpl + else CompilationResultsRMIWrapper(this) diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileServiceAsync.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileServiceAsync.kt new file mode 100644 index 00000000000..0aeb816d420 --- /dev/null +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileServiceAsync.kt @@ -0,0 +1,79 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ +package org.jetbrains.kotlin.daemon.common + +import org.jetbrains.kotlin.cli.common.repl.ReplCheckResult +import org.jetbrains.kotlin.cli.common.repl.ReplCodeLine +import org.jetbrains.kotlin.cli.common.repl.ReplCompileResult +import java.io.File + + +interface CompileServiceAsync { + + suspend fun checkCompilerId(expectedCompilerId: CompilerId): Boolean + + suspend fun getUsedMemory(): CompileService.CallResult + + suspend fun getDaemonOptions(): CompileService.CallResult + + suspend fun getDaemonInfo(): CompileService.CallResult + + suspend fun getDaemonJVMOptions(): CompileService.CallResult + + suspend fun registerClient(aliveFlagPath: String?): CompileService.CallResult + + // TODO: (-old-) consider adding another client alive checking mechanism, e.g. socket/socketPort + + suspend fun getClients(): CompileService.CallResult> + + suspend fun leaseCompileSession(aliveFlagPath: String?): CompileService.CallResult + + suspend fun releaseCompileSession(sessionId: Int): CompileService.CallResult + + suspend fun shutdown(): CompileService.CallResult + + suspend fun scheduleShutdown(graceful: Boolean): CompileService.CallResult + + suspend fun compile( + sessionId: Int, + compilerArguments: Array, + compilationOptions: CompilationOptions, + servicesFacade: CompilerServicesFacadeBaseAsync, + compilationResults: CompilationResultsAsync? + ): CompileService.CallResult + + suspend fun clearJarCache() + + suspend fun releaseReplSession(sessionId: Int): CompileService.CallResult + + suspend fun leaseReplSession( + aliveFlagPath: String?, + compilerArguments: Array, + compilationOptions: CompilationOptions, + servicesFacade: CompilerServicesFacadeBaseAsync, + templateClasspath: List, + templateClassName: String + ): CompileService.CallResult + + suspend fun replCreateState(sessionId: Int): CompileService.CallResult + + suspend fun replCheck( + sessionId: Int, + replStateId: Int, + codeLine: ReplCodeLine + ): CompileService.CallResult + + suspend fun replCompile( + sessionId: Int, + replStateId: Int, + codeLine: ReplCodeLine + ): CompileService.CallResult + + suspend fun classesFqNamesByFiles(sessionId: Int, sourceFiles: Set): CompileService.CallResult> + + val serverPort: Int + get() = 0 + +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileServiceAsyncWrapper.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileServiceAsyncWrapper.kt new file mode 100644 index 00000000000..d919395ad65 --- /dev/null +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileServiceAsyncWrapper.kt @@ -0,0 +1,109 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common + +import org.jetbrains.kotlin.cli.common.repl.ReplCodeLine +import java.io.File + +class CompileServiceAsyncWrapper( + val rmiCompileService: CompileService +) : CompileServiceAsync { + + override suspend fun classesFqNamesByFiles(sessionId: Int, sourceFiles: Set) = + rmiCompileService.classesFqNamesByFiles(sessionId, sourceFiles) + + override suspend fun compile( + sessionId: Int, + compilerArguments: Array, + compilationOptions: CompilationOptions, + servicesFacade: CompilerServicesFacadeBaseAsync, + compilationResults: CompilationResultsAsync? + ) = rmiCompileService.compile( + sessionId, + compilerArguments, + compilationOptions, + servicesFacade.toRMI(), + compilationResults?.toRMI() + ) + + override suspend fun leaseReplSession( + aliveFlagPath: String?, + compilerArguments: Array, + compilationOptions: CompilationOptions, + servicesFacade: CompilerServicesFacadeBaseAsync, + templateClasspath: List, + templateClassName: String + ) = rmiCompileService.leaseReplSession( + aliveFlagPath, + compilerArguments, + compilationOptions, + servicesFacade.toRMI(), + templateClasspath, + templateClassName + ) + + override suspend fun replCreateState(sessionId: Int) = + rmiCompileService.replCreateState(sessionId).toClient() + + override suspend fun getUsedMemory() = + rmiCompileService.getUsedMemory() + + override suspend fun getDaemonOptions() = + rmiCompileService.getDaemonOptions() + + + override suspend fun getDaemonInfo() = + rmiCompileService.getDaemonInfo() + + + override suspend fun getDaemonJVMOptions() = + rmiCompileService.getDaemonJVMOptions() + + override suspend fun registerClient(aliveFlagPath: String?) = + rmiCompileService.registerClient(aliveFlagPath) + + override suspend fun getClients() = + rmiCompileService.getClients() + + + override suspend fun leaseCompileSession(aliveFlagPath: String?) = + rmiCompileService.leaseCompileSession(aliveFlagPath) + + + override suspend fun releaseCompileSession(sessionId: Int) = + rmiCompileService.releaseCompileSession(sessionId) + + + override suspend fun shutdown() = + rmiCompileService.shutdown() + + + override suspend fun scheduleShutdown(graceful: Boolean) = + rmiCompileService.scheduleShutdown(graceful) + + override suspend fun clearJarCache() = + rmiCompileService.clearJarCache() + + + override suspend fun releaseReplSession(sessionId: Int) = + rmiCompileService.releaseReplSession(sessionId) + + + override suspend fun replCheck(sessionId: Int, replStateId: Int, codeLine: ReplCodeLine) = + rmiCompileService.replCheck(sessionId, replStateId, codeLine) + + override suspend fun replCompile( + sessionId: Int, + replStateId: Int, + codeLine: ReplCodeLine + ) = rmiCompileService.replCompile(sessionId, replStateId, codeLine) + + override suspend fun checkCompilerId(expectedCompilerId: CompilerId) = + rmiCompileService.checkCompilerId(expectedCompilerId) + +} + +fun CompileService.toClient() = CompileServiceAsyncWrapper(this) diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileServiceRMIWrapper.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileServiceRMIWrapper.kt new file mode 100644 index 00000000000..2d4547bfbed --- /dev/null +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileServiceRMIWrapper.kt @@ -0,0 +1,205 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common + +import kotlinx.coroutines.runBlocking +import org.jetbrains.kotlin.cli.common.repl.ReplCheckResult +import org.jetbrains.kotlin.cli.common.repl.ReplCodeLine +import org.jetbrains.kotlin.cli.common.repl.ReplCompileResult +import org.jetbrains.kotlin.cli.common.repl.ReplEvalResult +import java.io.File + +class CompileServiceClientRMIWrapper( + val asyncCompileService: CompileServiceAsync +) : CompileService { + + override fun classesFqNamesByFiles(sessionId: Int, sourceFiles: Set) = runBlocking { + asyncCompileService.classesFqNamesByFiles(sessionId, sourceFiles) + } + + private fun reportNotImplemented(): Nothing = throw IllegalStateException("Unexpected call to deprecated method") + + // deprecated methods : + override fun remoteCompile( + sessionId: Int, + targetPlatform: CompileService.TargetPlatform, + args: Array, + servicesFacade: CompilerCallbackServicesFacade, + compilerOutputStream: RemoteOutputStream, + outputFormat: CompileService.OutputFormat, + serviceOutputStream: RemoteOutputStream, + operationsTracer: RemoteOperationsTracer? + ): CompileService.CallResult { + reportNotImplemented() + } + + override fun remoteIncrementalCompile( + sessionId: Int, + targetPlatform: CompileService.TargetPlatform, + args: Array, + servicesFacade: CompilerCallbackServicesFacade, + compilerOutputStream: RemoteOutputStream, + compilerOutputFormat: CompileService.OutputFormat, + serviceOutputStream: RemoteOutputStream, + operationsTracer: RemoteOperationsTracer? + ): CompileService.CallResult { + reportNotImplemented() + } + + override fun leaseReplSession( + aliveFlagPath: String?, + targetPlatform: CompileService.TargetPlatform, + servicesFacade: CompilerCallbackServicesFacade, + templateClasspath: List, + templateClassName: String, + scriptArgs: Array?, + scriptArgsTypes: Array>?, + compilerMessagesOutputStream: RemoteOutputStream, + evalOutputStream: RemoteOutputStream?, + evalErrorStream: RemoteOutputStream?, + evalInputStream: RemoteInputStream?, + operationsTracer: RemoteOperationsTracer? + ): CompileService.CallResult { + reportNotImplemented() + } + + override fun remoteReplLineCheck(sessionId: Int, codeLine: ReplCodeLine): CompileService.CallResult { + reportNotImplemented() + } + + override fun remoteReplLineCompile( + sessionId: Int, + codeLine: ReplCodeLine, + history: List? + ): CompileService.CallResult { + reportNotImplemented() + } + + override fun remoteReplLineEval( + sessionId: Int, + codeLine: ReplCodeLine, + history: List? + ): CompileService.CallResult { + reportNotImplemented() + } + + // normal methods: + override fun compile( + sessionId: Int, + compilerArguments: Array, + compilationOptions: CompilationOptions, + servicesFacade: CompilerServicesFacadeBase, + compilationResults: CompilationResults? + ) = runBlocking { + asyncCompileService.compile( + sessionId, + compilerArguments, + compilationOptions, + servicesFacade.toClient(), + compilationResults?.toClient() // TODO + ) + } + + + override fun leaseReplSession( + aliveFlagPath: String?, + compilerArguments: Array, + compilationOptions: CompilationOptions, + servicesFacade: CompilerServicesFacadeBase, + templateClasspath: List, + templateClassName: String + ) = runBlocking { + asyncCompileService.leaseReplSession( + aliveFlagPath, + compilerArguments, + compilationOptions, + servicesFacade.toClient(), + templateClasspath, + templateClassName + ) + } + + override fun replCreateState(sessionId: Int) = runBlocking { + asyncCompileService.replCreateState(sessionId) + }.toRMI() + + override fun getUsedMemory() = runBlocking { + asyncCompileService.getUsedMemory() + } + + override fun getDaemonOptions() = runBlocking { + asyncCompileService.getDaemonOptions() + } + + override fun getDaemonInfo() = runBlocking { + asyncCompileService.getDaemonInfo() + } + + + override fun getDaemonJVMOptions() = runBlocking { + asyncCompileService.getDaemonJVMOptions() + } + + override fun registerClient(aliveFlagPath: String?) = runBlocking { + asyncCompileService.registerClient(aliveFlagPath) + } + + override fun getClients() = runBlocking { + asyncCompileService.getClients() + } + + + override fun leaseCompileSession(aliveFlagPath: String?) = runBlocking { + asyncCompileService.leaseCompileSession(aliveFlagPath) + } + + + override fun releaseCompileSession(sessionId: Int) = runBlocking { + asyncCompileService.releaseCompileSession(sessionId) + } + + + override fun shutdown() = runBlocking { + asyncCompileService.shutdown() + } + + + override fun scheduleShutdown(graceful: Boolean) = runBlocking { + asyncCompileService.scheduleShutdown(graceful) + } + + override fun clearJarCache() = runBlocking { + asyncCompileService.clearJarCache() + } + + + override fun releaseReplSession(sessionId: Int) = runBlocking { + asyncCompileService.releaseReplSession(sessionId) + } + + + override fun replCheck(sessionId: Int, replStateId: Int, codeLine: ReplCodeLine) = runBlocking { + asyncCompileService.replCheck(sessionId, replStateId, codeLine) + } + + override fun replCompile( + sessionId: Int, + replStateId: Int, + codeLine: ReplCodeLine + ) = runBlocking { + asyncCompileService.replCompile(sessionId, replStateId, codeLine) + } + + override fun checkCompilerId(expectedCompilerId: CompilerId) = runBlocking { + asyncCompileService.checkCompilerId(expectedCompilerId) + } + +} + +fun CompileServiceAsync.toRMI() = when (this) { + is CompileServiceAsyncWrapper -> this.rmiCompileService + else -> CompileServiceClientRMIWrapper(this) +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerCallbackServicesFacadeAsync.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerCallbackServicesFacadeAsync.kt new file mode 100644 index 00000000000..dfa628f1f6e --- /dev/null +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerCallbackServicesFacadeAsync.kt @@ -0,0 +1,49 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common + +import org.jetbrains.kotlin.incremental.components.LookupInfo +import org.jetbrains.kotlin.load.kotlin.incremental.components.JvmPackagePartProto +import org.jetbrains.kotlin.modules.TargetId + +interface CompilerCallbackServicesFacadeAsync : CompilerServicesFacadeBaseAsync { + + suspend fun hasIncrementalCaches(): Boolean + + suspend fun hasLookupTracker(): Boolean + + suspend fun hasCompilationCanceledStatus(): Boolean + + // ---------------------------------------------------- + // IncrementalCache + suspend fun incrementalCache_getObsoletePackageParts(target: TargetId): Collection + + suspend fun incrementalCache_getObsoleteMultifileClassFacades(target: TargetId): Collection + + suspend fun incrementalCache_getPackagePartData(target: TargetId, partInternalName: String): JvmPackagePartProto? + + suspend fun incrementalCache_getModuleMappingData(target: TargetId): ByteArray? + + suspend fun incrementalCache_registerInline(target: TargetId, fromPath: String, jvmSignature: String, toPath: String) + + suspend fun incrementalCache_getClassFilePath(target: TargetId, internalClassName: String): String + + suspend fun incrementalCache_close(target: TargetId) + + suspend fun incrementalCache_getMultifileFacadeParts(target: TargetId, internalName: String): Collection? + + // ---------------------------------------------------- + // LookupTracker + suspend fun lookupTracker_requiresPosition(): Boolean + + fun lookupTracker_record(lookups: Collection) + + suspend fun lookupTracker_isDoNothing(): Boolean + + // ---------------------------------------------------- + // CompilationCanceledStatus + suspend fun compilationCanceledStatus_checkCanceled(): Void? +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerServicesFacadeBaseAsync.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerServicesFacadeBaseAsync.kt new file mode 100644 index 00000000000..5ce066d11d7 --- /dev/null +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerServicesFacadeBaseAsync.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common + +import java.io.Serializable + +interface CompilerServicesFacadeBaseAsync { + /** + * Reports different kind of diagnostic messages from compile daemon to compile daemon clients (jps, gradle, ...) + */ + suspend fun report(category: Int, severity: Int, message: String?, attachment: Serializable?) +} + +suspend fun CompilerServicesFacadeBaseAsync.report( + category: ReportCategory, + severity: ReportSeverity, + message: String? = null, + attachment: Serializable? = null +) { + report(category.code, severity.code, message, attachment) +} + diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerServicesFacadeBaseAsyncWrapper.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerServicesFacadeBaseAsyncWrapper.kt new file mode 100644 index 00000000000..9fab8ea1bac --- /dev/null +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerServicesFacadeBaseAsyncWrapper.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common + +import java.io.Serializable + + +class CompilerServicesFacadeBaseAsyncWrapper( + val rmiImpl: CompilerServicesFacadeBase +) : CompilerServicesFacadeBaseAsync { + + override suspend fun report(category: Int, severity: Int, message: String?, attachment: Serializable?) = + rmiImpl.report(category, severity, message, attachment) + +} + +fun CompilerServicesFacadeBase.toClient() = + if (this is CompilerServicesFacadeBaseRMIWrapper) this.clientSide + else CompilerServicesFacadeBaseAsyncWrapper(this) diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerServicesFacadeBaseRMIWrapper.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerServicesFacadeBaseRMIWrapper.kt new file mode 100644 index 00000000000..8259b060444 --- /dev/null +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilerServicesFacadeBaseRMIWrapper.kt @@ -0,0 +1,21 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common + +import kotlinx.coroutines.runBlocking +import java.io.Serializable + +class CompilerServicesFacadeBaseRMIWrapper(val clientSide: CompilerServicesFacadeBaseAsync) : CompilerServicesFacadeBase, Serializable { + + override fun report(category: Int, severity: Int, message: String?, attachment: Serializable?) = runBlocking { + clientSide.report(category, severity, message, attachment) + } + +} + +fun CompilerServicesFacadeBaseAsync.toRMI() = + if (this is CompilerServicesFacadeBaseAsyncWrapper) this.rmiImpl + else CompilerServicesFacadeBaseRMIWrapper(this) diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonProtocolVariant.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonProtocolVariant.kt new file mode 100644 index 00000000000..a62b4fde010 --- /dev/null +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonProtocolVariant.kt @@ -0,0 +1,10 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common + +enum class DaemonProtocolVariant { + RMI, SOCKETS +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/IncrementalCompilerServicesFacadeAsync.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/IncrementalCompilerServicesFacadeAsync.kt new file mode 100644 index 00000000000..42a2d7929c9 --- /dev/null +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/IncrementalCompilerServicesFacadeAsync.kt @@ -0,0 +1,9 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common + + +interface IncrementalCompilerServicesFacadeAsync : CompilerServicesFacadeBaseAsync \ No newline at end of file diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/PerfUtils.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/PerfUtils.kt index ceec3e876f0..ef8791d0a2b 100644 --- a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/PerfUtils.kt +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/PerfUtils.kt @@ -34,7 +34,15 @@ interface Profiler { fun getCounters(): Map fun getTotalCounters(): PerfCounters - fun withMeasure(obj: Any?, body: () -> R): R + fun beginMeasure(obj: Any?) : List = listOf() + fun endMeasure(obj: Any?, startState: List) {} +} + +inline fun Profiler.withMeasure(obj: Any?, body: () -> R): R { + val startState = beginMeasure(obj) + val res = body() + endMeasure(obj, startState) + return res } @@ -85,57 +93,64 @@ inline fun usedMemory(withGC: Boolean): Long { } -inline fun withMeasureWallTime(perfCounters: PerfCounters, body: () -> R): R { - val startTime = System.nanoTime() - val res = body() +inline fun beginWithMeasureWallTime(perfCounters: PerfCounters) = listOf(System.nanoTime()) + +inline fun endWithMeasureWallTime(perfCounters: PerfCounters, startState: List) { + val (startTime) = startState perfCounters.addMeasurement(time = System.nanoTime() - startTime) // TODO: add support for time wrapping - return res } -inline fun withMeasureWallAndThreadTimes(perfCounters: PerfCounters, threadMXBean: ThreadMXBean, body: () -> R): R { +inline fun beginWithMeasureWallAndThreadTimes(perfCounters: PerfCounters, threadMXBean: ThreadMXBean): List { val startTime = System.nanoTime() val startThreadTime = threadMXBean.threadCpuTime() val startThreadUserTime = threadMXBean.threadUserTime() + return listOf(startTime, startThreadTime, startThreadUserTime) +} - val res = body() +inline fun endWithMeasureWallAndThreadTimes(perfCounters: PerfCounters, threadMXBean: ThreadMXBean, startState: List) { + val (startTime, startThreadTime, startThreadUserTime) = startState // TODO: add support for time wrapping perfCounters.addMeasurement(time = System.nanoTime() - startTime, thread = threadMXBean.threadCpuTime() - startThreadTime, threadUser = threadMXBean.threadUserTime() - startThreadUserTime) - return res } -inline fun withMeasureWallAndThreadTimes(perfCounters: PerfCounters, body: () -> R): R = withMeasureWallAndThreadTimes(perfCounters, ManagementFactory.getThreadMXBean(), body) +inline fun beginWithMeasureWallAndThreadTimes(perfCounters: PerfCounters) = + beginWithMeasureWallAndThreadTimes(perfCounters, ManagementFactory.getThreadMXBean()) +inline fun endWithMeasureWallAndThreadTimes(perfCounters: PerfCounters, startState: List) = + endWithMeasureWallAndThreadTimes(perfCounters, ManagementFactory.getThreadMXBean(), startState) - -inline fun withMeasureWallAndThreadTimesAndMemory(perfCounters: PerfCounters, withGC: Boolean = false, threadMXBean: ThreadMXBean, body: () -> R): R { +inline fun beginWithMeasureWallAndThreadTimesAndMemory(perfCounters: PerfCounters, withGC: Boolean = false, threadMXBean: ThreadMXBean): List { val startMem = usedMemory(withGC) val startTime = System.nanoTime() val startThreadTime = threadMXBean.threadCpuTime() val startThreadUserTime = threadMXBean.threadUserTime() - val res = body() + return listOf(startMem, startTime, startThreadTime, startThreadUserTime) +} + +inline fun endWithMeasureWallAndThreadTimesAndMemory(perfCounters: PerfCounters, withGC: Boolean = false, threadMXBean: ThreadMXBean, startState: List){ + val (startMem, startTime, startThreadTime, startThreadUserTime) = startState // TODO: add support for time wrapping perfCounters.addMeasurement(time = System.nanoTime() - startTime, thread = threadMXBean.threadCpuTime() - startThreadTime, threadUser = threadMXBean.threadUserTime() - startThreadUserTime, memory = usedMemory(withGC) - startMem) - return res } -inline fun withMeasureWallAndThreadTimesAndMemory(perfCounters: PerfCounters, withGC: Boolean, body: () -> R): R = - withMeasureWallAndThreadTimesAndMemory(perfCounters, withGC, ManagementFactory.getThreadMXBean(), body) +inline fun beginWithMeasureWallAndThreadTimesAndMemory(perfCounters: PerfCounters, withGC: Boolean) = + beginWithMeasureWallAndThreadTimesAndMemory(perfCounters, withGC, ManagementFactory.getThreadMXBean()) + +inline fun endWithMeasureWallAndThreadTimesAndMemory(perfCounters: PerfCounters, withGC: Boolean, startState: List) = + endWithMeasureWallAndThreadTimesAndMemory(perfCounters, withGC, ManagementFactory.getThreadMXBean(), startState) class DummyProfiler : Profiler { override fun getCounters(): Map = mapOf(null to SimplePerfCounters()) override fun getTotalCounters(): PerfCounters = SimplePerfCounters() - - @Suppress("OVERRIDE_BY_INLINE") - override inline fun withMeasure(obj: Any?, body: () -> R): R = body() } @@ -151,19 +166,27 @@ abstract class TotalProfiler : Profiler { class WallTotalProfiler : TotalProfiler() { @Suppress("OVERRIDE_BY_INLINE") - override inline fun withMeasure(obj: Any?, body: () -> R): R = withMeasureWallTime(total, body) + override inline fun beginMeasure(obj: Any?) = beginWithMeasureWallTime(total) + @Suppress("OVERRIDE_BY_INLINE") + override inline fun endMeasure(obj: Any?, startState: List) = endWithMeasureWallTime(total, startState) } class WallAndThreadTotalProfiler : TotalProfiler() { @Suppress("OVERRIDE_BY_INLINE") - override inline fun withMeasure(obj: Any?, body: () -> R): R = withMeasureWallAndThreadTimes(total, threadMXBean, body) + override inline fun beginMeasure(obj: Any?) = beginWithMeasureWallAndThreadTimes(total, threadMXBean) + @Suppress("OVERRIDE_BY_INLINE") + override inline fun endMeasure(obj: Any?, startState: List) = endWithMeasureWallAndThreadTimes(total, threadMXBean, startState) } class WallAndThreadAndMemoryTotalProfiler(val withGC: Boolean) : TotalProfiler() { @Suppress("OVERRIDE_BY_INLINE") - override inline fun withMeasure(obj: Any?, body: () -> R): R = withMeasureWallAndThreadTimesAndMemory(total, withGC, threadMXBean, body) + override inline fun beginMeasure(obj: Any?) = + beginWithMeasureWallAndThreadTimesAndMemory(total, withGC, threadMXBean) + @Suppress("OVERRIDE_BY_INLINE") + override inline fun endMeasure(obj: Any?, startState: List) = + endWithMeasureWallAndThreadTimesAndMemory(total, withGC, threadMXBean, startState) } @@ -174,6 +197,9 @@ class WallAndThreadByClassProfiler() : TotalProfiler() { override fun getCounters(): Map = counters @Suppress("OVERRIDE_BY_INLINE") - override inline fun withMeasure(obj: Any?, body: () -> R): R = - withMeasureWallAndThreadTimes(counters.getOrPut(obj?.javaClass?.name, { SimplePerfCountersWithTotal(total) }), threadMXBean, body) + override inline fun beginMeasure(obj: Any?) = + beginWithMeasureWallAndThreadTimes(counters.getOrPut(obj?.javaClass?.name, { SimplePerfCountersWithTotal(total) }), threadMXBean) + @Suppress("OVERRIDE_BY_INLINE") + override inline fun endMeasure(obj: Any?, startState: List) = + endWithMeasureWallAndThreadTimes(counters.getOrPut(obj?.javaClass?.name, { SimplePerfCountersWithTotal(total) }), threadMXBean, startState) } diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ReplStateFacadeAsync.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ReplStateFacadeAsync.kt new file mode 100644 index 00000000000..b230e2ac319 --- /dev/null +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ReplStateFacadeAsync.kt @@ -0,0 +1,20 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common + +import org.jetbrains.kotlin.cli.common.repl.ILineId + +interface ReplStateFacadeAsync { + suspend fun getId(): Int + + suspend fun getHistorySize(): Int + + suspend fun historyGet(index: Int): ILineId + + suspend fun historyReset(): List + + suspend fun historyResetTo(id: ILineId): List +} diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ReplStateFacadeAsyncWrapper.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ReplStateFacadeAsyncWrapper.kt new file mode 100644 index 00000000000..f7717a32aa8 --- /dev/null +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ReplStateFacadeAsyncWrapper.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common + +import org.jetbrains.kotlin.cli.common.repl.ILineId + +class ReplStateFacadeAsyncWrapper(val rmiReplStateFacade: ReplStateFacade) : ReplStateFacadeAsync { + + override suspend fun getId() = rmiReplStateFacade.getId() + + override suspend fun getHistorySize() = rmiReplStateFacade.getHistorySize() + + override suspend fun historyGet(index: Int) = rmiReplStateFacade.historyGet(index) + + override suspend fun historyReset() = rmiReplStateFacade.historyReset() + + override suspend fun historyResetTo(id: ILineId) = rmiReplStateFacade.historyResetTo(id) + +} + +fun ReplStateFacade.toClient() = ReplStateFacadeAsyncWrapper(this) +fun CompileService.CallResult.toClient() = when (this) { + is CompileService.CallResult.Good -> CompileService.CallResult.Good(this.result.toClient()) + is CompileService.CallResult.Dying -> this + is CompileService.CallResult.Error -> this + is CompileService.CallResult.Ok -> this +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ReplStateFacadeRMIWrapper.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ReplStateFacadeRMIWrapper.kt new file mode 100644 index 00000000000..d138a675060 --- /dev/null +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ReplStateFacadeRMIWrapper.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common + +import kotlinx.coroutines.runBlocking +import org.jetbrains.kotlin.cli.common.repl.ILineId +import java.io.Serializable + +class ReplStateFacadeRMIWrapper(val clientSide: ReplStateFacadeAsync) : ReplStateFacade, Serializable { + + override fun getId() = runBlocking { clientSide.getId() } + + override fun getHistorySize() = runBlocking { clientSide.getHistorySize() } + + override fun historyGet(index: Int) = runBlocking { clientSide.historyGet(index) } + + override fun historyReset() = runBlocking { clientSide.historyReset() } + + override fun historyResetTo(id: ILineId) = runBlocking { clientSide.historyResetTo(id) } + +} + +fun ReplStateFacadeAsync.toRMI() = ReplStateFacadeRMIWrapper(this) +fun CompileService.CallResult.toRMI() = when (this) { + is CompileService.CallResult.Good -> CompileService.CallResult.Good(this.result.toRMI()) + is CompileService.CallResult.Dying -> this + is CompileService.CallResult.Error -> this + is CompileService.CallResult.Ok -> this +} \ No newline at end of file diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/perfUtilsAsync.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/perfUtilsAsync.kt new file mode 100644 index 00000000000..af171bb7b6e --- /dev/null +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/perfUtilsAsync.kt @@ -0,0 +1,10 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.common + +import kotlinx.coroutines.runBlocking + +fun Profiler.withMeasureBlocking(obj: Any?, body: suspend () -> R): R = runBlocking { withMeasure(obj) { body() } } \ No newline at end of file diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt index a5395103949..68312737bd0 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt @@ -87,8 +87,8 @@ interface EventManager { fun onCompilationFinished(f: () -> Unit) } -private class EventManagerImpl : EventManager { - private val onCompilationFinished = arrayListOf<() -> Unit>() +class EventManagerImpl : EventManager { + val onCompilationFinished = arrayListOf<() -> Unit>() override fun onCompilationFinished(f: () -> Unit) { onCompilationFinished.add(f) @@ -99,25 +99,20 @@ private class EventManagerImpl : EventManager { } } -class CompileServiceImpl( - val registry: Registry, - val compiler: CompilerSelector, - val compilerId: CompilerId, +abstract class CompileServiceImplBase( val daemonOptions: DaemonOptions, - val daemonJVMOptions: DaemonJVMOptions, + val compilerId: CompilerId, val port: Int, - val timer: Timer, - val onShutdown: () -> Unit -) : CompileService { - - private val log by lazy { Logger.getLogger("compiler") } + val timer: Timer +) { + protected val log by lazy { Logger.getLogger("compiler") } init { System.setProperty(KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY, "true") } // wrapped in a class to encapsulate alive check logic - private class ClientOrSessionProxy( + protected class ClientOrSessionProxy( val aliveFlagPath: String?, val data: T? = null, private var disposable: Disposable? = null @@ -133,9 +128,9 @@ class CompileServiceImpl( } } - private val compilationsCounter = AtomicInteger(0) + protected val compilationsCounter = AtomicInteger(0) - private val classpathWatcher = LazyClasspathWatcher(compilerId.compilerClasspath) + protected val classpathWatcher = LazyClasspathWatcher(compilerId.compilerClasspath) enum class Aliveness { // !!! ordering of values is used in state comparison @@ -143,7 +138,7 @@ class CompileServiceImpl( LastSession, Alive } - private class SessionsContainer { + protected class SessionsContainer { private val lock = ReentrantReadWriteLock() private val sessions: MutableMap> = hashMapOf() @@ -183,8 +178,7 @@ class CompileServiceImpl( } // TODO: encapsulate operations on state here - private val state = object { - + protected class CompileServiceState { private val clientsLock = ReentrantReadWriteLock() private val clientProxies: MutableSet> = hashSetOf() @@ -211,42 +205,42 @@ class CompileServiceImpl( clientProxies.mapNotNull { it.aliveFlagPath } } + private inline fun Iterable.cleanMatching( + lock: ReentrantReadWriteLock, + crossinline pred: (T) -> Boolean, + crossinline clean: (T) -> Unit + ): Boolean { + var anyDead = false + lock.read { + val toRemove = filter(pred) + if (toRemove.isNotEmpty()) { + anyDead = true + lock.write { + toRemove.forEach(clean) + } + } + } + return anyDead + } + fun cleanDeadClients(): Boolean = clientProxies.cleanMatching(clientsLock, { !it.isAlive }, { if (clientProxies.remove(it)) it.dispose() }) } - private fun Int.toAlivenessName(): String = + protected val state = CompileServiceState() + + protected fun Int.toAlivenessName(): String = try { Aliveness.values()[this].name } catch (_: Throwable) { "invalid($this)" } - private inline fun Iterable.cleanMatching( - lock: ReentrantReadWriteLock, - crossinline pred: (T) -> Boolean, - crossinline clean: (T) -> Unit - ): Boolean { - var anyDead = false - lock.read { - val toRemove = filter(pred) - if (toRemove.isNotEmpty()) { - anyDead = true - lock.write { - toRemove.forEach(clean) - } - } - } - return anyDead - } - @Volatile - private var _lastUsedSeconds = nowSeconds() - val lastUsedSeconds: Long get() = if (rwlock.isWriteLocked || rwlock.readLockCount - rwlock.readHoldCount > 0) nowSeconds() else _lastUsedSeconds + protected var _lastUsedSeconds = nowSeconds() + abstract protected val lastUsedSeconds: Long - private val rwlock = ReentrantReadWriteLock() - - private var runFile: File + protected var runFile: File init { val runFileDir = File(daemonOptions.runFilesPathOrDefault) @@ -267,6 +261,411 @@ class CompileServiceImpl( runFile.deleteOnExit() } + protected fun postReleaseCompileSession(): CompileService.CallResult { + if (state.sessions.isEmpty()) { + // TODO: and some goes here + } + timer.schedule(0) { + periodicAndAfterSessionCheck() + } + return CompileService.CallResult.Ok() + } + + protected abstract fun periodicAndAfterSessionCheck() + protected abstract fun periodicSeldomCheck() + protected abstract fun initiateElections() + + protected inline fun exceptionLoggingTimerThread(body: () -> Unit) { + try { + body() + } catch (e: Throwable) { + System.err.println("Exception in timer thread: " + e.message) + e.printStackTrace(System.err) + log.log(Level.SEVERE, "Exception in timer thread", e) + } + } + + protected inline fun compileImpl( + sessionId: Int, + compilerArguments: Array, + compilationOptions: CompilationOptions, + servicesFacade: ServicesFacadeT, + compilationResults: CompilationResultsT, + hasIncrementalCaches: JpsServicesFacadeT.() -> Boolean, + createMessageCollector: (ServicesFacadeT, CompilationOptions) -> MessageCollector, + createReporter: (ServicesFacadeT, CompilationOptions) -> DaemonMessageReporter, + createServices: (JpsServicesFacadeT, EventManager, Profiler) -> Services, + getICReporter: (ServicesFacadeT, CompilationResultsT?, IncrementalCompilationOptions) -> RemoteICReporter + ) = kotlin.run { + val messageCollector = createMessageCollector(servicesFacade, compilationOptions) + val daemonReporter = createReporter(servicesFacade, compilationOptions) + val targetPlatform = compilationOptions.targetPlatform + log.info("Starting compilation with args: " + compilerArguments.joinToString(" ")) + + @Suppress("UNCHECKED_CAST") + val compiler = when (targetPlatform) { + CompileService.TargetPlatform.JVM -> K2JVMCompiler() + CompileService.TargetPlatform.JS -> K2JSCompiler() + CompileService.TargetPlatform.METADATA -> K2MetadataCompiler() + } as CLICompiler + + val k2PlatformArgs = compiler.createArguments() + parseCommandLineArguments(compilerArguments.asList(), k2PlatformArgs) + val argumentParseError = validateArguments(k2PlatformArgs.errors) + + if (argumentParseError != null) { + messageCollector.report(CompilerMessageSeverity.ERROR, argumentParseError) + CompileService.CallResult.Good(ExitCode.COMPILATION_ERROR.code) + } else when (compilationOptions.compilerMode) { + CompilerMode.JPS_COMPILER -> { + servicesFacade as JpsServicesFacadeT + withIC(enabled = servicesFacade.hasIncrementalCaches()) { + doCompile(sessionId, daemonReporter, tracer = null) { eventManger, profiler -> + val services = createServices(servicesFacade, eventManger, profiler) + compiler.exec(messageCollector, services, k2PlatformArgs) + } + } + } + CompilerMode.NON_INCREMENTAL_COMPILER -> { + doCompile(sessionId, daemonReporter, tracer = null) { _, _ -> + compiler.exec(messageCollector, Services.EMPTY, k2PlatformArgs) + } + } + CompilerMode.INCREMENTAL_COMPILER -> { + val gradleIncrementalArgs = compilationOptions as IncrementalCompilationOptions + val gradleIncrementalServicesFacade = servicesFacade + + when (targetPlatform) { + CompileService.TargetPlatform.JVM -> withIC { + doCompile(sessionId, daemonReporter, tracer = null) { _, _ -> + execIncrementalCompiler( + k2PlatformArgs as K2JVMCompilerArguments, + gradleIncrementalArgs, + messageCollector, + getICReporter( + gradleIncrementalServicesFacade, + compilationResults!!, + gradleIncrementalArgs + ) + ) + } + } + CompileService.TargetPlatform.JS -> withJsIC { + doCompile(sessionId, daemonReporter, tracer = null) { _, _ -> + execJsIncrementalCompiler( + k2PlatformArgs as K2JSCompilerArguments, + gradleIncrementalArgs, + messageCollector, + getICReporter( + gradleIncrementalServicesFacade, + compilationResults!!, + gradleIncrementalArgs + ) + ) + } + } + else -> throw IllegalStateException("Incremental compilation is not supported for target platform: $targetPlatform") + + } + } + else -> throw IllegalStateException("Unknown compilation mode ${compilationOptions.compilerMode}") + } + } + + + protected inline fun doCompile( + sessionId: Int, + daemonMessageReporter: DaemonMessageReporter, + tracer: RemoteOperationsTracer?, + body: (EventManager, Profiler) -> ExitCode + ): CompileService.CallResult = run { + log.fine("alive!") + withValidClientOrSessionProxy(sessionId) { + tracer?.before("compile") + val rpcProfiler = if (daemonOptions.reportPerf) WallAndThreadTotalProfiler() else DummyProfiler() + val eventManager = EventManagerImpl() + try { + log.fine("trying get exitCode") + val exitCode = checkedCompile(daemonMessageReporter, rpcProfiler) { + body(eventManager, rpcProfiler).code + } + log.fine("got exitCode") + CompileService.CallResult.Good(exitCode) + } finally { + eventManager.fireCompilationFinished() + tracer?.after("compile") + } + } + } + + fun Long.ms() = TimeUnit.NANOSECONDS.toMillis(this) + fun Long.kb() = this / 1024 + + protected inline fun checkedCompile( + daemonMessageReporter: DaemonMessageReporter, + rpcProfiler: Profiler, + body: () -> R + ): R { + try { + val profiler = if (daemonOptions.reportPerf) WallAndThreadAndMemoryTotalProfiler(withGC = false) else DummyProfiler() + + val res = profiler.withMeasure(null, body) + + val endMem = if (daemonOptions.reportPerf) usedMemory(withGC = false) else 0L + + log.info("Done with result " + res.toString()) + + if (daemonOptions.reportPerf) { + val pc = profiler.getTotalCounters() + val rpc = rpcProfiler.getTotalCounters() + + "PERF: Compile on daemon: ${pc.time.ms()} ms; thread: user ${pc.threadUserTime.ms()} ms, sys ${(pc.threadTime - pc.threadUserTime).ms()} ms; rpc: ${rpc.count} calls, ${rpc.time.ms()} ms, thread ${rpc.threadTime.ms()} ms; memory: ${endMem.kb()} kb (${"%+d".format( + pc.memory.kb() + )} kb)".let { + daemonMessageReporter.report(ReportSeverity.INFO, it) + log.info(it) + } + + // this will only be reported if if appropriate (e.g. ByClass) profiler is used + for ((obj, counters) in rpcProfiler.getCounters()) { + "PERF: rpc by $obj: ${counters.count} calls, ${counters.time.ms()} ms, thread ${counters.threadTime.ms()} ms".let { + daemonMessageReporter.report(ReportSeverity.INFO, it) + log.info(it) + } + } + } + return res + } + // TODO: consider possibilities to handle OutOfMemory + catch (e: Throwable) { + log.log( + Level.SEVERE, + "Exception: $e\n ${e.stackTrace.joinToString("\n ")}${ + if (e.cause != null && e.cause != e) { + "\nCaused by: ${e.cause}\n ${e.cause!!.stackTrace.joinToString("\n ")}" + } else "" + }" + ) + throw e + } + } + + + // ----------------------------------------------------------------------- + // internal implementation stuff + + // TODO: consider matching compilerId coming from outside with actual one + // private val selfCompilerId by lazy { + // CompilerId( + // compilerClasspath = System.getProperty("java.class.path") + // ?.split(File.pathSeparator) + // ?.map { File(it) } + // ?.filter { it.exists() } + // ?.map { it.absolutePath } + // ?: listOf(), + // compilerVersion = loadKotlinVersionFromResource() + // ) + // } + + fun startDaemonLife() { + timer.schedule(10) { + exceptionLoggingTimerThread { initiateElections() } + } + timer.schedule(delay = DAEMON_PERIODIC_CHECK_INTERVAL_MS, period = DAEMON_PERIODIC_CHECK_INTERVAL_MS) { + exceptionLoggingTimerThread { periodicAndAfterSessionCheck() } + } + timer.schedule(delay = DAEMON_PERIODIC_SELDOM_CHECK_INTERVAL_MS + 100, period = DAEMON_PERIODIC_SELDOM_CHECK_INTERVAL_MS) { + exceptionLoggingTimerThread { periodicSeldomCheck() } + } + } + + + protected inline fun ifAliveChecksImpl( + minAliveness: Aliveness = Aliveness.LastSession, + body: () -> CompileService.CallResult + ): CompileService.CallResult { + val curState = state.alive.get() + return when { + curState < minAliveness.ordinal -> { + log.info("Cannot perform operation, requested state: ${minAliveness.name} > actual: ${curState.toAlivenessName()}") + CompileService.CallResult.Dying() + } + else -> { + try { + body() + } catch (e: Throwable) { + log.log(Level.SEVERE, "Exception", e) + CompileService.CallResult.Error(e.message ?: "unknown") + } + } + } + } + + protected inline fun withValidClientOrSessionProxy( + sessionId: Int, + body: (ClientOrSessionProxy?) -> CompileService.CallResult + ): CompileService.CallResult { + val session: ClientOrSessionProxy? = + if (sessionId == CompileService.NO_SESSION) null + else state.sessions[sessionId] ?: return CompileService.CallResult.Error("Unknown or invalid session $sessionId") + try { + compilationsCounter.incrementAndGet() + return body(session) + } finally { + _lastUsedSeconds = nowSeconds() + } + } + + protected inline fun withValidReplImpl( + sessionId: Int, + body: KotlinJvmReplServiceT.() -> R + ): CompileService.CallResult = + withValidClientOrSessionProxy(sessionId) { session -> + (session?.data as? KotlinJvmReplServiceT?)?.let { + CompileService.CallResult.Good(it.body()) + } ?: CompileService.CallResult.Error("Not a REPL session $sessionId") + } + + protected fun execJsIncrementalCompiler( + args: K2JSCompilerArguments, + incrementalCompilationOptions: IncrementalCompilationOptions, + compilerMessageCollector: MessageCollector, + reporter: ICReporter + ): ExitCode { + val allKotlinFiles = arrayListOf() + val freeArgsWithoutKotlinFiles = arrayListOf() + args.freeArgs.forEach { + if (it.endsWith(".kt") && File(it).exists()) { + allKotlinFiles.add(File(it)) + } else { + freeArgsWithoutKotlinFiles.add(it) + } + } + args.freeArgs = freeArgsWithoutKotlinFiles + + val changedFiles = if (incrementalCompilationOptions.areFileChangesKnown) { + ChangedFiles.Known(incrementalCompilationOptions.modifiedFiles!!, incrementalCompilationOptions.deletedFiles!!) + } else { + ChangedFiles.Unknown() + } + + val workingDir = incrementalCompilationOptions.workingDir + val modulesApiHistory = ModulesApiHistoryJs(incrementalCompilationOptions.modulesInfo) + + val compiler = IncrementalJsCompilerRunner( + workingDir = workingDir, + reporter = reporter, + buildHistoryFile = incrementalCompilationOptions.multiModuleICSettings.buildHistoryFile, + modulesApiHistory = modulesApiHistory + ) + return try { + compiler.compile(allKotlinFiles, args, compilerMessageCollector, changedFiles) + } finally { + (reporter as RemoteICReporter).flush() + } + } + + protected fun execIncrementalCompiler( + k2jvmArgs: K2JVMCompilerArguments, + incrementalCompilationOptions: IncrementalCompilationOptions, + compilerMessageCollector: MessageCollector, + reporter: ICReporter + ): ExitCode { + val moduleFile = k2jvmArgs.buildFile?.let(::File) + assert(moduleFile?.exists() ?: false) { "Module does not exist ${k2jvmArgs.buildFile}" } + + // todo: pass javaSourceRoots and allKotlinFiles using IncrementalCompilationOptions + val parsedModule = run { + val bytesOut = ByteArrayOutputStream() + val printStream = PrintStream(bytesOut) + val mc = PrintingMessageCollector(printStream, MessageRenderer.PLAIN_FULL_PATHS, false) + val parsedModule = ModuleXmlParser.parseModuleScript(k2jvmArgs.buildFile!!, mc) + parsedModule + val allKotlinExtensions = (DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS + + (incrementalCompilationOptions.kotlinScriptExtensions ?: emptyArray())).distinct() + val dotExtensions = allKotlinExtensions.map { ".$it" } + val freeArgs = arrayListOf() + val allKotlinFiles = arrayListOf() + for (arg in k2jvmArgs.freeArgs) { + val file = File(arg) + if (file.isFile && dotExtensions.any { ext -> file.path.endsWith(ext, ignoreCase = true) }) { + allKotlinFiles.add(file) + } else { + freeArgs.add(arg) + } + } + k2jvmArgs.freeArgs = freeArgs + + val changedFiles = if (incrementalCompilationOptions.areFileChangesKnown) { + ChangedFiles.Known(incrementalCompilationOptions.modifiedFiles!!, incrementalCompilationOptions.deletedFiles!!) + } else { + ChangedFiles.Unknown() + } + + val workingDir = incrementalCompilationOptions.workingDir + + val modulesApiHistory = incrementalCompilationOptions.run { + reporter.report { "Use module detection: ${multiModuleICSettings.useModuleDetection}" } + + if (!multiModuleICSettings.useModuleDetection) { + ModulesApiHistoryJvm(modulesInfo) + } else { + ModulesApiHistoryAndroid(modulesInfo) + } + } + + val outputFiles = incrementalCompilationOptions.outputFiles.toMutableList() + incrementalCompilationOptions.classpathFqNamesHistory?.let { outputFiles.add(it) } + + val compiler = IncrementalJvmCompilerRunner( + workingDir, + reporter, + buildHistoryFile = incrementalCompilationOptions.multiModuleICSettings.buildHistoryFile, + outputFiles = outputFiles, + usePreciseJavaTracking = incrementalCompilationOptions.usePreciseJavaTracking, + modulesApiHistory = modulesApiHistory, + kotlinSourceFilesExtensions = allKotlinExtensions, + classpathFqNamesHistory = incrementalCompilationOptions.classpathFqNamesHistory + ) + return try { + compiler.compile(allKotlinFiles, k2jvmArgs, compilerMessageCollector, changedFiles) + } finally { + (reporter as RemoteICReporter).flush() + } + } + + @JvmName("withValidReplImpl1") + protected inline fun withValidReplImpl( + sessionId: Int, + body: KotlinJvmReplServiceT.() -> CompileService.CallResult + ): CompileService.CallResult = + withValidClientOrSessionProxy(sessionId) { session -> + (session?.data as? KotlinJvmReplServiceT?)?.body() ?: CompileService.CallResult.Error("Not a REPL session $sessionId") + } + +} + +class CompileServiceImpl( + val registry: Registry, + val compiler: CompilerSelector, + compilerId: CompilerId, + daemonOptions: DaemonOptions, + val daemonJVMOptions: DaemonJVMOptions, + port: Int, + timer: Timer, + val onShutdown: () -> Unit +) : CompileService, CompileServiceImplBase(daemonOptions, compilerId, port, timer) { + + private inline fun withValidRepl( + sessionId: Int, + body: KotlinJvmReplService.() -> CompileService.CallResult + ) = withValidReplImpl(sessionId, body) + + override val lastUsedSeconds: Long get() = + if (rwlock.isWriteLocked || rwlock.readLockCount - rwlock.readHoldCount > 0) nowSeconds() else _lastUsedSeconds + + private val rwlock = ReentrantReadWriteLock() + // RMI-exposed API override fun getDaemonInfo(): CompileService.CallResult = ifAlive(minAliveness = Aliveness.Dying) { @@ -308,13 +707,7 @@ class CompileServiceImpl( rwlock.write { clearJarCache() } - if (state.sessions.isEmpty()) { - // TODO: and some goes here - } - timer.schedule(0) { - periodicAndAfterSessionCheck() - } - CompileService.CallResult.Ok() + postReleaseCompileSession() } override fun checkCompilerId(expectedCompilerId: CompilerId): Boolean = @@ -406,181 +799,18 @@ class CompileServiceImpl( compilationOptions: CompilationOptions, servicesFacade: CompilerServicesFacadeBase, compilationResults: CompilationResults? - ): CompileService.CallResult = ifAlive { - withValidClientOrSessionProxy(sessionId) { - val messageCollector = CompileServicesFacadeMessageCollector(servicesFacade, compilationOptions) - val daemonReporter = DaemonMessageReporter(servicesFacade, compilationOptions) - val targetPlatform = compilationOptions.targetPlatform - log.info("Starting compilation with args: " + compilerArguments.joinToString(" ")) - - @Suppress("UNCHECKED_CAST") - val compiler = when (targetPlatform) { - CompileService.TargetPlatform.JVM -> K2JVMCompiler() - CompileService.TargetPlatform.JS -> K2JSCompiler() - CompileService.TargetPlatform.METADATA -> K2MetadataCompiler() - } as CLICompiler - - val k2PlatformArgs = compiler.createArguments() - parseCommandLineArguments(compilerArguments.asList(), k2PlatformArgs) - val argumentParseError = validateArguments(k2PlatformArgs.errors) - if (argumentParseError != null) { - messageCollector.report(CompilerMessageSeverity.ERROR, argumentParseError) - CompileService.CallResult.Good(ExitCode.COMPILATION_ERROR.code) - } else when (compilationOptions.compilerMode) { - CompilerMode.JPS_COMPILER -> { - val jpsServicesFacade = servicesFacade as JpsCompilerServicesFacade - - withIC(enabled = servicesFacade.hasIncrementalCaches()) { - doCompile(sessionId, daemonReporter, tracer = null) { eventManger, profiler -> - val services = createCompileServices(jpsServicesFacade, eventManger, profiler) - compiler.exec(messageCollector, services, k2PlatformArgs) - } - } - } - CompilerMode.NON_INCREMENTAL_COMPILER -> { - doCompile(sessionId, daemonReporter, tracer = null) { _, _ -> - compiler.exec(messageCollector, Services.EMPTY, k2PlatformArgs) - } - } - CompilerMode.INCREMENTAL_COMPILER -> { - val gradleIncrementalArgs = compilationOptions as IncrementalCompilationOptions - val gradleIncrementalServicesFacade = servicesFacade as IncrementalCompilerServicesFacade - - when (targetPlatform) { - CompileService.TargetPlatform.JVM -> { - val k2jvmArgs = k2PlatformArgs as K2JVMCompilerArguments - - withIC { - doCompile(sessionId, daemonReporter, tracer = null) { _, _ -> - execIncrementalCompiler( - k2jvmArgs, gradleIncrementalArgs, gradleIncrementalServicesFacade, compilationResults!!, - messageCollector - ) - } - } - } - CompileService.TargetPlatform.JS -> { - val k2jsArgs = k2PlatformArgs as K2JSCompilerArguments - - withJsIC { - doCompile(sessionId, daemonReporter, tracer = null) { _, _ -> - execJsIncrementalCompiler( - k2jsArgs, - gradleIncrementalArgs, - gradleIncrementalServicesFacade, - compilationResults!!, - messageCollector - ) - } - } - } - else -> throw IllegalStateException("Incremental compilation is not supported for target platform: $targetPlatform") - - } - } - else -> throw IllegalStateException("Unknown compilation mode ${compilationOptions.compilerMode}") - } - } - } - - private fun execJsIncrementalCompiler( - args: K2JSCompilerArguments, - incrementalCompilationOptions: IncrementalCompilationOptions, - servicesFacade: IncrementalCompilerServicesFacade, - compilationResults: CompilationResults, - compilerMessageCollector: MessageCollector - ): ExitCode { - val allKotlinFiles = arrayListOf() - val freeArgsWithoutKotlinFiles = arrayListOf() - args.freeArgs.forEach { - if (it.endsWith(".kt") && File(it).exists()) { - allKotlinFiles.add(File(it)) - } else { - freeArgsWithoutKotlinFiles.add(it) - } - } - args.freeArgs = freeArgsWithoutKotlinFiles - - val reporter = getICReporter(servicesFacade, compilationResults, incrementalCompilationOptions) - - val changedFiles = if (incrementalCompilationOptions.areFileChangesKnown) { - ChangedFiles.Known(incrementalCompilationOptions.modifiedFiles!!, incrementalCompilationOptions.deletedFiles!!) - } else { - ChangedFiles.Unknown() - } - - val workingDir = incrementalCompilationOptions.workingDir - val modulesApiHistory = ModulesApiHistoryJs(incrementalCompilationOptions.modulesInfo) - - val compiler = IncrementalJsCompilerRunner( - workingDir = workingDir, - reporter = reporter, - buildHistoryFile = incrementalCompilationOptions.multiModuleICSettings.buildHistoryFile, - modulesApiHistory = modulesApiHistory - ) - return try { - compiler.compile(allKotlinFiles, args, compilerMessageCollector, changedFiles) - } finally { - reporter.flush() - } - } - - private fun execIncrementalCompiler( - k2jvmArgs: K2JVMCompilerArguments, - incrementalCompilationOptions: IncrementalCompilationOptions, - servicesFacade: IncrementalCompilerServicesFacade, - compilationResults: CompilationResults, - compilerMessageCollector: MessageCollector - ): ExitCode { - val allKotlinExtensions = (DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS + - (incrementalCompilationOptions.kotlinScriptExtensions ?: emptyArray())).distinct() - val dotExtensions = allKotlinExtensions.map { ".$it" } - val freeArgs = arrayListOf() - val allKotlinFiles = arrayListOf() - for (arg in k2jvmArgs.freeArgs) { - val file = File(arg) - if (file.isFile && dotExtensions.any { ext -> file.path.endsWith(ext, ignoreCase = true) }) { - allKotlinFiles.add(file) - } else { - freeArgs.add(arg) - } - } - k2jvmArgs.freeArgs = freeArgs - - val changedFiles = if (incrementalCompilationOptions.areFileChangesKnown) { - ChangedFiles.Known(incrementalCompilationOptions.modifiedFiles!!, incrementalCompilationOptions.deletedFiles!!) - } else { - ChangedFiles.Unknown() - } - - val workingDir = incrementalCompilationOptions.workingDir - - val reporter = getICReporter(servicesFacade, compilationResults, incrementalCompilationOptions) - val modulesApiHistory = incrementalCompilationOptions.run { - reporter.report { "Use module detection: ${multiModuleICSettings.useModuleDetection}" } - - if (!multiModuleICSettings.useModuleDetection) { - ModulesApiHistoryJvm(modulesInfo) - } else { - ModulesApiHistoryAndroid(modulesInfo) - } - } - - val compiler = IncrementalJvmCompilerRunner( - workingDir, - reporter, - buildHistoryFile = incrementalCompilationOptions.multiModuleICSettings.buildHistoryFile, - outputFiles = incrementalCompilationOptions.outputFiles, - usePreciseJavaTracking = incrementalCompilationOptions.usePreciseJavaTracking, - modulesApiHistory = modulesApiHistory, - kotlinSourceFilesExtensions = allKotlinExtensions - ) - return try { - compiler.compile(allKotlinFiles, k2jvmArgs, compilerMessageCollector, changedFiles) - } finally { - reporter.flush() - } - } + ) = compileImpl( + sessionId, + compilerArguments, + compilationOptions, + servicesFacade, + compilationResults, + hasIncrementalCaches = JpsCompilerServicesFacade::hasIncrementalCaches, + createMessageCollector = ::CompileServicesFacadeMessageCollector, + createReporter = ::DaemonMessageReporter, + createServices = this::createCompileServices, + getICReporter = { a, b, c -> getICReporter(a, b!!, c)} + ) override fun leaseReplSession( aliveFlagPath: String?, @@ -608,7 +838,7 @@ class CompileServiceImpl( ) val messageCollector = KeepFirstErrorMessageCollector(compilerMessagesStream) val repl = KotlinJvmReplService( - disposable, port, compilerId, templateClasspath, templateClassName, + disposable, port, templateClasspath, templateClassName, messageCollector, operationsTracer ) val sessionId = state.sessions.leaseSession(ClientOrSessionProxy(aliveFlagPath, repl, disposable)) @@ -627,6 +857,37 @@ class CompileServiceImpl( } } + private fun createCompileServices( + facade: CompilerCallbackServicesFacade, + eventManager: EventManager, + rpcProfiler: Profiler + ): Services { + val builder = Services.Builder() + if (facade.hasIncrementalCaches()) { + builder.register( + IncrementalCompilationComponents::class.java, + RemoteIncrementalCompilationComponentsClient(facade, eventManager, rpcProfiler) + ) + } + if (facade.hasLookupTracker()) { + builder.register(LookupTracker::class.java, RemoteLookupTrackerClient(facade, eventManager, rpcProfiler)) + } + if (facade.hasCompilationCanceledStatus()) { + builder.register(CompilationCanceledStatus::class.java, RemoteCompilationCanceledStatusClient(facade, rpcProfiler)) + } + if (facade.hasExpectActualTracker()) { + builder.register(ExpectActualTracker::class.java, RemoteExpectActualTracker(facade, rpcProfiler)) + } + if (facade.hasIncrementalResultsConsumer()) { + builder.register(IncrementalResultsConsumer::class.java, RemoteIncrementalResultsConsumer(facade, eventManager, rpcProfiler)) + } + if (facade.hasIncrementalDataProvider()) { + builder.register(IncrementalDataProvider::class.java, RemoteIncrementalDataProvider(facade, rpcProfiler)) + } + + return builder.build() + } + override fun remoteReplLineCompile( sessionId: Int, codeLine: ReplCodeLine, @@ -661,7 +922,7 @@ class CompileServiceImpl( val disposable = Disposer.newDisposable() val messageCollector = CompileServicesFacadeMessageCollector(servicesFacade, compilationOptions) val repl = KotlinJvmReplService( - disposable, port, compilerId, templateClasspath, templateClassName, + disposable, port, templateClasspath, templateClassName, messageCollector, null ) val sessionId = state.sessions.leaseSession(ClientOrSessionProxy(aliveFlagPath, repl, disposable)) @@ -695,61 +956,7 @@ class CompileServiceImpl( } } - // ----------------------------------------------------------------------- - // internal implementation stuff - - // TODO: consider matching compilerId coming from outside with actual one - // private val selfCompilerId by lazy { - // CompilerId( - // compilerClasspath = System.getProperty("java.class.path") - // ?.split(File.pathSeparator) - // ?.map { File(it) } - // ?.filter { it.exists() } - // ?.map { it.absolutePath } - // ?: listOf(), - // compilerVersion = loadKotlinVersionFromResource() - // ) - // } - - init { - // assuming logically synchronized - try { - // cleanup for the case of incorrect restart and many other situations - UnicastRemoteObject.unexportObject(this, false) - } catch (e: NoSuchObjectException) { - // ignoring if object already exported - } - - val stub = UnicastRemoteObject.exportObject( - this, - port, - LoopbackNetworkInterface.clientLoopbackSocketFactory, - LoopbackNetworkInterface.serverLoopbackSocketFactory - ) as CompileService - registry.rebind(COMPILER_SERVICE_RMI_NAME, stub) - - timer.schedule(10) { - exceptionLoggingTimerThread { initiateElections() } - } - timer.schedule(delay = DAEMON_PERIODIC_CHECK_INTERVAL_MS, period = DAEMON_PERIODIC_CHECK_INTERVAL_MS) { - exceptionLoggingTimerThread { periodicAndAfterSessionCheck() } - } - timer.schedule(delay = DAEMON_PERIODIC_SELDOM_CHECK_INTERVAL_MS + 100, period = DAEMON_PERIODIC_SELDOM_CHECK_INTERVAL_MS) { - exceptionLoggingTimerThread { periodicSeldomCheck() } - } - } - - private inline fun exceptionLoggingTimerThread(body: () -> Unit) { - try { - body() - } catch (e: Throwable) { - System.err.println("Exception in timer thread: " + e.message) - e.printStackTrace(System.err) - log.log(Level.SEVERE, "Exception in timer thread", e) - } - } - - private fun periodicAndAfterSessionCheck() { + override fun periodicAndAfterSessionCheck() { if (state.delayedShutdownQueued.get()) return @@ -794,7 +1001,7 @@ class CompileServiceImpl( } } - private fun periodicSeldomCheck() { + override fun periodicSeldomCheck() { ifAliveUnit(minAliveness = Aliveness.Alive) { // compiler changed (seldom check) - shutdown @@ -807,7 +1014,7 @@ class CompileServiceImpl( // TODO: handover should include mechanism for client to switch to a new daemon then previous "handed over responsibilities" and shot down - private fun initiateElections() { + override fun initiateElections() { ifAliveUnit { @@ -989,102 +1196,22 @@ class CompileServiceImpl( } } - private fun doCompile( - sessionId: Int, - daemonMessageReporter: DaemonMessageReporter, - tracer: RemoteOperationsTracer?, - body: (EventManager, Profiler) -> ExitCode - ): CompileService.CallResult = - ifAlive { - withValidClientOrSessionProxy(sessionId) { - tracer?.before("compile") - val rpcProfiler = if (daemonOptions.reportPerf) WallAndThreadTotalProfiler() else DummyProfiler() - val eventManger = EventManagerImpl() - try { - val exitCode = checkedCompile(daemonMessageReporter, rpcProfiler) { - body(eventManger, rpcProfiler).code - } - CompileService.CallResult.Good(exitCode) - } finally { - eventManger.fireCompilationFinished() - tracer?.after("compile") - } - } - } - - private fun createCompileServices(facade: CompilerCallbackServicesFacade, eventManager: EventManager, rpcProfiler: Profiler): Services { - val builder = Services.Builder() - if (facade.hasIncrementalCaches()) { - builder.register( - IncrementalCompilationComponents::class.java, - RemoteIncrementalCompilationComponentsClient(facade, eventManager, rpcProfiler) - ) - } - if (facade.hasLookupTracker()) { - builder.register(LookupTracker::class.java, RemoteLookupTrackerClient(facade, eventManager, rpcProfiler)) - } - if (facade.hasCompilationCanceledStatus()) { - builder.register(CompilationCanceledStatus::class.java, RemoteCompilationCanceledStatusClient(facade, rpcProfiler)) - } - if (facade.hasExpectActualTracker()) { - builder.register(ExpectActualTracker::class.java, RemoteExpectActualTracker(facade, rpcProfiler)) - } - if (facade.hasIncrementalResultsConsumer()) { - builder.register(IncrementalResultsConsumer::class.java, RemoteIncrementalResultsConsumer(facade, eventManager, rpcProfiler)) - } - if (facade.hasIncrementalDataProvider()) { - builder.register(IncrementalDataProvider::class.java, RemoteIncrementalDataProvider(facade, rpcProfiler)) - } - - return builder.build() - } - - - private fun checkedCompile(daemonMessageReporter: DaemonMessageReporter, rpcProfiler: Profiler, body: () -> R): R { + init { + // assuming logicaly synchronized try { - val profiler = if (daemonOptions.reportPerf) WallAndThreadAndMemoryTotalProfiler(withGC = false) else DummyProfiler() - - val res = profiler.withMeasure(null, body) - - val endMem = if (daemonOptions.reportPerf) usedMemory(withGC = false) else 0L - - log.info("Done with result " + res.toString()) - - if (daemonOptions.reportPerf) { - fun Long.ms() = TimeUnit.NANOSECONDS.toMillis(this) - fun Long.kb() = this / 1024 - val pc = profiler.getTotalCounters() - val rpc = rpcProfiler.getTotalCounters() - - "PERF: Compile on daemon: ${pc.time.ms()} ms; thread: user ${pc.threadUserTime.ms()} ms, sys ${(pc.threadTime - pc.threadUserTime).ms()} ms; rpc: ${rpc.count} calls, ${rpc.time.ms()} ms, thread ${rpc.threadTime.ms()} ms; memory: ${endMem.kb()} kb (${"%+d".format( - pc.memory.kb() - )} kb)".let { - daemonMessageReporter.report(ReportSeverity.INFO, it) - log.info(it) - } - - // this will only be reported if if appropriate (e.g. ByClass) profiler is used - for ((obj, counters) in rpcProfiler.getCounters()) { - "PERF: rpc by $obj: ${counters.count} calls, ${counters.time.ms()} ms, thread ${counters.threadTime.ms()} ms".let { - daemonMessageReporter.report(ReportSeverity.INFO, it) - log.info(it) - } - } - } - return res - } - // TODO: consider possibilities to handle OutOfMemory - catch (e: Throwable) { - log.log( - Level.SEVERE, - "Exception: $e\n ${e.stackTrace.joinToString("\n ")}${ - if (e.cause != null && e.cause != e) { - "\nCaused by: ${e.cause}\n ${e.cause!!.stackTrace.joinToString("\n ")}" - } else "" - }" - ) - throw e + // cleanup for the case of incorrect restart and many other situations + UnicastRemoteObject.unexportObject(this, false) + } catch (e: NoSuchObjectException) { + // ignoring if object already exported } + + val stub = UnicastRemoteObject.exportObject( + this, + port, + LoopbackNetworkInterface.clientLoopbackSocketFactory, + LoopbackNetworkInterface.serverLoopbackSocketFactory + ) as CompileService + registry.rebind(COMPILER_SERVICE_RMI_NAME, stub) } override fun clearJarCache() { @@ -1119,56 +1246,4 @@ class CompileServiceImpl( CompileService.CallResult.Ok() } } - - private inline fun ifAliveChecksImpl( - minAliveness: Aliveness = Aliveness.LastSession, - body: () -> CompileService.CallResult - ): CompileService.CallResult { - val curState = state.alive.get() - return when { - curState < minAliveness.ordinal -> { - log.info("Cannot perform operation, requested state: ${minAliveness.name} > actual: ${curState.toAlivenessName()}") - CompileService.CallResult.Dying() - } - else -> { - try { - body() - } catch (e: Throwable) { - log.log(Level.SEVERE, "Exception", e) - CompileService.CallResult.Error(e.message ?: "unknown") - } - } - } - } - - private inline fun withValidClientOrSessionProxy( - sessionId: Int, - body: (ClientOrSessionProxy?) -> CompileService.CallResult - ): CompileService.CallResult { - val session: ClientOrSessionProxy? = - if (sessionId == CompileService.NO_SESSION) null - else state.sessions[sessionId] ?: return CompileService.CallResult.Error("Unknown or invalid session $sessionId") - try { - compilationsCounter.incrementAndGet() - return body(session) - } finally { - _lastUsedSeconds = nowSeconds() - } - } - - private inline fun withValidRepl(sessionId: Int, body: KotlinJvmReplService.() -> R): CompileService.CallResult = - withValidClientOrSessionProxy(sessionId) { session -> - (session?.data as? KotlinJvmReplService?)?.let { - CompileService.CallResult.Good(it.body()) - } ?: CompileService.CallResult.Error("Not a REPL session $sessionId") - } - - @JvmName("withValidRepl1") - private inline fun withValidRepl( - sessionId: Int, - body: KotlinJvmReplService.() -> CompileService.CallResult - ): CompileService.CallResult = - withValidClientOrSessionProxy(sessionId) { session -> - (session?.data as? KotlinJvmReplService?)?.body() ?: CompileService.CallResult.Error("Not a REPL session $sessionId") - } } diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinCompileDaemon.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinCompileDaemon.kt index caa62fc2122..de4295063f8 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinCompileDaemon.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinCompileDaemon.kt @@ -161,6 +161,7 @@ object KotlinCompileDaemon { timer.cancel() } }) + compilerService.startDaemonLife() println(COMPILE_DAEMON_IS_READY_MESSAGE) log.info("daemon is listening on port: $port") diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinRemoteReplService.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinRemoteReplService.kt index 5177786847b..7859c6fbdee 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinRemoteReplService.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinRemoteReplService.kt @@ -40,15 +40,11 @@ import java.util.logging.Logger import kotlin.concurrent.read import kotlin.concurrent.write -open class KotlinJvmReplService( - disposable: Disposable, - val portForServers: Int, - val compilerId: CompilerId, - templateClasspath: List, - templateClassName: String, - protected val messageCollector: MessageCollector, - @Deprecated("drop it") - protected val operationsTracer: RemoteOperationsTracer? +abstract class KotlinJvmReplServiceBase( + disposable: Disposable, + val compilerId: CompilerId,templateClasspath: List, + templateClassName: String, + protected val messageCollector: MessageCollector ) : ReplCompileAction, ReplCheckAction, CreateReplStageStateAction { private val log by lazy { Logger.getLogger("replService") } @@ -64,7 +60,28 @@ open class KotlinJvmReplService( configureScripting(compilerId) } - private val replCompiler: ReplCompiler? by lazy { + protected fun makeScriptDefinition(templateClasspath: List, templateClassName: String): KotlinScriptDefinition? { + val classloader = URLClassLoader(templateClasspath.map { it.toURI().toURL() }.toTypedArray(), this::class.java.classLoader) + + try { + val cls = classloader.loadClass(templateClassName) + val def = KotlinScriptDefinitionFromAnnotatedTemplate(cls.kotlin, emptyMap()) + messageCollector.report(INFO, "New script definition $templateClassName: files pattern = \"${def.scriptFilePattern}\", " + + "resolver = ${def.dependencyResolver.javaClass.name}") + return def + } + catch (ex: ClassNotFoundException) { + messageCollector.report(ERROR, "Cannot find script definition template class $templateClassName") + } + catch (ex: Exception) { + messageCollector.report(ERROR, "Error processing script definition template $templateClassName: ${ex.message}") + } + return null + } + + private val scriptDef = makeScriptDefinition(templateClasspath, templateClassName) + + protected val replCompiler: ReplCompiler? by lazy { try { val projectEnvironment = KotlinCoreEnvironment.ProjectEnvironment( @@ -94,35 +111,56 @@ open class KotlinJvmReplService( } protected val statesLock = ReentrantReadWriteLock() - // TODO: consider using values here for session cleanup - protected val states = WeakHashMap() // used as (missing) WeakHashSet protected val stateIdCounter = AtomicInteger() - @Deprecated("remove after removal state-less check/compile/eval methods") - protected val defaultStateFacade: RemoteReplStateFacadeServer by lazy { createRemoteState() } override fun createState(lock: ReentrantReadWriteLock): IReplStageState<*> = - replCompiler?.createState(lock) ?: throw IllegalStateException("repl compiler is not initialized properly") + replCompiler?.createState(lock) ?: throw IllegalStateException("repl compiler is not initialized properly") + + protected open fun before(s: String) {} + protected open fun after(s: String) {} override fun check(state: IReplStageState<*>, codeLine: ReplCodeLine): ReplCheckResult { - operationsTracer?.before("check") + before("check") try { return replCompiler?.check(state, codeLine) ?: ReplCheckResult.Error("Initialization error") - } - finally { - operationsTracer?.after("check") + } finally { + after("check") } } override fun compile(state: IReplStageState<*>, codeLine: ReplCodeLine): ReplCompileResult { - operationsTracer?.before("compile") + before("compile") try { return replCompiler?.compile(state, codeLine) ?: ReplCompileResult.Error("Initialization error") - } - finally { - operationsTracer?.after("compile") + } finally { + after("compile") } } +} + +open class KotlinJvmReplService( + disposable: Disposable, + val portForServers: Int, + templateClasspath: List, + templateClassName: String, + messageCollector: MessageCollector, + @Deprecated("drop it") + protected val operationsTracer: RemoteOperationsTracer? +) : KotlinJvmReplServiceBase(disposable, templateClasspath, templateClassName, messageCollector) { + + override fun before(s: String) { + operationsTracer?.before(s) + } + + override fun after(s: String) { + operationsTracer?.after(s) + } + + protected val states = WeakHashMap() // used as (missing) WeakHashSet + @Deprecated("remove after removal state-less check/compile/eval methods") + protected val defaultStateFacade: RemoteReplStateFacadeServer by lazy { createRemoteState() } + @Deprecated("Use check(state, line) instead") fun check(codeLine: ReplCodeLine): ReplCheckResult = check(defaultStateFacade.state, codeLine) @@ -130,17 +168,17 @@ open class KotlinJvmReplService( fun compile(codeLine: ReplCodeLine, verifyHistory: List?): ReplCompileResult = compile(defaultStateFacade.state, codeLine) fun createRemoteState(port: Int = portForServers): RemoteReplStateFacadeServer = statesLock.write { - val id = getValidId(stateIdCounter) { id -> states.none { it.key.getId() == id} } + val id = getValidId(stateIdCounter) { id -> states.none { it.key.getId() == id } } val stateFacade = RemoteReplStateFacadeServer(id, createState(), port) states.put(stateFacade, true) stateFacade } - fun withValidReplState(stateId: Int, body: (IReplStageState<*>) -> R): CompileService.CallResult = statesLock.read { + fun withValidReplState(stateId: Int, body: (IReplStageState<*>) -> R): CompileService.CallResult = statesLock.read { states.keys.firstOrNull { it.getId() == stateId }?.let { CompileService.CallResult.Good(body(it.state)) } - ?: CompileService.CallResult.Error("No REPL state with id $stateId found") + ?: CompileService.CallResult.Error("No REPL state with id $stateId found") } } @@ -165,9 +203,9 @@ internal class KeepFirstErrorMessageCollector(compilerMessagesStream: PrintStrea } } -internal val internalRng = Random() +val internalRng = Random() -inline internal fun getValidId(counter: AtomicInteger, check: (Int) -> Boolean): Int { +inline fun getValidId(counter: AtomicInteger, check: (Int) -> Boolean): Int { // fighting hypothetical integer wrapping var newId = counter.incrementAndGet() var attemptsLeft = 100 @@ -181,7 +219,7 @@ inline internal fun getValidId(counter: AtomicInteger, check: (Int) -> Boolean): return newId } -private fun CompilerConfiguration.configureScripting(compilerId: CompilerId) { +fun CompilerConfiguration.configureScripting(compilerId: CompilerId) { val error = try { val componentRegistrars = (this::class.java.classLoader as? URLClassLoader)?.let { diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteCompilationCanceledStatusClient.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteCompilationCanceledStatusClient.kt index 7f4a154d92c..d6c388c396d 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteCompilationCanceledStatusClient.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteCompilationCanceledStatusClient.kt @@ -16,11 +16,8 @@ package org.jetbrains.kotlin.daemon +import org.jetbrains.kotlin.daemon.common.* import org.jetbrains.kotlin.progress.CompilationCanceledStatus -import org.jetbrains.kotlin.daemon.common.CompilerCallbackServicesFacade -import org.jetbrains.kotlin.daemon.common.DummyProfiler -import org.jetbrains.kotlin.daemon.common.Profiler -import org.jetbrains.kotlin.daemon.common.RmiFriendlyCompilationCanceledException import org.jetbrains.kotlin.progress.CompilationCanceledException import java.util.concurrent.TimeUnit import java.util.logging.Logger diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteExpectActualTracker.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteExpectActualTracker.kt index ad75a680d8b..5e145bc1d59 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteExpectActualTracker.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteExpectActualTracker.kt @@ -8,6 +8,7 @@ package org.jetbrains.kotlin.daemon import org.jetbrains.kotlin.daemon.common.CompilerCallbackServicesFacade import org.jetbrains.kotlin.daemon.common.DummyProfiler import org.jetbrains.kotlin.daemon.common.Profiler +import org.jetbrains.kotlin.daemon.common.withMeasure import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import java.io.File diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalCacheClient.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalCacheClient.kt index f08a63bcf54..fc22b09022a 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalCacheClient.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalCacheClient.kt @@ -19,6 +19,7 @@ package org.jetbrains.kotlin.daemon import org.jetbrains.kotlin.daemon.common.CompilerCallbackServicesFacade import org.jetbrains.kotlin.daemon.common.DummyProfiler import org.jetbrains.kotlin.daemon.common.Profiler +import org.jetbrains.kotlin.daemon.common.withMeasure import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache import org.jetbrains.kotlin.load.kotlin.incremental.components.JvmPackagePartProto import org.jetbrains.kotlin.modules.TargetId diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalDataProvider.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalDataProvider.kt index 627c5ce93bb..35e1276b9c5 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalDataProvider.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalDataProvider.kt @@ -7,6 +7,7 @@ package org.jetbrains.kotlin.daemon import org.jetbrains.kotlin.daemon.common.CompilerCallbackServicesFacade import org.jetbrains.kotlin.daemon.common.Profiler +import org.jetbrains.kotlin.daemon.common.withMeasure import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider import org.jetbrains.kotlin.incremental.js.TranslationResultValue import java.io.File diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalResultsConsumer.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalResultsConsumer.kt index f7c1b75c17b..c7cc6af584a 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalResultsConsumer.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalResultsConsumer.kt @@ -7,6 +7,7 @@ package org.jetbrains.kotlin.daemon import org.jetbrains.kotlin.daemon.common.CompilerCallbackServicesFacade import org.jetbrains.kotlin.daemon.common.Profiler +import org.jetbrains.kotlin.daemon.common.withMeasure import org.jetbrains.kotlin.incremental.js.FunctionWithSourceInfo import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer import org.jetbrains.kotlin.incremental.js.JsInlineFunctionHash @@ -15,7 +16,8 @@ import java.io.File class RemoteIncrementalResultsConsumer(val facade: CompilerCallbackServicesFacade, eventManager: EventManager, val rpcProfiler: Profiler) : IncrementalResultsConsumer { init { - eventManager.onCompilationFinished(this::flush) + eventManager. + onCompilationFinished(this::flush) } override fun processHeader(headerMetadata: ByteArray) { diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteInputStreamClient.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteInputStreamClient.kt index 55559040fb2..a69cdb5d0fa 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteInputStreamClient.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteInputStreamClient.kt @@ -19,6 +19,7 @@ package org.jetbrains.kotlin.daemon import org.jetbrains.kotlin.daemon.common.DummyProfiler import org.jetbrains.kotlin.daemon.common.Profiler import org.jetbrains.kotlin.daemon.common.RemoteInputStream +import org.jetbrains.kotlin.daemon.common.withMeasure import java.io.InputStream class RemoteInputStreamClient(val remote: RemoteInputStream, val profiler: Profiler = DummyProfiler()): InputStream() { diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteLookupTrackerClient.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteLookupTrackerClient.kt index 88e9b839bc0..a01aa402b36 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteLookupTrackerClient.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteLookupTrackerClient.kt @@ -22,6 +22,7 @@ import gnu.trove.THashSet import org.jetbrains.kotlin.daemon.common.CompilerCallbackServicesFacade import org.jetbrains.kotlin.daemon.common.DummyProfiler import org.jetbrains.kotlin.daemon.common.Profiler +import org.jetbrains.kotlin.daemon.common.withMeasure import org.jetbrains.kotlin.incremental.components.LookupInfo import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.components.Position diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteOutputStreamClient.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteOutputStreamClient.kt index 7af18cd681b..707dc40dbab 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteOutputStreamClient.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteOutputStreamClient.kt @@ -19,6 +19,7 @@ package org.jetbrains.kotlin.daemon import org.jetbrains.kotlin.daemon.common.DummyProfiler import org.jetbrains.kotlin.daemon.common.Profiler import org.jetbrains.kotlin.daemon.common.RemoteOutputStream +import org.jetbrains.kotlin.daemon.common.withMeasure import java.io.OutputStream class RemoteOutputStreamClient(val remote: RemoteOutputStream, val profiler: Profiler = DummyProfiler()): OutputStream() { diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/CompileServiceServerSideImpl.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/CompileServiceServerSideImpl.kt new file mode 100644 index 00000000000..235c316fbb6 --- /dev/null +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/CompileServiceServerSideImpl.kt @@ -0,0 +1,749 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +@file:Suppress("UNCHECKED_CAST") + +package org.jetbrains.kotlin.daemon.experimental + +import com.intellij.openapi.util.Disposer +import com.intellij.openapi.vfs.impl.ZipHandler +import com.intellij.openapi.vfs.impl.jar.CoreJarFileSystem +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.actor +import kotlinx.coroutines.channels.consumeEach +import org.jetbrains.kotlin.cli.common.CLICompiler +import org.jetbrains.kotlin.cli.common.ExitCode +import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments +import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments +import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.cli.common.repl.ReplCheckResult +import org.jetbrains.kotlin.cli.common.repl.ReplCodeLine +import org.jetbrains.kotlin.cli.common.repl.ReplCompileResult +import org.jetbrains.kotlin.cli.js.K2JSCompiler +import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler +import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment +import org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler +import org.jetbrains.kotlin.config.Services +import org.jetbrains.kotlin.daemon.CompileServiceImplBase +import org.jetbrains.kotlin.daemon.CompilerSelector +import org.jetbrains.kotlin.daemon.common.* +import org.jetbrains.kotlin.daemon.experimental.CompileServiceTaskScheduler.* +import org.jetbrains.kotlin.daemon.nowSeconds +import org.jetbrains.kotlin.daemon.report.experimental.CompileServicesFacadeMessageCollector +import org.jetbrains.kotlin.daemon.report.experimental.DaemonMessageReporterAsync +import org.jetbrains.kotlin.daemon.report.experimental.getICReporterAsync +import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.incremental.parsing.classesFqNames +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents +import org.jetbrains.kotlin.progress.experimental.CompilationCanceledStatus +import java.io.File +import java.util.* +import java.util.concurrent.TimeUnit +import java.util.logging.Level +import java.util.logging.Logger +import kotlin.concurrent.schedule +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.* +import org.jetbrains.kotlin.daemon.common.experimental.* +import io.ktor.network.sockets.* +import org.jetbrains.kotlin.cli.common.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY +import org.jetbrains.kotlin.daemon.EventManager +import org.jetbrains.kotlin.daemon.report.DaemonMessageReporter + +// TODO: this classes should replace their non-experimental versions eventually. + +private class CompileServiceTaskScheduler(log: Logger) { + interface CompileServiceTask + interface CompileServiceTaskWithResult : CompileServiceTask + + open class ExclusiveTask(val completed: CompletableDeferred, val shutdownAction: suspend () -> Any) : CompileServiceTask + open class ShutdownTaskWithResult(val result: CompletableDeferred, shutdownAction: suspend () -> Any) : + ExclusiveTask(CompletableDeferred(), shutdownAction), CompileServiceTaskWithResult + + open class OrdinaryTask(val completed: CompletableDeferred, val action: suspend () -> Any) : CompileServiceTask + class OrdinaryTaskWithResult(val result: CompletableDeferred, action: suspend () -> Any) : + OrdinaryTask(CompletableDeferred(), action), + CompileServiceTaskWithResult + + class TaskFinished(val taskId: Int) : CompileServiceTask + class ExclusiveTaskFinished : CompileServiceTask + + private var shutdownActionInProgress = false + private var readLocksCount = 0 + + suspend fun scheduleTask(task: CompileServiceTask) = queriesActor.send(task) + + fun getReadLocksCnt() = readLocksCount + + fun isShutdownActionInProgress() = shutdownActionInProgress + + private val queriesActor = GlobalScope.actor(capacity = Channel.UNLIMITED) { + var currentTaskId = 0 + var shutdownTask: ExclusiveTask? = null + val activeTaskIds = arrayListOf() + val waitingTasks = arrayListOf() + fun shutdownIfInactive(reason: String) { + log.fine("invoked \'shutdownIfInactive\', reason : $reason") + if (activeTaskIds.isEmpty()) { + shutdownTask?.let { task -> + shutdownActionInProgress = true + GlobalScope.async { + val res = task.shutdownAction() + task.completed.complete(true) + if (task is ShutdownTaskWithResult) { + task.result.complete(res) + } + channel.send(ExclusiveTaskFinished()) + } + } + } + } + consumeEach { task -> + when (task) { + is ExclusiveTask -> { + if (shutdownTask == null) { + shutdownTask = task + shutdownIfInactive("ExclusiveTask") + } else { + waitingTasks.add(task) + } + } + is OrdinaryTask -> { + if (shutdownTask == null) { + val id = currentTaskId++ + activeTaskIds.add(id) + readLocksCount++ + GlobalScope.async { + val res = task.action() + if (task is OrdinaryTaskWithResult) { + task.result.complete(res) + } + task.completed.complete(true) + channel.send(TaskFinished(id)) + } + } else { + waitingTasks.add(task) + } + } + is TaskFinished -> { + activeTaskIds.remove(task.taskId) + readLocksCount-- + shutdownIfInactive("TaskFinished") + } + is ExclusiveTaskFinished -> { + shutdownTask = null + shutdownActionInProgress = false + waitingTasks.forEach { + channel.send(it) + } + waitingTasks.clear() + } + } + } + } +} + +class CompileServiceServerSideImpl( + override val serverSocketWithPort: ServerSocketWrapper, + val compiler: CompilerSelector, + compilerId: CompilerId, + daemonOptions: DaemonOptions, + val daemonJVMOptions: DaemonJVMOptions, + port: Int, + timer: Timer, + val onShutdown: () -> Unit +) : CompileServiceServerSide, CompileServiceImplBase(daemonOptions, compilerId, port, timer) { + + lateinit var rmiServer: CompileServiceRMIWrapper + + private inline fun withValidRepl( + sessionId: Int, + body: KotlinJvmReplServiceAsync.() -> CompileService.CallResult + ) = withValidReplImpl(sessionId, body) + + override val serverPort: Int + get() = serverSocketWithPort.port + + override val clients = hashMapOf() + + object KeepAliveServer : Server { + override val serverSocketWithPort = findCallbackServerSocket() + override val clients = hashMapOf() + + } + + override suspend fun checkClientCanReadFile(clientInputChannel: ByteReadChannelWrapper): Boolean = runWithTimeout { + getSignatureAndVerify(clientInputChannel, securityData.token, securityData.publicKey) + } ?: false + + override suspend fun serverHandshake(input: ByteReadChannelWrapper, output: ByteWriteChannelWrapper, log: Logger): Boolean { + return tryAcquireHandshakeMessage(input, log) && trySendHandshakeMessage(output, log) + } + + private lateinit var scheduler: CompileServiceTaskScheduler + + constructor( + serverSocket: ServerSocketWrapper, + compilerId: CompilerId, + daemonOptions: DaemonOptions, + daemonJVMOptions: DaemonJVMOptions, + port: Int, + timer: Timer, + onShutdown: () -> Unit + ) : this( + serverSocket, + object : CompilerSelector { + private val jvm by lazy { K2JVMCompiler() } + private val js by lazy { K2JSCompiler() } + private val metadata by lazy { K2MetadataCompiler() } + override fun get(targetPlatform: CompileService.TargetPlatform): CLICompiler<*> = when (targetPlatform) { + CompileService.TargetPlatform.JVM -> jvm + CompileService.TargetPlatform.JS -> js + CompileService.TargetPlatform.METADATA -> metadata + } + }, + compilerId, + daemonOptions, + daemonJVMOptions, + port, + timer, + onShutdown + ) + + override val lastUsedSeconds: Long + get() = (if (scheduler.getReadLocksCnt() > 1 || scheduler.isShutdownActionInProgress()) nowSeconds() else _lastUsedSeconds).also { + log.fine( + "lastUsedSeconds .. isReadLockedCNT : ${scheduler.getReadLocksCnt()} , " + + "shutdownActionInProgress : ${scheduler.isShutdownActionInProgress()}" + ) + } + + private var securityData: SecurityData = generateKeysAndToken().also { sdata -> + runFile.outputStream().use { + sendTokenKeyPair(it, sdata.token, sdata.privateKey) + } + } + + // RMI-exposed API + + override suspend fun getDaemonInfo(): CompileService.CallResult = + ifAlive(minAliveness = Aliveness.Dying) { + CompileService.CallResult.Good("Kotlin daemon on socketPort $port") + } + + override suspend fun getDaemonOptions(): CompileService.CallResult = ifAlive { + CompileService.CallResult.Good(daemonOptions) + } + + override suspend fun getDaemonJVMOptions(): CompileService.CallResult = ifAlive { + log.info("getDaemonJVMOptions: $daemonJVMOptions")// + daemonJVMOptions.mappers.flatMap { it.toArgs("-") }) + CompileService.CallResult.Good(daemonJVMOptions) + } + + override suspend fun registerClient(aliveFlagPath: String?): CompileService.CallResult { + log.fine("fun registerClient") + return ifAlive(minAliveness = Aliveness.Alive) { + registerClientImpl(aliveFlagPath) + } + } + + override suspend fun classesFqNamesByFiles( + sessionId: Int, sourceFiles: Set + ): CompileService.CallResult> = + ifAlive { + withValidClientOrSessionProxy(sessionId) { + CompileService.CallResult.Good(classesFqNames(sourceFiles)) + } + } + + private fun registerClientImpl(aliveFlagPath: String?): CompileService.CallResult { + state.addClient(aliveFlagPath) + log.info("Registered a client alive file: $aliveFlagPath") + return CompileService.CallResult.Ok() + } + + override suspend fun getClients(): CompileService.CallResult> = ifAlive { + getClientsImpl() + } + + private fun getClientsImpl() = CompileService.CallResult.Good(state.getClientsFlagPaths()) + + // TODO: consider tying a session to a client and use this info to cleanup + override suspend fun leaseCompileSession(aliveFlagPath: String?): CompileService.CallResult = + ifAlive(minAliveness = Aliveness.Alive) { + CompileService.CallResult.Good( + state.sessions.leaseSession(ClientOrSessionProxy(aliveFlagPath)).apply { + log.info("leased a new session $this, session alive file: $aliveFlagPath") + }) + } + + override suspend fun releaseCompileSession(sessionId: Int) = ifAlive(minAliveness = Aliveness.LastSession) { + state.sessions.remove(sessionId) + log.info("cleaning after session $sessionId") + val completed = CompletableDeferred() + scheduler.scheduleTask(ExclusiveTask(completed, { clearJarCache() })) + completed.await() + postReleaseCompileSession() + } + + + override suspend fun checkCompilerId(expectedCompilerId: CompilerId): Boolean = + (compilerId.compilerVersion.isEmpty() || compilerId.compilerVersion == expectedCompilerId.compilerVersion) && + (compilerId.compilerClasspath.all { expectedCompilerId.compilerClasspath.contains(it) }) && + !classpathWatcher.isChanged + + override suspend fun getUsedMemory(): CompileService.CallResult = + ifAlive { CompileService.CallResult.Good(usedMemory(withGC = true)) } + + override suspend fun shutdown(): CompileService.CallResult = + ifAliveExclusive(minAliveness = Aliveness.LastSession) { + shutdownWithDelay() + CompileService.CallResult.Ok() + } + + override suspend fun scheduleShutdown(graceful: Boolean): CompileService.CallResult = + ifAlive(minAliveness = Aliveness.LastSession) { + scheduleShutdownImpl(graceful) + } + + private fun scheduleShutdownImpl(graceful: Boolean): CompileService.CallResult { + val res = when { + graceful -> gracefulShutdown(true) + else -> { + shutdownWithDelay() + true + } + } + return CompileService.CallResult.Good(res) + } + + override suspend fun compile( + sessionId: Int, + compilerArguments: Array, + compilationOptions: CompilationOptions, + servicesFacade: CompilerServicesFacadeBaseAsync, + compilationResults: CompilationResultsAsync? + ) = compileImpl( + sessionId, + compilerArguments, + compilationOptions, + servicesFacade, + compilationResults, + hasIncrementalCaches = CompilerCallbackServicesFacadeClientSide::hasIncrementalCaches, + createMessageCollector = ::CompileServicesFacadeMessageCollector, + createReporter = ::DaemonMessageReporterAsync, + createServices = this::createCompileServices, + getICReporter = ::getICReporterAsync + ) + + override suspend fun leaseReplSession( + aliveFlagPath: String?, + compilerArguments: Array, + compilationOptions: CompilationOptions, + servicesFacade: CompilerServicesFacadeBaseAsync, + templateClasspath: List, + templateClassName: String + ): CompileService.CallResult = ifAlive(minAliveness = Aliveness.Alive) { + if (compilationOptions.targetPlatform != CompileService.TargetPlatform.JVM) + CompileService.CallResult.Error("Sorry, only JVM target platform is supported now") + else { + val disposable = Disposer.newDisposable() + val messageCollector = + CompileServicesFacadeMessageCollector(servicesFacade, compilationOptions) + val repl = KotlinJvmReplServiceAsync( + disposable, serverSocketWithPort, templateClasspath, templateClassName, + messageCollector + ) + val sessionId = state.sessions.leaseSession(ClientOrSessionProxy(aliveFlagPath, repl, disposable)) + + CompileService.CallResult.Good(sessionId) + } + } + + // TODO: add more checks (e.g. is it a repl session) + override suspend fun releaseReplSession(sessionId: Int): CompileService.CallResult = releaseCompileSession(sessionId) + + override suspend fun replCreateState(sessionId: Int): CompileService.CallResult = + ifAlive(minAliveness = Aliveness.Alive) { + withValidRepl(sessionId) { + CompileService.CallResult.Good( + createRemoteState(findReplServerSocket()).clientSide + ) + } + } + + override suspend fun replCheck( + sessionId: Int, + replStateId: Int, + codeLine: ReplCodeLine + ): CompileService.CallResult = ifAlive(minAliveness = Aliveness.Alive) { + withValidRepl(sessionId) { + withValidReplState(replStateId) { state -> + check(state, codeLine) + } + } + } + + override suspend fun replCompile( + sessionId: Int, + replStateId: Int, + codeLine: ReplCodeLine + ): CompileService.CallResult = + ifAlive(minAliveness = Aliveness.Alive) { + withValidRepl(sessionId) { + withValidReplState(replStateId) { state -> + compile(state, codeLine) + } + } + } + + init { + + scheduler = CompileServiceTaskScheduler(log) + + // assuming logically synchronized + System.setProperty(KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY, "true") + + // TODO UNCOMMENT THIS : this.toRMIServer(daemonOptions, compilerId) // also create RMI server in order to support old clients + rmiServer = this.toRMIServer(daemonOptions, compilerId) + + KeepAliveServer.runServer() + } + + override fun periodicAndAfterSessionCheck() { + if (state.delayedShutdownQueued.get()) return + + val anyDead = state.sessions.cleanDead() || state.cleanDeadClients() + + GlobalScope.async { + ifAliveUnit(minAliveness = Aliveness.LastSession) { + when { + // check if in graceful shutdown state and all sessions are closed + state.alive.get() == Aliveness.LastSession.ordinal && state.sessions.isEmpty() -> { + log.info("All sessions finished") + shutdownWithDelay() + return@ifAliveUnit + } + state.aliveClientsCount == 0 -> { + log.info("No more clients left") + shutdownWithDelay() + return@ifAliveUnit + } + // discovery file removed - shutdown + !runFile.exists() -> { + log.info("Run file removed") + shutdownWithDelay() + return@ifAliveUnit + } + } + } + + + ifAliveUnit(minAliveness = Aliveness.Alive) { + when { + daemonOptions.autoshutdownUnusedSeconds != COMPILE_DAEMON_TIMEOUT_INFINITE_S && compilationsCounter.get() == 0 && nowSeconds() - lastUsedSeconds > daemonOptions.autoshutdownUnusedSeconds -> { + log.info("Unused timeout exceeded ${daemonOptions.autoshutdownUnusedSeconds}s") + gracefulShutdown(false) + } + daemonOptions.autoshutdownIdleSeconds != COMPILE_DAEMON_TIMEOUT_INFINITE_S && nowSeconds() - lastUsedSeconds > daemonOptions.autoshutdownIdleSeconds -> { + log.info("Idle timeout exceeded ${daemonOptions.autoshutdownIdleSeconds}s") + gracefulShutdown(false) + } + anyDead -> { + clearJarCache() + } + } + } + } + } + + override fun periodicSeldomCheck() { + GlobalScope.async { + ifAliveUnit(minAliveness = Aliveness.Alive) { + // compiler changed (seldom check) - shutdown + if (classpathWatcher.isChanged) { + log.info("Compiler changed.") + gracefulShutdown(false) + } + } + } + } + + + // TODO: handover should include mechanism for client to switch to a new daemon then previous "handed over responsibilities" and shot down + override fun initiateElections() { + runBlocking(Dispatchers.Unconfined) { + ifAliveUnit { + log.info("initiate elections") + val aliveWithOpts = walkDaemonsAsync( + File(daemonOptions.runFilesPathOrDefault), + compilerId, + runFile, + filter = { _, p -> p != port }, + report = { _, msg -> log.info(msg) }, + useRMI = false + ) + log.fine("aliveWithOpts : ${aliveWithOpts.map { it.daemon.javaClass.name }}") + val comparator = compareByDescending( + DaemonJVMOptionsMemoryComparator(), + { it.jvmOptions } + ) + .thenBy { + when (it.daemon) { + is CompileServiceAsyncWrapper -> 0 + else -> 1 + } + } + .thenBy(FileAgeComparator()) { it.runFile } + .thenBy { it.daemon.serverPort } + aliveWithOpts.maxWith(comparator)?.let { bestDaemonWithMetadata -> + val fattestOpts = bestDaemonWithMetadata.jvmOptions + if (fattestOpts memorywiseFitsInto daemonJVMOptions && FileAgeComparator().compare( + bestDaemonWithMetadata.runFile, + runFile + ) < 0 + ) { + // all others are smaller that me, take overs' clients and shut them down + log.info("$LOG_PREFIX_ASSUMING_OTHER_DAEMONS_HAVE lower prio, taking clients from them and schedule them to shutdown: my runfile: ${runFile.name} (${runFile.lastModified()}) vs best other runfile: ${bestDaemonWithMetadata.runFile.name} (${bestDaemonWithMetadata.runFile.lastModified()})") + aliveWithOpts.forEach { (daemon, runFile, _) -> + try { + log.fine("other : $daemon") + daemon.getClients().takeIf { it.isGood }?.let { + it.get().forEach { clientAliveFile -> + registerClientImpl(clientAliveFile) + } + } + log.fine("other : CLIENTS_OK") + daemon.scheduleShutdown(true) + log.fine("other : SHUTDOWN_OK") + } catch (e: Throwable) { + log.info("Cannot connect to a daemon, assuming dying ('${runFile.canonicalPath}'): ${e.message}") + } + } + } + // TODO: seems that the second part of condition is incorrect, reconsider: + // the comment by @tsvtkv from review: + // Algorithm in plain english: + // (1) If the best daemon fits into me and the best daemon is younger than me, then I take over all other daemons clients. + // (2) If I fit into the best daemon and the best daemon is older than me, then I give my clients to that daemon. + // + // For example: + // + // daemon A starts with params: maxMem=100, codeCache=50 + // daemon B starts with params: maxMem=200, codeCache=50 + // daemon C starts with params: maxMem=150, codeCache=100 + // A performs election: (1) is false because neither B nor C does not fit into A, (2) is false because both B and C are younger than A. + // B performs election: (1) is false because neither A nor C does not fit into B, (2) is false because B does not fit into neither A nor C. + // C performs election: (1) is false because B is better than A and B does not fit into C, (2) is false C does not fit into neither A nor B. + // Result: all daemons are alive and well. + else if (daemonJVMOptions memorywiseFitsInto fattestOpts && FileAgeComparator().compare( + bestDaemonWithMetadata.runFile, + runFile + ) > 0 + ) { + // there is at least one bigger, handover my clients to it and shutdown + log.info("$LOG_PREFIX_ASSUMING_OTHER_DAEMONS_HAVE higher prio, handover clients to it and schedule shutdown: my runfile: ${runFile.name} (${runFile.lastModified()}) vs best other runfile: ${bestDaemonWithMetadata.runFile.name} (${bestDaemonWithMetadata.runFile.lastModified()})") + getClientsImpl().takeIf { it.isGood }?.let { + it.get().forEach { bestDaemonWithMetadata.daemon.registerClient(it) } + } + scheduleShutdownImpl(true) + } else { + // undecided, do nothing + log.info("$LOG_PREFIX_ASSUMING_OTHER_DAEMONS_HAVE equal prio, continue: ${runFile.name} (${runFile.lastModified()}) vs best other runfile: ${bestDaemonWithMetadata.runFile.name} (${bestDaemonWithMetadata.runFile.lastModified()})") + // TODO: implement some behaviour here, e.g.: + // - shutdown/takeover smaller daemon + // - runServer (or better persuade client to runServer) a bigger daemon (in fact may be even simple shutdown will do, because of client's daemon choosing logic) + } + } + + } + } + } + + private fun shutdownNow() { + log.info("Shutdown started") + fun Long.mb() = this / (1024 * 1024) + with(Runtime.getRuntime()) { + log.info("Memory stats: total: ${totalMemory().mb()}mb, free: ${freeMemory().mb()}mb, max: ${maxMemory().mb()}mb") + } + state.alive.set(Aliveness.Dying.ordinal) + shutdownServer() + log.info("Shutdown complete") + onShutdown() + log.handlers.forEach { it.flush() } + } + + private fun shutdownWithDelayImpl(currentClientsCount: Int, currentSessionId: Int, currentCompilationsCount: Int) { + log.fine("${log.name} .......shutdowning........") + log.fine("${log.name} currentCompilationsCount = $currentCompilationsCount, compilationsCounter.get(): ${compilationsCounter.get()}") + state.delayedShutdownQueued.set(false) + if (currentClientsCount == state.clientsCounter && + currentCompilationsCount == compilationsCounter.get() && + currentSessionId == state.sessions.lastSessionId + ) { + log.fine("currentCompilationsCount == compilationsCounter.get()") + runBlocking(Dispatchers.Unconfined) { + ifAliveExclusiveUnit(minAliveness = Aliveness.LastSession) { + log.info("Execute delayed shutdown") + shutdownNow() + } + } + } else { + log.info("Cancel delayed shutdown due to a new activity") + } + } + + private fun shutdownWithDelay() { + state.delayedShutdownQueued.set(true) + val currentClientsCount = state.clientsCounter + val currentSessionId = state.sessions.lastSessionId + val currentCompilationsCount = compilationsCounter.get() + log.info("Delayed shutdown in ${daemonOptions.shutdownDelayMilliseconds}ms") + timer.schedule(daemonOptions.shutdownDelayMilliseconds) { + shutdownWithDelayImpl(currentClientsCount, currentSessionId, currentCompilationsCount) + } + } + + private fun gracefulShutdown(onAnotherThread: Boolean): Boolean { + + if (!state.alive.compareAndSet(Aliveness.Alive.ordinal, Aliveness.LastSession.ordinal)) { + log.info("Invalid state for graceful shutdown: ${state.alive.get().toAlivenessName()}") + return false + } + log.info("Graceful shutdown signalled") + + if (!onAnotherThread) { + shutdownIfIdle() + } else { + timer.schedule(1) { + gracefulShutdownImpl() + } + + } + return true + } + + private fun gracefulShutdownImpl() { + runBlocking(Dispatchers.Unconfined) { + ifAliveExclusiveUnit(minAliveness = Aliveness.LastSession) { + shutdownIfIdle() + } + } + } + + private fun shutdownIfIdle() = when { + state.sessions.isEmpty() -> shutdownWithDelay() + else -> { + daemonOptions.autoshutdownIdleSeconds = + TimeUnit.MILLISECONDS.toSeconds(daemonOptions.forceShutdownTimeoutMilliseconds).toInt() + daemonOptions.autoshutdownUnusedSeconds = daemonOptions.autoshutdownIdleSeconds + log.info("Some sessions are active, waiting for them to finish") + log.info("Unused/idle timeouts are set to ${daemonOptions.autoshutdownUnusedSeconds}/${daemonOptions.autoshutdownIdleSeconds}s") + } + } + + private suspend fun createCompileServices( + facade: CompilerCallbackServicesFacadeClientSide, + eventManager: EventManager, + rpcProfiler: Profiler + ): Services { + val builder = Services.Builder() + if (facade.hasIncrementalCaches()) { + builder.register( + IncrementalCompilationComponents::class.java, + RemoteIncrementalCompilationComponentsClient(facade, eventManager, rpcProfiler) + ) + } + if (facade.hasLookupTracker()) { + builder.register(LookupTracker::class.java, RemoteLookupTrackerClient(facade, eventManager, rpcProfiler)) + } + if (facade.hasCompilationCanceledStatus()) { + log.fine("facade.hasCompilationCanceledStatus() = true") + builder.register(CompilationCanceledStatus::class.java, RemoteCompilationCanceledStatusClient(facade, rpcProfiler)) + } else { + log.fine("facade.hasCompilationCanceledStatus() = false") + } + return builder.build() + } + + override suspend fun clearJarCache() { + ZipHandler.clearFileAccessorCache() + (KotlinCoreEnvironment.applicationEnvironment?.jarFileSystem as? CoreJarFileSystem)?.clearHandlersCache() + } + + private suspend fun ifAlive( + minAliveness: Aliveness = Aliveness.LastSession, + body: suspend () -> CompileService.CallResult + ): CompileService.CallResult { + val result = CompletableDeferred() + scheduler.scheduleTask(OrdinaryTaskWithResult(result) { + ifAliveChecksImplSuspend(minAliveness, body) + }) + return result.await() as CompileService.CallResult + } + + private suspend fun ifAliveUnit( + minAliveness: Aliveness = Aliveness.LastSession, + body: suspend () -> Unit + ) { + val completed = CompletableDeferred() + scheduler.scheduleTask( + OrdinaryTask(completed) { + ifAliveChecksImplSuspend(minAliveness) { + body() + CompileService.CallResult.Ok() + } + } + ) + completed.await() + } + + private suspend fun ifAliveExclusive( + minAliveness: Aliveness = Aliveness.LastSession, + body: suspend () -> CompileService.CallResult + ): CompileService.CallResult { + val result = CompletableDeferred() + scheduler.scheduleTask(ShutdownTaskWithResult(result) { + ifAliveChecksImplSuspend(minAliveness, body) + }) + return result.await() as CompileService.CallResult + } + + private suspend fun ifAliveExclusiveUnit( + minAliveness: Aliveness = Aliveness.LastSession, + body: suspend () -> Unit + ): CompileService.CallResult { + val result = CompletableDeferred() + scheduler.scheduleTask(ShutdownTaskWithResult(result) { + ifAliveChecksImplSuspend(minAliveness) { + body() + CompileService.CallResult.Ok() + } + }) + return result.await() as CompileService.CallResult + } + + private suspend fun ifAliveChecksImplSuspend( + minAliveness: Aliveness = Aliveness.LastSession, + body: suspend () -> CompileService.CallResult + ): CompileService.CallResult { + val curState = state.alive.get() + return when { + curState < minAliveness.ordinal -> { + log.info("Cannot perform operation, requested state: ${minAliveness.name} > actual: ${curState.toAlivenessName()}") + CompileService.CallResult.Dying() + } + else -> { + try { + body() + } catch (e: Throwable) { + log.log(Level.SEVERE, "Exception", e) + CompileService.CallResult.Error(e.message ?: "unknown") + } + } + } + } + +} diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/KotlinCompileDaemon.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/KotlinCompileDaemon.kt new file mode 100644 index 00000000000..0f467d991ac --- /dev/null +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/KotlinCompileDaemon.kt @@ -0,0 +1,202 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.experimental + +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.runBlocking +import org.jetbrains.kotlin.cli.common.CLICompiler +import org.jetbrains.kotlin.cli.common.environment.setIdeaIoUseFallback +import org.jetbrains.kotlin.cli.js.K2JSCompiler +import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler +import org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler +import org.jetbrains.kotlin.daemon.CompilerSelector +import org.jetbrains.kotlin.daemon.common.* +import org.jetbrains.kotlin.daemon.common.experimental.findPortForSocket +import org.jetbrains.kotlin.daemon.common.ensureServerHostnameIsSetUp +import java.io.File +import java.io.IOException +import java.io.OutputStream +import java.io.PrintStream +import java.lang.management.ManagementFactory +import java.net.URLClassLoader +import java.text.SimpleDateFormat +import java.util.* +import java.util.jar.Manifest +import java.util.logging.Level +import java.util.logging.LogManager +import java.util.logging.Logger +import kotlin.concurrent.schedule + + +class LogStream(name: String) : OutputStream() { + + val log by lazy { Logger.getLogger(name) } + + val lineBuf = StringBuilder() + + override fun write(byte: Int) { + if (byte.toChar() == '\n') flush() + else lineBuf.append(byte.toChar()) + } + + override fun flush() { + log.info(lineBuf.toString()) + lineBuf.setLength(0) + } +} + +object KotlinCompileDaemon { + + init { + + val logTime: String = SimpleDateFormat("yyyy-MM-dd.HH-mm-ss-SSS").format(Date()) + val (logPath: String, fileIsGiven: Boolean) = + System.getProperty(COMPILE_DAEMON_LOG_PATH_PROPERTY)?.trimQuotes()?.let { Pair(it, File(it).isFile) } ?: Pair("%t", false) + val cfg: String = + "handlers = java.util.logging.FileHandler\n" + + "java.util.logging.FileHandler.level = ALL\n" + + "java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter\n" + + "java.util.logging.FileHandler.encoding = UTF-8\n" + + "java.util.logging.FileHandler.limit = ${if (fileIsGiven) 0 else (1 shl 20)}\n" + // if file is provided - disabled, else - 1Mb + "java.util.logging.FileHandler.count = ${if (fileIsGiven) 1 else 3}\n" + + "java.util.logging.FileHandler.append = $fileIsGiven\n" + + "java.util.logging.FileHandler.pattern = ${if (fileIsGiven) logPath else (logPath + File.separator + "$COMPILE_DAEMON_DEFAULT_FILES_PREFIX.$logTime.%u%g.log")}\n" + + "java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s%n\n" + + LogManager.getLogManager().readConfiguration(cfg.byteInputStream()) + } + + val log by lazy { Logger.getLogger("daemon") } + + private fun loadVersionFromResource(): String? { + (KotlinCompileDaemon::class.java.classLoader as? URLClassLoader) + ?.findResource("META-INF/MANIFEST.MF") + ?.let { + try { + return Manifest(it.openStream()).mainAttributes.getValue("Implementation-Version") ?: null + } catch (e: IOException) { + } + } + return null + } + + @JvmStatic + fun main(args: Array) { + ensureServerHostnameIsSetUp() + + val jvmArguments = ManagementFactory.getRuntimeMXBean().inputArguments + + log.info("Kotlin compiler daemon version " + (loadVersionFromResource() ?: "")) + log.info("daemon JVM args: " + jvmArguments.joinToString(" ")) + log.info("daemon args: " + args.joinToString(" ")) + + setIdeaIoUseFallback() + + val compilerId = CompilerId() + val daemonOptions = DaemonOptions() + + runBlocking { + + var serverRun: Deferred? + + try { + val daemonJVMOptions = configureDaemonJVMOptions( + inheritMemoryLimits = true, + inheritOtherJvmOptions = true, + inheritAdditionalProperties = true + ) + + val filteredArgs = args.asIterable() + .filterExtractProps( + compilerId, + daemonOptions, + prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX + ) + + if (filteredArgs.any()) { + val helpLine = "usage: " + + log.info(helpLine) + println(helpLine) + throw IllegalArgumentException("Unknown arguments: " + filteredArgs.joinToString(" ")) + } + + log.info("starting daemon") + + // TODO: find minimal set of permissions and restore security management + // note: may be not needed anymore since (hopefully) server is now loopback-only + // if (System.getSecurityManager() == null) + // System.setSecurityManager (RMISecurityManager()) + // + // setDaemonPermissions(daemonOptions.socketPort) + + val port = findPortForSocket( + COMPILE_DAEMON_FIND_PORT_ATTEMPTS, + COMPILE_DAEMON_PORTS_RANGE_START, + COMPILE_DAEMON_PORTS_RANGE_END + ) + + + val compilerSelector = object : CompilerSelector { + private val jvm by lazy { K2JVMCompiler() } + private val js by lazy { K2JSCompiler() } + private val metadata by lazy { K2MetadataCompiler() } + override fun get(targetPlatform: CompileService.TargetPlatform): CLICompiler<*> = when (targetPlatform) { + CompileService.TargetPlatform.JVM -> jvm + CompileService.TargetPlatform.JS -> js + CompileService.TargetPlatform.METADATA -> metadata + } + } + + // timer with a daemon thread, meaning it should not prevent JVM to exit normally + val timer = Timer(true) + val compilerService = CompileServiceServerSideImpl( + port, + compilerSelector, + compilerId, + daemonOptions, + daemonJVMOptions, + port.port, + timer, + { + if (daemonOptions.forceShutdownTimeoutMilliseconds != COMPILE_DAEMON_TIMEOUT_INFINITE_MS) { + // running a watcher thread that ensures that if the daemon is not exited normally (may be due to RMI leftovers), it's forced to exit + timer.schedule(daemonOptions.forceShutdownTimeoutMilliseconds) { + cancel() + log.info("force JVM shutdown") + System.exit(0) + } + } else { + timer.cancel() + } + }) + compilerService.startDaemonLife() + serverRun = compilerService.runServer() + + + println(COMPILE_DAEMON_IS_READY_MESSAGE) + log.info("daemon is listening on port: ${port.port}") + + // this supposed to stop redirected streams reader(s) on the client side and prevent some situations with hanging threads, but doesn't work reliably + // TODO: implement more reliable scheme + System.out.close() + System.err.close() + + System.setErr(PrintStream(LogStream("stderr"))) + System.setOut(PrintStream(LogStream("stdout"))) + } catch (e: Exception) { + log.log(Level.ALL, "Exception: " + e.message) + e.printStackTrace(System.err) + // repeating it to log for the cases when stderr is not redirected yet + log.log(Level.INFO, "Exception: ", e) + // TODO consider exiting without throwing + throw e + } + serverRun.await() + } + } + +} \ No newline at end of file diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/KotlinRemoteReplService.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/KotlinRemoteReplService.kt new file mode 100644 index 00000000000..4d5cbfe4927 --- /dev/null +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/KotlinRemoteReplService.kt @@ -0,0 +1,51 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.experimental + +import com.intellij.openapi.Disposable +import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.cli.common.repl.IReplStageState +import org.jetbrains.kotlin.cli.jvm.repl.GenericReplCompilerState +import org.jetbrains.kotlin.daemon.KotlinJvmReplServiceBase +import org.jetbrains.kotlin.daemon.common.CompileService +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.ServerSocketWrapper +import org.jetbrains.kotlin.daemon.getValidId +import java.io.File +import java.util.* +import kotlin.concurrent.read +import kotlin.concurrent.write + +open class KotlinJvmReplServiceAsync( + disposable: Disposable, + val portForServers: ServerSocketWrapper, + templateClasspath: List, + templateClassName: String, + messageCollector: MessageCollector +) : KotlinJvmReplServiceBase(disposable, templateClasspath, templateClassName, messageCollector) { + + protected val states = WeakHashMap() // used as (missing) WeakHashSet + + suspend fun createRemoteState(port: ServerSocketWrapper = portForServers): RemoteReplStateFacadeServerSide = statesLock.write { + val id = getValidId(stateIdCounter) { id -> states.none { it.key.getId() == id } } + val stateFacade = RemoteReplStateFacadeServerSide( + id, + createState().asState(GenericReplCompilerState::class.java), + port + ) + stateFacade.runServer() + states.put(stateFacade, true) + stateFacade + } + + suspend fun withValidReplState( + stateId: Int, + body: (IReplStageState<*>) -> R + ): CompileService.CallResult = statesLock.read { + states.keys.firstOrNull { it.getId() == stateId }?.let { + CompileService.CallResult.Good(body(it.state)) + } ?: CompileService.CallResult.Error("No REPL state with id $stateId found") + } +} \ No newline at end of file diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteCompilationCanceledStatusClient.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteCompilationCanceledStatusClient.kt new file mode 100644 index 00000000000..5a1b349ac79 --- /dev/null +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteCompilationCanceledStatusClient.kt @@ -0,0 +1,69 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.daemon.experimental + +import org.jetbrains.kotlin.daemon.common.RmiFriendlyCompilationCanceledException +import org.jetbrains.kotlin.daemon.common.experimental.CompilerCallbackServicesFacadeClientSide +import org.jetbrains.kotlin.daemon.common.DummyProfiler +import org.jetbrains.kotlin.daemon.common.Profiler +import org.jetbrains.kotlin.daemon.common.withMeasure +import org.jetbrains.kotlin.progress.CompilationCanceledException +import org.jetbrains.kotlin.progress.experimental.CompilationCanceledStatus +import java.util.concurrent.TimeUnit +import java.util.logging.Logger + +val CANCELED_STATUS_CHECK_THRESHOLD_NS = TimeUnit.MILLISECONDS.toNanos(100) + +class RemoteCompilationCanceledStatusClient( + val facade: CompilerCallbackServicesFacadeClientSide, + val profiler: Profiler = DummyProfiler() +) : CompilationCanceledStatus { + + private val log by lazy { Logger.getLogger("RemoteCompilationCanceledStatusClient") } + + @Volatile + var lastChecked: Long = System.nanoTime() + + override suspend fun checkCanceled() { + + fun cancelOnError(e: Exception) { + log.warning("error communicating with host, assuming compilation canceled (${e.message})") + throw CompilationCanceledException() + } + + val curNanos = System.nanoTime() + if (curNanos - lastChecked > CANCELED_STATUS_CHECK_THRESHOLD_NS) { + profiler.withMeasure(this) { + try { + facade.compilationCanceledStatus_checkCanceled() + } catch (e: RmiFriendlyCompilationCanceledException) { + throw CompilationCanceledException() + } catch (e: java.rmi.ConnectIOException) { + cancelOnError(e) + } catch (e: java.rmi.ConnectException) { + cancelOnError(e) + } catch (e: java.rmi.NoSuchObjectException) { + // this was added mostly for tests since others are more difficult to emulate + cancelOnError(e) + } catch (e: java.rmi.UnmarshalException) { + cancelOnError(e) + } + } + lastChecked = curNanos + } + } +} diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteIncrementalCacheClient.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteIncrementalCacheClient.kt new file mode 100644 index 00000000000..11e09df7526 --- /dev/null +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteIncrementalCacheClient.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.experimental + +import org.jetbrains.kotlin.daemon.common.experimental.CompilerCallbackServicesFacadeClientSide +import org.jetbrains.kotlin.daemon.common.DummyProfiler +import org.jetbrains.kotlin.daemon.common.Profiler +import org.jetbrains.kotlin.daemon.common.withMeasureBlocking +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache +import org.jetbrains.kotlin.load.kotlin.incremental.components.JvmPackagePartProto +import org.jetbrains.kotlin.modules.TargetId + +class RemoteIncrementalCacheClient(val facade: CompilerCallbackServicesFacadeClientSide, val target: TargetId, val profiler: Profiler = DummyProfiler()): + IncrementalCache { + + override fun getObsoletePackageParts(): Collection = profiler.withMeasureBlocking(this) { facade.incrementalCache_getObsoletePackageParts(target) } + + override fun getObsoleteMultifileClasses(): Collection = profiler.withMeasureBlocking(this) { facade.incrementalCache_getObsoleteMultifileClassFacades(target) } + + override fun getStableMultifileFacadeParts(facadeInternalName: String): Collection? = profiler.withMeasureBlocking(this) { facade.incrementalCache_getMultifileFacadeParts(target, facadeInternalName) } + + override fun getPackagePartData(partInternalName: String): JvmPackagePartProto? = profiler.withMeasureBlocking(this) { facade.incrementalCache_getPackagePartData(target, partInternalName) } + + override fun getModuleMappingData(): ByteArray? = profiler.withMeasureBlocking(this) { facade.incrementalCache_getModuleMappingData(target) } + + override fun getClassFilePath(internalClassName: String): String = profiler.withMeasureBlocking(this) { facade.incrementalCache_getClassFilePath(target, internalClassName) } + + override fun close(): Unit = profiler.withMeasureBlocking(this) { facade.incrementalCache_close(target) } +} diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteIncrementalCompilationComponentsClient.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteIncrementalCompilationComponentsClient.kt new file mode 100644 index 00000000000..13447f97b5e --- /dev/null +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteIncrementalCompilationComponentsClient.kt @@ -0,0 +1,18 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.experimental + +import org.jetbrains.kotlin.daemon.EventManager +import org.jetbrains.kotlin.daemon.common.experimental.CompilerCallbackServicesFacadeClientSide +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache +import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents +import org.jetbrains.kotlin.modules.TargetId +import org.jetbrains.kotlin.daemon.common.DummyProfiler +import org.jetbrains.kotlin.daemon.common.Profiler + +class RemoteIncrementalCompilationComponentsClient(val facade: CompilerCallbackServicesFacadeClientSide, eventManager: EventManager, val profiler: Profiler = DummyProfiler()) : IncrementalCompilationComponents { + override fun getIncrementalCache(target: TargetId): IncrementalCache = RemoteIncrementalCacheClient(facade, target, profiler) +} diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteLookupTrackerClient.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteLookupTrackerClient.kt new file mode 100644 index 00000000000..1ae91b8335d --- /dev/null +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteLookupTrackerClient.kt @@ -0,0 +1,59 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.experimental + +import com.intellij.util.containers.StringInterner +import kotlinx.coroutines.runBlocking +import org.jetbrains.kotlin.daemon.EventManager +import org.jetbrains.kotlin.daemon.common.experimental.CompilerCallbackServicesFacadeClientSide +import org.jetbrains.kotlin.daemon.common.DummyProfiler +import org.jetbrains.kotlin.daemon.common.Profiler +import org.jetbrains.kotlin.daemon.common.withMeasure +import org.jetbrains.kotlin.daemon.common.withMeasureBlocking +import org.jetbrains.kotlin.incremental.components.LookupInfo +import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.incremental.components.Position +import org.jetbrains.kotlin.incremental.components.ScopeKind + + +class RemoteLookupTrackerClient( + val facade: CompilerCallbackServicesFacadeClientSide, + eventManager: EventManager, + val profiler: Profiler = DummyProfiler() +) : LookupTracker { + private val isDoNothing = runBlocking { profiler.withMeasure(this) { facade.lookupTracker_isDoNothing() } } + + private val lookups = hashSetOf() + private val interner = StringInterner() + + override val requiresPosition: Boolean = runBlocking { profiler.withMeasure(this) { facade.lookupTracker_requiresPosition() } } + + override fun record(filePath: String, position: Position, scopeFqName: String, scopeKind: ScopeKind, name: String) { + if (isDoNothing) return + + val internedFilePath = interner.intern(filePath) + val internedScopeFqName = interner.intern(scopeFqName) + val internedName = interner.intern(name) + + lookups.add(LookupInfo(internedFilePath, position, internedScopeFqName, scopeKind, internedName)) + } + + init { + runBlocking { + eventManager.onCompilationFinished { flush() } + } + } + + private fun flush() { + if (isDoNothing || lookups.isEmpty()) return + + profiler.withMeasureBlocking(this) { + facade.lookupTracker_record(lookups) + } + + lookups.clear() + } +} diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteOutputStreamClient.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteOutputStreamClient.kt new file mode 100644 index 00000000000..a2944a1efe6 --- /dev/null +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteOutputStreamClient.kt @@ -0,0 +1,27 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.experimental + +import kotlinx.coroutines.runBlocking +import org.jetbrains.kotlin.daemon.common.experimental.RemoteOutputStreamAsyncClientSide +import org.jetbrains.kotlin.daemon.common.DummyProfiler +import org.jetbrains.kotlin.daemon.common.Profiler +import org.jetbrains.kotlin.daemon.common.withMeasure +import java.io.OutputStream + +class RemoteOutputStreamClient(val remote: RemoteOutputStreamAsyncClientSide, val profiler: Profiler = DummyProfiler()) : OutputStream() { + override fun write(data: ByteArray) = runBlocking { + profiler.withMeasure(this) { remote.write(data, 0, data.size) } + } + + override fun write(data: ByteArray, offset: Int, length: Int) = runBlocking { + profiler.withMeasure(this) { remote.write(data, offset, length) } + } + + override fun write(byte: Int) = runBlocking { + profiler.withMeasure(this) { remote.write(byte) } + } +} diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteReplStateFacadeImpl.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteReplStateFacadeImpl.kt new file mode 100644 index 00000000000..cd3baae7538 --- /dev/null +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/experimental/RemoteReplStateFacadeImpl.kt @@ -0,0 +1,73 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.experimental + +import io.ktor.network.sockets.Socket +import org.jetbrains.kotlin.cli.common.repl.ILineId +import org.jetbrains.kotlin.cli.jvm.repl.GenericReplCompilerState +import org.jetbrains.kotlin.daemon.common.COMPILE_DAEMON_FIND_PORT_ATTEMPTS +import org.jetbrains.kotlin.daemon.common.experimental.* +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.* + +@Suppress("UNCHECKED_CAST") +class RemoteReplStateFacadeServerSide( + val _id: Int, + val state: GenericReplCompilerState, + override val serverSocketWithPort: ServerSocketWrapper = findPortForSocket( + COMPILE_DAEMON_FIND_PORT_ATTEMPTS, + REPL_SERVER_PORTS_RANGE_START, + REPL_SERVER_PORTS_RANGE_END + ) +) : ReplStateFacadeServerSide { + + override val clients = hashMapOf() + + override suspend fun getId(): Int = _id + + override suspend fun getHistorySize(): Int = state.history.size + + override suspend fun historyGet(index: Int): ILineId = state.history[index].id + + override suspend fun historyReset(): List = state.history.reset().toList() + + override suspend fun historyResetTo(id: ILineId): List = state.history.resetTo(id).toList() + + val clientSide: RemoteReplStateFacadeClientSide + get() = RemoteReplStateFacadeClientSide(serverSocketWithPort.port) + +} + + +class RemoteReplStateFacadeClientSide(val serverPort: Int) : ReplStateFacadeClientSide, Client by DefaultClient(serverPort) { + + override suspend fun getId(): Int { + val id = sendMessage(ReplStateFacadeServerSide.GetIdMessage()) + return readMessage(id) + } + + override suspend fun getHistorySize(): Int { + val id = sendMessage(ReplStateFacadeServerSide.GetHistorySizeMessage()) + return readMessage(id) + } + + override suspend fun historyGet(index: Int): ILineId { + val id = sendMessage(ReplStateFacadeServerSide.HistoryGetMessage(index)) + return readMessage(id) + } + + override suspend fun historyReset(): List { + val id = sendMessage(ReplStateFacadeServerSide.HistoryResetMessage()) + return readMessage(id) + } + + override suspend fun historyResetTo(id: ILineId): List { + val id = sendMessage(ReplStateFacadeServerSide.HistoryResetToMessage(id)) + return readMessage(id) + } + +} + + diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/DaemonMessageReporter.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/DaemonMessageReporter.kt index 4cb49f3a7c3..4ba022c0a19 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/DaemonMessageReporter.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/DaemonMessageReporter.kt @@ -19,11 +19,11 @@ package org.jetbrains.kotlin.daemon.report import org.jetbrains.kotlin.daemon.common.* import java.io.PrintStream -internal interface DaemonMessageReporter { +interface DaemonMessageReporter { fun report(severity: ReportSeverity, message: String) } -internal fun DaemonMessageReporter( +fun DaemonMessageReporter( servicesFacade: CompilerServicesFacadeBase, compilationOptions: CompilationOptions ): DaemonMessageReporter = diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/RemoteICReporter.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/RemoteICReporter.kt index 4dc2bfaca04..0a10d1141c2 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/RemoteICReporter.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/RemoteICReporter.kt @@ -7,6 +7,6 @@ package org.jetbrains.kotlin.daemon.report import org.jetbrains.kotlin.incremental.ICReporter -internal interface RemoteICReporter : ICReporter { +interface RemoteICReporter : ICReporter { fun flush() } \ No newline at end of file diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/experimental/CompileServicesFacadeMessageCollector.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/experimental/CompileServicesFacadeMessageCollector.kt new file mode 100644 index 00000000000..460c783151f --- /dev/null +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/experimental/CompileServicesFacadeMessageCollector.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.report.experimental + +import kotlinx.coroutines.* +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity +import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.cli.common.messages.MessageRenderer +import org.jetbrains.kotlin.daemon.KotlinCompileDaemon.log +import org.jetbrains.kotlin.daemon.common.CompilationOptions +import org.jetbrains.kotlin.daemon.common.ReportCategory +import org.jetbrains.kotlin.daemon.common.ReportSeverity +import org.jetbrains.kotlin.daemon.common.CompilerServicesFacadeBaseAsync +import org.jetbrains.kotlin.daemon.common.report + +internal class CompileServicesFacadeMessageCollector( + private val servicesFacade: CompilerServicesFacadeBaseAsync, + compilationOptions: CompilationOptions +) : MessageCollector { + private val mySeverity = compilationOptions.reportSeverity + private var hasErrors = false + + override fun clear() { + hasErrors = false + } + + override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation?) { + GlobalScope.async { + log.info("Message: " + MessageRenderer.WITHOUT_PATHS.render(severity, message, location)) + when (severity) { + CompilerMessageSeverity.OUTPUT -> { + servicesFacade.report(ReportCategory.OUTPUT_MESSAGE, ReportSeverity.ERROR, message) + } + CompilerMessageSeverity.EXCEPTION -> { + servicesFacade.report(ReportCategory.EXCEPTION, ReportSeverity.ERROR, message) + } + else -> { + val reportSeverity = when (severity) { + CompilerMessageSeverity.ERROR -> ReportSeverity.ERROR + CompilerMessageSeverity.WARNING, CompilerMessageSeverity.STRONG_WARNING -> ReportSeverity.WARNING + CompilerMessageSeverity.INFO -> ReportSeverity.INFO + else -> ReportSeverity.DEBUG + } + + if (reportSeverity.code <= mySeverity) { + servicesFacade.report(ReportCategory.COMPILER_MESSAGE, reportSeverity, message, location) + } + } + } + + hasErrors = hasErrors || severity.isError + } + } + + override fun hasErrors(): Boolean = hasErrors +} diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/experimental/DaemonMessageReporterAsync.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/experimental/DaemonMessageReporterAsync.kt new file mode 100644 index 00000000000..6a3529b136a --- /dev/null +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/experimental/DaemonMessageReporterAsync.kt @@ -0,0 +1,49 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.report.experimental + +import kotlinx.coroutines.* +import org.jetbrains.kotlin.daemon.common.CompilationOptions +import org.jetbrains.kotlin.daemon.common.ReportCategory +import org.jetbrains.kotlin.daemon.common.ReportSeverity +import org.jetbrains.kotlin.daemon.common.CompilerServicesFacadeBaseAsync +import org.jetbrains.kotlin.daemon.report.DaemonMessageReporter +import java.io.PrintStream + +internal fun DaemonMessageReporterAsync( + servicesFacade: CompilerServicesFacadeBaseAsync, + compilationOptions: CompilationOptions +): DaemonMessageReporter = + if (ReportCategory.DAEMON_MESSAGE.code in compilationOptions.reportCategories) { + val mySeverity = ReportSeverity.fromCode(compilationOptions.reportSeverity)!! + DaemonMessageReporterAsyncAsyncImpl(servicesFacade, mySeverity) + } else { + DummyDaemonMessageReporterAsync + } + +internal class DaemonMessageReporterAsyncPrintStreamAdapter(private val out: PrintStream) : DaemonMessageReporter { + override fun report(severity: ReportSeverity, message: String) { + out.print("[Kotlin compile daemon][$severity] $message") + } +} + +private class DaemonMessageReporterAsyncAsyncImpl( + private val servicesFacade: CompilerServicesFacadeBaseAsync, + private val mySeverity: ReportSeverity +) : DaemonMessageReporter { + override fun report(severity: ReportSeverity, message: String) { + GlobalScope.async { + if (severity.code <= mySeverity.code) { + servicesFacade.report(ReportCategory.DAEMON_MESSAGE.code, severity.code, message, attachment = null) + } + } + } +} + +private object DummyDaemonMessageReporterAsync : DaemonMessageReporter { + override fun report(severity: ReportSeverity, message: String) { + } +} \ No newline at end of file diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/experimental/RemoteICReporter.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/experimental/RemoteICReporter.kt new file mode 100644 index 00000000000..7dd3220a1a8 --- /dev/null +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/experimental/RemoteICReporter.kt @@ -0,0 +1,170 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.report.experimental + +import kotlinx.coroutines.* +import org.jetbrains.kotlin.cli.common.ExitCode +import org.jetbrains.kotlin.daemon.common.* +import org.jetbrains.kotlin.daemon.report.RemoteICReporter +import org.jetbrains.kotlin.incremental.ICReporter +import org.jetbrains.kotlin.incremental.ICReporterBase +import java.io.File +import java.io.Serializable + +internal class DebugMessagesICReporterAsync( + private val servicesFacade: CompilerServicesFacadeBaseAsync, + rootDir: File, + private val isVerbose: Boolean +) : ICReporterBase(rootDir), RemoteICReporter { + override fun report(message: () -> String) { + GlobalScope.async { + servicesFacade.report( + ReportCategory.IC_MESSAGE, + ReportSeverity.DEBUG, message() + ) + } + } + + override fun reportVerbose(message: () -> String) { + if (isVerbose) { + report(message) + } + } + + override fun reportCompileIteration(incremental: Boolean, sourceFiles: Collection, exitCode: ExitCode) { + } + + override fun flush() { + } +} + +internal class CompileIterationICReporterAsync( + private val compilationResults: CompilationResultsAsync? +) : ICReporterBase(), RemoteICReporter { + override fun reportCompileIteration(incremental: Boolean, sourceFiles: Collection, exitCode: ExitCode) { + GlobalScope.async { + compilationResults?.add( + CompilationResultCategory.IC_COMPILE_ITERATION.code, + CompileIterationResult(sourceFiles, exitCode.toString()) + ) + } + } + + override fun report(message: () -> String) { + } + + override fun reportVerbose(message: () -> String) { + } + + override fun flush() { + } +} + +internal class BuildReportICReporterAsync( + private val compilationResults: CompilationResultsAsync?, + rootDir: File, + private val isVerbose: Boolean = false +) : ICReporterBase(rootDir), RemoteICReporter { + private val icLogLines = arrayListOf() + private val recompilationReason = HashMap() + + override fun report(message: () -> String) { + icLogLines.add(message()) + } + + override fun reportVerbose(message: () -> String) { + if (isVerbose) { + report(message) + } + } + + override fun reportCompileIteration(incremental: Boolean, sourceFiles: Collection, exitCode: ExitCode) { + if (!incremental) return + + icLogLines.add("Compile iteration:") + for (file in sourceFiles) { + val reason = recompilationReason[file]?.let { " <- $it" } ?: "" + icLogLines.add(" ${file.relativeOrCanonical()}$reason") + } + recompilationReason.clear() + } + + override fun reportMarkDirty(affectedFiles: Iterable, reason: String) { + affectedFiles.forEach { recompilationReason[it] = reason } + } + + override fun flush() { + GlobalScope.async { + compilationResults?.add(CompilationResultCategory.BUILD_REPORT_LINES.code, icLogLines) + } + } +} + +internal class CompositeICReporterAsync(private val reporters: Iterable) : + RemoteICReporter { + override fun report(message: () -> String) { + reporters.forEach { it.report(message) } + } + + override fun reportVerbose(message: () -> String) { + reporters.forEach { it.reportVerbose(message) } + } + + override fun reportCompileIteration(incremental: Boolean, sourceFiles: Collection, exitCode: ExitCode) { + reporters.forEach { it.reportCompileIteration(incremental, sourceFiles, exitCode) } + } + + override fun reportMarkDirtyClass(affectedFiles: Iterable, classFqName: String) { + reporters.forEach { it.reportMarkDirtyClass(affectedFiles, classFqName) } + } + + override fun reportMarkDirtyMember(affectedFiles: Iterable, scope: String, name: String) { + reporters.forEach { it.reportMarkDirtyMember(affectedFiles, scope, name) } + } + + override fun reportMarkDirty(affectedFiles: Iterable, reason: String) { + reporters.forEach { it.reportMarkDirty(affectedFiles, reason) } + } + + override fun flush() { + reporters.forEach { it.flush() } + } +} + +fun getICReporterAsync( + servicesFacade: CompilerServicesFacadeBaseAsync, + compilationResults: CompilationResultsAsync?, + compilationOptions: IncrementalCompilationOptions +): RemoteICReporter { + val root = compilationOptions.modulesInfo.projectRoot + val reporters = ArrayList() + + if (ReportCategory.IC_MESSAGE.code in compilationOptions.reportCategories) { + val isVerbose = compilationOptions.reportSeverity == ReportSeverity.DEBUG.code + reporters.add(DebugMessagesICReporterAsync(servicesFacade, root, isVerbose = isVerbose)) + } + + val requestedResults = compilationOptions + .requestedCompilationResults + .mapNotNullTo(HashSet()) { resultCode -> + CompilationResultCategory.values().getOrNull(resultCode) + } + requestedResults.mapTo(reporters) { requestedResult -> + when (requestedResult) { + CompilationResultCategory.IC_COMPILE_ITERATION -> { + CompileIterationICReporterAsync(compilationResults) + } + CompilationResultCategory.BUILD_REPORT_LINES -> { + BuildReportICReporterAsync(compilationResults, root) + } + CompilationResultCategory.VERBOSE_BUILD_REPORT_LINES -> { + BuildReportICReporterAsync(compilationResults, root, isVerbose = true) + } + } + } + + return CompositeICReporterAsync(reporters) +} \ No newline at end of file diff --git a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/getICReporter.kt b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/getICReporter.kt index c5849dafad0..02292783073 100644 --- a/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/getICReporter.kt +++ b/compiler/daemon/src/org/jetbrains/kotlin/daemon/report/getICReporter.kt @@ -20,7 +20,7 @@ import org.jetbrains.kotlin.daemon.common.* import java.io.File import java.util.* -internal fun getICReporter( +fun getICReporter( servicesFacade: CompilerServicesFacadeBase, compilationResults: CompilationResults, compilationOptions: IncrementalCompilationOptions diff --git a/compiler/tests-common/build.gradle.kts b/compiler/tests-common/build.gradle.kts index 5e1cc10d502..02ce517fb90 100644 --- a/compiler/tests-common/build.gradle.kts +++ b/compiler/tests-common/build.gradle.kts @@ -27,6 +27,7 @@ dependencies { testCompile(project(":compiler:serialization")) testCompile(project(":kotlin-preloader")) testCompile(project(":compiler:daemon-common")) + testCompile(project(":compiler:daemon-common-new")) testCompile(project(":js:js.serializer")) testCompile(project(":js:js.frontend")) testCompile(project(":js:js.translator")) diff --git a/compiler/tests/org/jetbrains/kotlin/daemon/experimental/integration/CompilerApiTest.kt b/compiler/tests/org/jetbrains/kotlin/daemon/experimental/integration/CompilerApiTest.kt new file mode 100644 index 00000000000..2c8e30417e9 --- /dev/null +++ b/compiler/tests/org/jetbrains/kotlin/daemon/experimental/integration/CompilerApiTest.kt @@ -0,0 +1,326 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.experimental.integration + +import com.intellij.openapi.application.ApplicationManager +import kotlinx.coroutines.runBlocking +import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity +import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.cli.common.messages.OutputMessageUtil +import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler +import org.jetbrains.kotlin.config.Services +import org.jetbrains.kotlin.daemon.client.KotlinCompilerDaemonClient +import org.jetbrains.kotlin.daemon.client.DaemonReportingTargets +import org.jetbrains.kotlin.daemon.common.* +import org.jetbrains.kotlin.daemon.common.ReportSeverity +import org.jetbrains.kotlin.daemon.loggerCompatiblePath +import org.jetbrains.kotlin.integration.KotlinIntegrationTestBase +import org.jetbrains.kotlin.scripts.captureOut +import org.jetbrains.kotlin.test.KotlinTestUtils +import org.jetbrains.kotlin.test.testFramework.KtUsefulTestCase +import org.junit.Assert +import java.io.File +import java.net.URLClassLoader +import java.util.logging.LogManager +import java.util.logging.Logger + +private val logFiles = arrayListOf() + +// TODO: remove ignore annotation from tests. + +class CompilerApiTest : KotlinIntegrationTestBase() { + + val kotlinCompilerClient = KotlinCompilerDaemonClient + .instantiate(DaemonProtocolVariant.RMI) // TODO(SOCKETS) + + private val compilerLibDir = getCompilerLib() + + private fun createNewLogFile(): File { + println("creating logFile") + val newLogFile = createTempFile("kotlin-daemon-experimental-test.", ".log") + println("logFile created (${newLogFile.loggerCompatiblePath})") + logFiles.add(newLogFile.loggerCompatiblePath) + return newLogFile + } + + private val currentLogFile: File by lazy { + val newLogFile = createNewLogFile() + val cfg: String = + "handlers = java.util.logging.FileHandler\n" + + "java.util.logging.FileHandler.level = ALL\n" + + "java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter\n" + + "java.util.logging.FileHandler.encoding = UTF-8\n" + + "java.util.logging.FileHandler.limit = 0\n" + // if file is provided - disabled, else - 1Mb + "java.util.logging.FileHandler.count = 1\n" + + "java.util.logging.FileHandler.append = true\n" + + "java.util.logging.FileHandler.pattern = ${newLogFile.loggerCompatiblePath}\n" + + "java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s%n\n" + LogManager.getLogManager().readConfiguration(cfg.byteInputStream()) + newLogFile + } + + private val externalLogFile: File by lazy { createNewLogFile() } + + private val log by lazy { + currentLogFile + Logger.getLogger("test") + } + + val compilerClassPath = listOf( + File(compilerLibDir, "kotlin-compiler.jar") + ) + val scriptRuntimeClassPath = listOf( + File(compilerLibDir, "kotlin-runtime.jar"), + File(compilerLibDir, "kotlin-script-runtime.jar") + ) + val compilerId by lazy(LazyThreadSafetyMode.NONE) { CompilerId.makeCompilerId(compilerClassPath) } + + private fun compileLocally( + messageCollector: TestMessageCollector, + vararg args: String + ): Pair> { + val application = ApplicationManager.getApplication() + try { + val code = K2JVMCompiler().exec(messageCollector, + Services.EMPTY, + K2JVMCompilerArguments().apply { K2JVMCompiler().parseArguments(args, this) }).code + val outputs = messageCollector.messages.filter { it.severity == CompilerMessageSeverity.OUTPUT }.mapNotNull { + OutputMessageUtil.parseOutputMessage(it.message)?.let { outs -> + outs.outputFile?.let { OutputMessageUtil.Output(outs.sourceFiles, it) } + } + } + return code to outputs + } finally { + KtUsefulTestCase.resetApplicationToNull(application) + } + } + + private fun compileOnDaemon( + clientAliveFile: File, + compilerId: CompilerId, + daemonJVMOptions: DaemonJVMOptions, + daemonOptions: DaemonOptions, + messageCollector: MessageCollector, + vararg args: String + ): Pair> = runBlocking { + + log.info("kotlinCompilerClient.connectToCompileService() call") + val daemon = kotlinCompilerClient.connectToCompileService( + compilerId, + clientAliveFile, + daemonJVMOptions, + daemonOptions, + DaemonReportingTargets(messageCollector = messageCollector), + autostart = true + ) + log.info("kotlinCompilerClient.connectToCompileService() called! (daemon = $daemon)") + + assertNotNull("failed to connect daemon", daemon) + + log.info("runBlocking { ") + log.info("register client...") + daemon?.registerClient(clientAliveFile.absolutePath) + log.info(" client registered") + log.info("} ^ runBlocking") + + + val outputs = arrayListOf() + + val code = kotlinCompilerClient.compile( + daemon!!, + CompileService.NO_SESSION, + CompileService.TargetPlatform.JVM, + args, + messageCollector, + { outFile, srcFiles -> outputs.add(OutputMessageUtil.Output(srcFiles, outFile)) }, + reportSeverity = ReportSeverity.DEBUG + ) + code to outputs + } + + private fun getHelloAppBaseDir(): String = KotlinTestUtils.getTestDataPathBase() + "/integration/smoke/helloApp" + private fun getSimpleScriptBaseDir(): String = KotlinTestUtils.getTestDataPathBase() + "/integration/smoke/simpleScript" + + private fun run(baseDir: String, logName: String, vararg args: String): Int = runJava(baseDir, logName, *args) + + private fun runScriptWithArgs( + testDataDir: String, + logName: String?, + scriptClassName: String, + classpath: List, + vararg arguments: String + ) { + + val cl = URLClassLoader(classpath.map { it.toURI().toURL() }.toTypedArray()) + val scriptClass = cl.loadClass(scriptClassName) + + val scriptOut = captureOut { scriptClass.constructors.first().newInstance(arguments) } + + if (logName != null) { + val expectedFile = File(testDataDir, logName + ".expected") + val normalizedContent = normalizeOutput(File(testDataDir), "OUT:\n$scriptOut\nReturn code: 0") + + KotlinTestUtils.assertEqualsToFile(expectedFile, normalizedContent) + } + } + + fun ignore_testHelloAppLocal() { + val messageCollector = TestMessageCollector() + val jar = tmpdir.absolutePath + File.separator + "hello.jar" + val (code, outputs) = compileLocally( + messageCollector, "-include-runtime", File(getHelloAppBaseDir(), "hello.kt").absolutePath, + "-d", jar, "-Xreport-output-files" + ) + Assert.assertEquals(0, code) + Assert.assertTrue(outputs.isNotEmpty()) + Assert.assertEquals(jar, outputs.first().outputFile?.absolutePath) + run(getHelloAppBaseDir(), "hello.run", "-cp", jar, "Hello.HelloKt") + } + + private fun terminate(daemonOptions: DaemonOptions) { + println("\n\nkillall -9 Console && open ${logFiles.joinToString(" ")}\n\n") + log.info("in finally") +// runBlocking { +// log.info("in runBlocking") +// delay(1000L) +// kotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) +// } +// currentLogFile.delete() +// externalLogFile.delete() + } + + fun ignore_testHelloApp() { + withFlagFile(getTestName(true), ".alive") { flagFile -> + log.info("sarting test...") + + log.info("assigning daemonOptions") + val daemonOptions = DaemonOptions( + runFilesPath = File(tmpdir, getTestName(true)).absolutePath, + verbose = true, + reportPerf = true + ) + log.info("daemonOptions assigned") + + log.info("creating daemonJVMOptions") + val daemonJVMOptions = configureDaemonJVMOptions( + "D$COMPILE_DAEMON_LOG_PATH_PROPERTY=\"${externalLogFile.loggerCompatiblePath}\"", + inheritMemoryLimits = false, + inheritOtherJvmOptions = false, + inheritAdditionalProperties = false + ) + log.info("daemonJVMOptions created") + + log.info("creating jar") + val jar = tmpdir.absolutePath + File.separator + "hello.jar" + log.info("jar created") + + try { + log.info("compileOnDaemon call") + val (code, outputs) = compileOnDaemon( + flagFile, + compilerId, + daemonJVMOptions, + daemonOptions, + TestMessageCollector(), + "-include-runtime", + File(getHelloAppBaseDir(), "hello.kt").absolutePath, + "-d", + jar, + "-Xreport-output-files" + ) + log.info("compileOnDaemon called") + + Assert.assertEquals(0, code) + Assert.assertTrue(outputs.isNotEmpty()) + Assert.assertEquals(jar, outputs.first().outputFile?.absolutePath) + run(getHelloAppBaseDir(), "hello.run", "-cp", jar, "Hello.HelloKt") + } finally { + terminate(daemonOptions) + } + println("test passed") + } + } + + fun ignore_testSimpleScriptLocal() { + val messageCollector = TestMessageCollector() + val (code, outputs) = compileLocally( + messageCollector, + File(getSimpleScriptBaseDir(), "script.kts").absolutePath, + "-d", + tmpdir.absolutePath, + "-Xreport-output-files" + ) + Assert.assertEquals(0, code) + Assert.assertTrue(outputs.isNotEmpty()) + Assert.assertEquals(File(tmpdir, "Script.class").absolutePath, outputs.first().outputFile?.absolutePath) + runScriptWithArgs(getSimpleScriptBaseDir(), "script", "Script", scriptRuntimeClassPath + tmpdir, "hi", "there") + } + + fun ignore_testSimpleScript() { + withFlagFile(getTestName(true), ".alive") { flagFile -> + val daemonOptions = DaemonOptions( + runFilesPath = File(tmpdir, getTestName(true)).absolutePath, + verbose = true, + reportPerf = true + ) + val daemonJVMOptions = configureDaemonJVMOptions( + "D$COMPILE_DAEMON_LOG_PATH_PROPERTY=\"${externalLogFile.loggerCompatiblePath}\"", + inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false + ) + try { + val (code, outputs) = compileOnDaemon( + flagFile, + compilerId, + daemonJVMOptions, + daemonOptions, + TestMessageCollector(), + File(getSimpleScriptBaseDir(), "script.kts").absolutePath, + "-Xreport-output-files", + "-d", + tmpdir.absolutePath + ) + Assert.assertEquals(0, code) + Assert.assertTrue(outputs.isNotEmpty()) + Assert.assertEquals(File(tmpdir, "Script.class").absolutePath, outputs.first().outputFile?.absolutePath) + runScriptWithArgs(getSimpleScriptBaseDir(), "script", "Script", scriptRuntimeClassPath + tmpdir, "hi", "there") + } finally { + terminate(daemonOptions) + } + } + } + +} + +class TestMessageCollector : MessageCollector { + data class Message(val severity: CompilerMessageSeverity, val message: String, val location: CompilerMessageLocation?) + + val messages = arrayListOf() + + override fun clear() { + messages.clear() + } + + override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation?) { + messages.add(Message(severity, message, location)) + } + + override fun hasErrors(): Boolean = + messages.any { it.severity == CompilerMessageSeverity.EXCEPTION || it.severity == CompilerMessageSeverity.ERROR } + + override fun toString(): String { + return messages.joinToString("\n") { "${it.severity}: ${it.message}${it.location?.let { " at $it" } ?: ""}" } + } +} + +fun TestMessageCollector.assertHasMessage(msg: String, desiredSeverity: CompilerMessageSeverity? = null) { + assert(messages.any { it.message.contains(msg) && (desiredSeverity == null || it.severity == desiredSeverity) }) { + "Expecting message \"$msg\" with severity ${desiredSeverity?.toString() ?: "Any"}, actual:\n" + + messages.joinToString("\n") { it.severity.toString() + ": " + it.message } + } +} + diff --git a/compiler/tests/org/jetbrains/kotlin/daemon/experimental/integration/CompilerDaemonTest.kt b/compiler/tests/org/jetbrains/kotlin/daemon/experimental/integration/CompilerDaemonTest.kt new file mode 100644 index 00000000000..34c8afab97a --- /dev/null +++ b/compiler/tests/org/jetbrains/kotlin/daemon/experimental/integration/CompilerDaemonTest.kt @@ -0,0 +1,1436 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.experimental.integration + +import junit.framework.TestCase +import kotlinx.coroutines.* +import org.jetbrains.kotlin.cli.AbstractCliTest +import org.jetbrains.kotlin.cli.common.CLICompiler +import org.jetbrains.kotlin.cli.common.messages.MessageRenderer +import org.jetbrains.kotlin.cli.common.messages.PrintingMessageCollector +import org.jetbrains.kotlin.cli.common.repl.* +import org.jetbrains.kotlin.cli.js.K2JSCompiler +import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler +import org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler +import org.jetbrains.kotlin.daemon.* +import org.jetbrains.kotlin.daemon.client.DaemonReportingTargets +import org.jetbrains.kotlin.daemon.client.KotlinCompilerClientInstance +import org.jetbrains.kotlin.daemon.client.KotlinCompilerDaemonClient +import org.jetbrains.kotlin.daemon.common.* +import org.jetbrains.kotlin.integration.KotlinIntegrationTestBase +import org.jetbrains.kotlin.progress.experimental.CompilationCanceledStatus +import org.jetbrains.kotlin.test.KotlinTestUtils +import java.io.ByteArrayOutputStream +import java.io.File +import java.io.IOException +import java.io.PrintStream +import java.lang.management.ManagementFactory +import java.net.URL +import java.net.URLClassLoader +import java.nio.channels.ClosedChannelException +import java.nio.charset.Charset +import java.rmi.ConnectException +import java.rmi.ConnectIOException +import java.rmi.UnmarshalException +import java.util.* +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.logging.LogManager +import kotlin.concurrent.thread +import kotlin.script.dependencies.Environment +import kotlin.script.dependencies.ScriptContents +import kotlin.script.experimental.dependencies.DependenciesResolver +import kotlin.script.experimental.dependencies.DependenciesResolver.ResolveResult +import kotlin.script.experimental.dependencies.ScriptDependencies +import kotlin.script.experimental.dependencies.asSuccess +import kotlin.script.templates.ScriptTemplateDefinition +import kotlin.test.fail +import org.jetbrains.kotlin.daemon.client.experimental.* +import org.jetbrains.kotlin.daemon.common.experimental.* +import org.jetbrains.kotlin.daemon.common.experimental.LoopbackNetworkInterface + +val TIMEOUT_DAEMON_RUNNER_EXIT_MS = 10000L + +// TODO: remove ignore annotation from tests. + +class CompilerDaemonTest : KotlinIntegrationTestBase() { + + val kotlinCompilerClientInstance = KotlinCompilerDaemonClient.instantiate(DaemonProtocolVariant.SOCKETS) + + private fun createNewLogFile(): File { + println("creating logFile") + val newLogFile = createTempFile("kotlin-daemon-experimental-test.", ".log") + println("logFile created (${newLogFile.loggerCompatiblePath})") + return newLogFile + } + + init { + val newLogFile = createNewLogFile() + val cfg: String = + "handlers = java.util.logging.FileHandler\n" + + "java.util.logging.FileHandler.level = ALL\n" + + "java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter\n" + + "java.util.logging.FileHandler.encoding = UTF-8\n" + + "java.util.logging.FileHandler.limit = 0\n" + // if file is provided - disabled, else - 1Mb + "java.util.logging.FileHandler.count = 1\n" + + "java.util.logging.FileHandler.append = true\n" + + "java.util.logging.FileHandler.pattern = ${newLogFile.loggerCompatiblePath}\n" + + "java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s%n\n" + LogManager.getLogManager().readConfiguration(cfg.byteInputStream()) + } + + data class CompilerResults(val resultCode: Int, val out: String) + + val compilerClassPath = listOf( + File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-compiler.jar") + ) + + val scriptingCompilerClassPath = listOf( + File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-scripting-compiler.jar"), + File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-scripting-common.jar"), + File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-scripting-jvm.jar") + ) + + val daemonClientClassPath = listOf( + File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-daemon-client-new.jar"), + File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-compiler.jar") + ) + + val compilerId by lazy(LazyThreadSafetyMode.NONE) { CompilerId.makeCompilerId(compilerClassPath) } + + val compilerWithScriptingId by lazy(LazyThreadSafetyMode.NONE) { + CompilerId.makeCompilerId(compilerClassPath + scriptingCompilerClassPath) + } + + private fun compileOnDaemon( + clientAliveFile: File, + compilerId: CompilerId, + daemonJVMOptions: DaemonJVMOptions, + daemonOptions: DaemonOptions, + vararg args: String + ): Deferred = GlobalScope.async { + val daemon = kotlinCompilerClientInstance.connectToCompileService( + compilerId, + clientAliveFile, + daemonJVMOptions, + daemonOptions, + DaemonReportingTargets(out = System.err), + autostart = true + ) + assertNotNull("failed to connect daemon", daemon) + daemon?.registerClient(clientAliveFile.absolutePath) + val strm = ByteArrayOutputStream() + val code = kotlinCompilerClientInstance.compile( + daemon!!, CompileService.NO_SESSION, CompileService.TargetPlatform.JVM, + args, PrintingMessageCollector(PrintStream(strm), MessageRenderer.WITHOUT_PATHS, true), + reportSeverity = ReportSeverity.DEBUG + ) + CompilerResults(code, strm.toString()) + } + + private fun runDaemonCompilerTwice( + clientAliveFile: File, + compilerId: CompilerId, + daemonJVMOptions: DaemonJVMOptions, + daemonOptions: DaemonOptions, + vararg args: String + ) { + runBlocking { + val res1 = compileOnDaemon(clientAliveFile, compilerId, daemonJVMOptions, daemonOptions, *args).await() + assertEquals("first compilation failed:\n${res1.out}", 0, res1.resultCode) + val res2 = compileOnDaemon(clientAliveFile, compilerId, daemonJVMOptions, daemonOptions, *args).await() + assertEquals("second compilation failed:\n${res2.out}", 0, res2.resultCode) + assertEquals( + "build results differ", + AbstractCliTest.removePerfOutput(res1.out).split('\n').toSortedSet(), + AbstractCliTest.removePerfOutput(res2.out).split('\n').toSortedSet() + ) + } + } + + private fun getTestBaseDir(): String = KotlinTestUtils.getTestDataPathBase() + "/integration/smoke/" + getTestName(true) + private fun getHelloAppBaseDir(): String = KotlinTestUtils.getTestDataPathBase() + "/integration/smoke/helloApp" + + private fun run(logName: String, vararg args: String): Int = runJava(getTestBaseDir(), logName, *args) + + fun makeTestDaemonOptions(testName: String, shutdownDelay: Int = 5) = + DaemonOptions( + runFilesPath = File(tmpdir, testName).absolutePath, + shutdownDelayMilliseconds = shutdownDelay.toLong(), + verbose = true, + reportPerf = true + ) + + fun makeTestDaemonJvmOptions(logFile: File? = null, xmx: Int = 384, args: Iterable = listOf()): DaemonJVMOptions { + val additionalArgs = arrayListOf() + if (logFile != null) { + additionalArgs.add("D$COMPILE_DAEMON_LOG_PATH_PROPERTY=\"${logFile.loggerCompatiblePath}\"") + } + args.forEach { additionalArgs.add(it) } + val baseOpts = if (xmx > 0) DaemonJVMOptions(maxMemory = "${xmx}m") else DaemonJVMOptions() + return configureDaemonJVMOptions( + baseOpts, + *additionalArgs.toTypedArray(), + inheritMemoryLimits = xmx <= 0, + inheritAdditionalProperties = false, + inheritOtherJvmOptions = false + ) + } + + fun testHelloApp() { + withFlagFile(getTestName(true), ".alive") { flagFile -> + runBlocking { + val daemonOptions = makeTestDaemonOptions(getTestName(true)) + + withLogFile("kotlin-daemon-test") { logFile -> + val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) + var daemonShotDown = false + + try { + val jar = tmpdir.absolutePath + File.separator + "hello.jar" + runDaemonCompilerTwice( + flagFile, compilerId, daemonJVMOptions, daemonOptions, + "-include-runtime", File(getTestBaseDir(), "hello.kt").absolutePath, "-d", jar + ) + kotlinCompilerClientInstance.shutdownCompileService(compilerId, daemonOptions) + delay(100) + daemonShotDown = true + var compileTime1 = 0L + var compileTime2 = 0L + logFile.assertLogContainsSequence( + LinePattern("Kotlin compiler daemon version"), + LinePattern("Starting compilation with args: "), + LinePattern("Compile on daemon: (\\d+) ms", { it.groups[1]?.value?.toLong()?.let { compileTime1 = it }; true }), + LinePattern("Starting compilation with args: "), + LinePattern("Compile on daemon: (\\d+) ms", { it.groups[1]?.value?.toLong()?.let { compileTime2 = it }; true }), + LinePattern("Shutdown started") + ) + assertTrue( + "Expecting that compilation 1 ($compileTime1 ms) is at least two times longer than compilation 2 ($compileTime2 ms)", + compileTime1 > compileTime2 * 2 + ) + logFile.delete() + run("hello.run", "-cp", jar, "Hello.HelloKt") + } finally { + if (!daemonShotDown) + kotlinCompilerClientInstance.shutdownCompileService(compilerId, daemonOptions) + } + } + } + } + } + + fun testDaemonJvmOptionsParsing() { + val backupJvmOptions = System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY) + try { + System.setProperty( + COMPILE_DAEMON_JVM_OPTIONS_PROPERTY, + "-aaa,-bbb\\,ccc,-ddd,-Xmx200m,-XX:MaxPermSize=10k,-XX:ReservedCodeCacheSize=100,-xxx\\,yyy" + ) + val opts = + configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false, inheritOtherJvmOptions = false) + assertEquals("200m", opts.maxMemory) + assertEquals("10k", opts.maxPermSize) + assertEquals("100", opts.reservedCodeCacheSize) + assertEquals(arrayListOf("aaa", "bbb,ccc", "ddd", "xxx,yyy"), opts.jvmParams) + + System.setProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY, "-Xmx300m,-XX:MaxPermSize=10k,-XX:ReservedCodeCacheSize=100") + val opts2 = + configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false, inheritOtherJvmOptions = false) + assertEquals("300m", opts2.maxMemory) + assertEquals(-1, DaemonJVMOptionsMemoryComparator().compare(opts, opts2)) + assertEquals("300m", listOf(opts, opts2).maxWith(DaemonJVMOptionsMemoryComparator())?.maxMemory) + + val myXmxParam = ManagementFactory.getRuntimeMXBean().inputArguments.first { it.startsWith("-Xmx") } + TestCase.assertNotNull(myXmxParam) + val myXmxVal = myXmxParam.substring(4) + System.clearProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY) + val opts3 = configureDaemonJVMOptions( + inheritMemoryLimits = true, + inheritOtherJvmOptions = true, + inheritAdditionalProperties = false + ) + assertEquals(myXmxVal, opts3.maxMemory) + } finally { + restoreSystemProperty( + COMPILE_DAEMON_JVM_OPTIONS_PROPERTY, + backupJvmOptions + ) + } + } + + fun testDaemonOptionsParsing() { + val backupOptions = System.getProperty(COMPILE_DAEMON_OPTIONS_PROPERTY) + try { + System.setProperty(COMPILE_DAEMON_OPTIONS_PROPERTY, "runFilesPath=abcd,autoshutdownIdleSeconds=1111") + val opts = configureDaemonOptions(DaemonOptions(shutdownDelayMilliseconds = 1)) + assertEquals("abcd", opts.runFilesPath) + assertEquals(1111, opts.autoshutdownIdleSeconds) + } finally { + restoreSystemProperty(COMPILE_DAEMON_OPTIONS_PROPERTY, backupOptions) + } + } + + fun testDaemonInstancesSimple() { + withFlagFile(getTestName(true), ".alive") { flagFile -> + runBlocking { + val daemonOptions = makeTestDaemonOptions(getTestName(true)) + val compilerId2 = CompilerId.makeCompilerId( + compilerClassPath + + File(getCompilerLib(), "kotlin-compiler-sources.jar") + ) + + withLogFile("kotlin-daemon1-test") { logFile1 -> + withLogFile("kotlin-daemon2-test") { logFile2 -> + val daemonJVMOptions1 = makeTestDaemonJvmOptions(logFile1) + val daemonJVMOptions2 = makeTestDaemonJvmOptions(logFile2) + + assertTrue(logFile1.length() == 0L && logFile2.length() == 0L) + + val jar1 = tmpdir.absolutePath + File.separator + "hello1.jar" + val res1 = compileOnDaemon( + flagFile, + compilerId, + daemonJVMOptions1, + daemonOptions, + "-include-runtime", + File(getHelloAppBaseDir(), "hello.kt").absolutePath, + "-d", + jar1 + ).await() + assertEquals("first compilation failed:\n${res1.out}", 0, res1.resultCode) + + logFile1.assertLogContainsSequence("Starting compilation with args: ") + assertEquals("expecting '${logFile2.absolutePath}' to be empty", 0L, logFile2.length()) + + val jar2 = tmpdir.absolutePath + File.separator + "hello2.jar" + val res2 = compileOnDaemon( + flagFile, + compilerId2, + daemonJVMOptions2, + daemonOptions, + "-include-runtime", + File(getHelloAppBaseDir(), "hello.kt").absolutePath, + "-d", + jar2 + ).await() + assertEquals("second compilation failed:\n${res2.out}", 0, res1.resultCode) + + logFile2.assertLogContainsSequence("Starting compilation with args: ") + + kotlinCompilerClientInstance.shutdownCompileService(compilerId, daemonOptions) + kotlinCompilerClientInstance.shutdownCompileService(compilerId2, daemonOptions) + + delay(100) + + logFile1.assertLogContainsSequence("Shutdown started") + logFile2.assertLogContainsSequence("Shutdown started") + } + } + } + } + } + + fun testDaemonRunError() { + withFlagFile(getTestName(true), ".alive") { flagFile -> + runBlocking { + val daemonOptions = + DaemonOptions( + shutdownDelayMilliseconds = 1, + verbose = true, + runFilesPath = File(tmpdir, getTestName(true)).absolutePath + ) + + val daemonJVMOptions = configureDaemonJVMOptions( + "-abracadabra", + inheritMemoryLimits = false, + inheritOtherJvmOptions = false, + inheritAdditionalProperties = false + ) + + val messageCollector = TestMessageCollector() + + val daemon = kotlinCompilerClientInstance.connectToCompileService( + compilerId, flagFile, daemonJVMOptions, daemonOptions, + DaemonReportingTargets(messageCollector = messageCollector), autostart = true + ) + + assertNull(daemon) + + messageCollector.assertHasMessage("Unrecognized option: --abracadabra") + } + } + } + + // TODO: find out how to reliably cause the retry + fun ignore_testDaemonStartRetry() { + withFlagFile(getTestName(true), ".alive") { flagFile -> + runBlocking { + val daemonOptions = + DaemonOptions( + shutdownDelayMilliseconds = 1, + verbose = true, + runFilesPath = File(tmpdir, getTestName(true)).absolutePath + ) + + val daemonJVMOptions = + configureDaemonJVMOptions( + inheritMemoryLimits = false, + inheritOtherJvmOptions = false, + inheritAdditionalProperties = false + ) + + val messageCollector = TestMessageCollector() + + val daemon = kotlinCompilerClientInstance.connectToCompileService( + compilerId, flagFile, daemonJVMOptions, daemonOptions, + DaemonReportingTargets(messageCollector = messageCollector), autostart = true + ) + + assertNull(daemon) + + messageCollector.assertHasMessage("retrying(0) on:") + messageCollector.assertHasMessage("retrying(1) on:") + // TODO: messageCollector.assertHasNoMessage("retrying(2) on:") + messageCollector.assertHasMessage("no more retries on:") + } + } + } + + fun testDaemonAutoshutdownOnUnused() { + withFlagFile(getTestName(true), ".alive") { flagFile -> + runBlocking { + val daemonOptions = DaemonOptions( + autoshutdownUnusedSeconds = 1, + shutdownDelayMilliseconds = 1, + runFilesPath = File(tmpdir, getTestName(true)).absolutePath + ) + + withLogFile("kotlin-daemon-test") { logFile -> + val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) + + val daemon = kotlinCompilerClientInstance.connectToCompileService( + compilerId, + flagFile, + daemonJVMOptions, + daemonOptions, + DaemonReportingTargets(out = System.err), + autostart = true + ) + assertNotNull("failed to connect daemon", daemon) + daemon?.registerClient(flagFile.absolutePath) + + // wait up to 4s (more than 1s unused timeout) + for (attempts in 1..20) { + if (logFile.isLogContainsSequence("Unused timeout exceeded 1s")) break + delay(200) + } + delay(200) + + logFile.assertLogContainsSequence( + "Unused timeout exceeded 1s", + "Shutdown started" + ) + } + } + } + } + + fun testDaemonAutoshutdownOnIdle() { + withFlagFile(getTestName(true), ".alive") { flagFile -> + runBlocking { + val daemonOptions = DaemonOptions( + autoshutdownIdleSeconds = 1, + shutdownDelayMilliseconds = 1, + runFilesPath = File(tmpdir, getTestName(true)).absolutePath + ) + + withLogFile("kotlin-daemon-test") { logFile -> + val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) + + val daemon = kotlinCompilerClientInstance.connectToCompileService( + compilerId, + flagFile, + daemonJVMOptions, + daemonOptions, + DaemonReportingTargets(out = System.err), + autostart = true + ) + assertNotNull("failed to connect daemon", daemon) + daemon?.registerClient(flagFile.absolutePath) + val jar = tmpdir.absolutePath + File.separator + "hello1.jar" + val strm = ByteArrayOutputStream() + val code = kotlinCompilerClientInstance.compile( + daemon!!, CompileService.NO_SESSION, CompileService.TargetPlatform.JVM, + arrayOf("-include-runtime", File(getHelloAppBaseDir(), "hello.kt").absolutePath, "-d", jar), + PrintingMessageCollector(PrintStream(strm), MessageRenderer.WITHOUT_PATHS, true), + reportSeverity = ReportSeverity.DEBUG + ) + assertEquals("compilation failed:\n$strm", 0, code) + + logFile.assertLogContainsSequence("Starting compilation with args: ") + + // wait up to 4s (more than 1s idle timeout) + for (attempts in 1..20) { + if (logFile.isLogContainsSequence("Idle timeout exceeded 1s")) break + delay(200) + } + delay(200) + logFile.assertLogContainsSequence( + "Idle timeout exceeded 1s", + "Shutdown started" + ) + } + } + } + } + + fun testDaemonGracefulShutdown() { + withFlagFile(getTestName(true), ".alive") { flagFile -> + runBlocking { + val daemonOptions = DaemonOptions( + autoshutdownIdleSeconds = 1, + shutdownDelayMilliseconds = 1, + forceShutdownTimeoutMilliseconds = 60000, + runFilesPath = File(tmpdir, getTestName(true)).absolutePath + ) + + withLogFile("kotlin-daemon-test") { logFile -> + val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) + + val daemon = kotlinCompilerClientInstance.connectToCompileService( + compilerId, + flagFile, + daemonJVMOptions, + daemonOptions, + DaemonReportingTargets(out = System.err), + autostart = true + ) + assertNotNull("failed to connect daemon", daemon) + daemon?.registerClient(flagFile.absolutePath) + val sessionId = daemon?.leaseCompileSession(null) + val scheduleShutdownRes = daemon?.scheduleShutdown(true) + + assertTrue( + "failed to schedule shutdown ($scheduleShutdownRes)", + scheduleShutdownRes?.let { it.isGood && it.get() } ?: false) + + delay(100) // to allow timer task to run in the daemon + + logFile.assertLogContainsSequence("Some sessions are active, waiting for them to finish") + + val res = daemon?.leaseCompileSession(null) + + assertEquals("Invalid state", CompileService.CallResult.Dying(), res) + + daemon?.releaseCompileSession(sessionId!!.get()) + + delay(100) // allow after session timed action to run + + logFile.assertLogContainsSequence( + "All sessions finished", + "Shutdown started" + ) + } + } + } + } + + fun testDaemonExitsOnClientFlagDeletedWithActiveSessions() { + runBlocking { + val daemonOptions = DaemonOptions( + autoshutdownIdleSeconds = 1000, + shutdownDelayMilliseconds = 1, + runFilesPath = File(tmpdir, getTestName(true)).absolutePath + ) + val clientFlag = createTempFile(getTestName(true), "-client.alive") + val sessionFlag = createTempFile(getTestName(true), "-session.alive") + try { + withLogFile("kotlin-daemon-test") { logFile -> + val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) + val daemon = kotlinCompilerClientInstance.connectToCompileService( + compilerId, + clientFlag, + daemonJVMOptions, + daemonOptions, + DaemonReportingTargets(out = System.err), + autostart = true + ) + assertNotNull("failed to connect daemon", daemon) + daemon?.leaseCompileSession(sessionFlag.canonicalPath) + + clientFlag.delete() + + delay(2100) // allow deleted file detection, should be 2 * DAEMON_PERIODIC_CHECK_INTERVAL_MS + o + // TODO: consider possibility to set DAEMON_PERIODIC_CHECK_INTERVAL_MS from tests, to allow shorter sleeps + + logFile.assertLogContainsSequence( + "No more clients left", + "Shutdown started" + ) + } + } finally { + sessionFlag.delete() + clientFlag.delete() + } + } + } + + fun testDaemonExitsOnClientFlagDeletedWithAllSessionsReleased() { + runBlocking { + val daemonOptions = DaemonOptions( + autoshutdownIdleSeconds = 1000, + shutdownDelayMilliseconds = 1, + runFilesPath = File(tmpdir, getTestName(true)).absolutePath + ) + val clientFlag = createTempFile(getTestName(true), "-client.alive") + val sessionFlag = createTempFile(getTestName(true), "-session.alive") + try { + withLogFile("kotlin-daemon-test") { logFile -> + val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) + val daemon = kotlinCompilerClientInstance.connectToCompileService( + compilerId, + clientFlag, + daemonJVMOptions, + daemonOptions, + DaemonReportingTargets(out = System.err), + autostart = true + ) + assertNotNull("failed to connect daemon", daemon) + daemon?.leaseCompileSession(sessionFlag.canonicalPath) + + sessionFlag.delete() + + delay(2100) // allow deleted file detection, should be 2 * DAEMON_PERIODIC_CHECK_INTERVAL_MS + o + + clientFlag.delete() + + delay(2100) // allow deleted file detection, should be 2 * DAEMON_PERIODIC_CHECK_INTERVAL_MS + o + + logFile.assertLogContainsSequence( + "No more clients left", + "Shutdown started" + ) + } + } finally { + sessionFlag.delete() + clientFlag.delete() + } + } + } + + fun testDaemonCancelShutdownOnANewClient() { + runBlocking { + val daemonOptions = DaemonOptions( + autoshutdownIdleSeconds = 1000, + shutdownDelayMilliseconds = 3000, + runFilesPath = File(tmpdir, getTestName(true)).absolutePath + ) + val clientFlag = createTempFile(getTestName(true), "-client.alive") + val clientFlag2 = createTempFile(getTestName(true), "-client.alive") + try { + withLogFile("kotlin-daemon-test") { logFile -> + val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) + val daemon = kotlinCompilerClientInstance.connectToCompileService( + compilerId, + clientFlag, + daemonJVMOptions, + daemonOptions, + DaemonReportingTargets(out = System.err), + autostart = true + ) + assertNotNull("failed to connect daemon", daemon) + daemon?.leaseCompileSession(null) + + clientFlag.delete() + + delay(2100) // allow deleted file detection, should be 2 * DAEMON_PERIODIC_CHECK_INTERVAL_MS + o + // TODO: consider possibility to set DAEMON_PERIODIC_CHECK_INTERVAL_MS from tests, to allow shorter sleeps + + val daemon2 = kotlinCompilerClientInstance.connectToCompileService( + compilerId, + clientFlag2, + daemonJVMOptions, + daemonOptions, + DaemonReportingTargets(out = System.err), + autostart = true + ) + assertNotNull("failed to connect daemon", daemon2) + daemon2?.leaseCompileSession(null) + + delay(3000) // allow to trigger delayed shutdown timer + + logFile.assertLogContainsSequence( + "No more clients left", + "Delayed shutdown in 3000ms", + "Cancel delayed shutdown due to a new activity" + ) + } + } finally { + clientFlag.delete() + clientFlag2.delete() + } + } + } + + /** Testing that running daemon in the child process doesn't block on s child process.waitFor() + * that may happen on windows if simple processBuilder.start is used due to handles inheritance: + * - process A starts process B using ProcessBuilder and waits for it using process.waitFor() + * - process B starts daemon and exits + * - due to default behavior of CreateProcess on windows, the handles of process B are inherited by the daemon + * (in particular handles of stdin/out/err) and therefore these handles remain open while daemon is running + * - (seems) due to the way how waiting for process is implemented, waitFor() hangs until daemon is killed + * This seems a known problem, e.g. gradle uses a library with native code that prevents io handles inheritance when launching it's daemon + * (the same solution is used in kotlin daemon client - see next commit) + */ + fun testDaemonExecutionViaIntermediateProcess() { + val clientAliveFile = createTempFile("kotlin-daemon-transitive-run-test", ".run") + val daemonOptions = makeTestDaemonOptions(getTestName(true)) + val jar = tmpdir.absolutePath + File.separator + "hello.jar" + val args = listOf( + File(File(System.getProperty("java.home"), "bin"), "java").absolutePath, + "-Xmx256m", + "-D$COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY", + "-cp", + daemonClientClassPath.joinToString(File.pathSeparator) { it.absolutePath }, + KotlinCompilerClientInstance::class.qualifiedName!! + ) + + daemonOptions.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } + + compilerId.mappers.flatMap { it.toArgs(COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX) } + + File(getHelloAppBaseDir(), "hello.kt").absolutePath + + "-d" + jar + + KotlinCompilerClientInstance.SOCKETS_FLAG + try { + var resOutput: String? = null + var resCode: Int? = null + // running intermediate process (daemon command line controller) that executes the daemon + val runnerProcess = ProcessBuilder(args).redirectErrorStream(true).start() + thread { + resOutput = runnerProcess.inputStream.reader().readText() + } + val waitThread = thread { + resCode = runnerProcess.waitFor() + } + waitThread.join(TIMEOUT_DAEMON_RUNNER_EXIT_MS) + + assertFalse("process.waitFor() hangs:\n$resOutput", waitThread.isAlive) + assertEquals("Compilation failed:\n$resOutput", 0, resCode) + println("OK") + } finally { + if (clientAliveFile.exists()) + clientAliveFile.delete() + } + } + + private val PARALLEL_THREADS_TO_COMPILE = 10 + private val PARALLEL_WAIT_TIMEOUT_S = 60L + + private fun runCompile( + daemon: CompileServiceAsync, + resultCodes: Array, + localEndSignal: CountDownLatch, + outStreams: Array, + threadNo: Int + ) = GlobalScope.async(newSingleThreadContext(name = "tread$threadNo")) { + println("thread : ${Thread.currentThread().name}") + val jar = tmpdir.absolutePath + File.separator + "hello.$threadNo.jar" + val res = kotlinCompilerClientInstance.compile( + daemon, + CompileService.NO_SESSION, + CompileService.TargetPlatform.JVM, + arrayOf("-include-runtime", File(getHelloAppBaseDir(), "hello.kt").absolutePath, "-d", jar), + PrintingMessageCollector(PrintStream(outStreams[threadNo]), MessageRenderer.WITHOUT_PATHS, true) + ) + synchronized(resultCodes) { + resultCodes[threadNo] = res + } + localEndSignal.countDown() + } + + fun testParallelCompilationOnDaemon() { + + assertTrue(PARALLEL_THREADS_TO_COMPILE <= LoopbackNetworkInterface.SERVER_SOCKET_BACKLOG_SIZE) + + withFlagFile(getTestName(true), ".alive") { flagFile -> + runBlocking { + val daemonOptions = makeTestDaemonOptions(getTestName(true)) + + withLogFile("kotlin-daemon-test") { logFile -> + + + // val cfg: String = +// "handlers = java.util.logging.FileHandler\n" + +// "java.util.logging.FileHandler.level = ALL\n" + +// "java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter\n" + +// "java.util.logging.FileHandler.encoding = UTF-8\n" + +// "java.util.logging.FileHandler.limit = 0\n" + // if file is provided - disabled, else - 1Mb +// "java.util.logging.FileHandler.count = 1\n" + +// "java.util.logging.FileHandler.append = true\n" + +// "java.util.logging.FileHandler.pattern = ${logFile.loggerCompatiblePath}\n" + +// "java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s%n\n" +// LogManager.getLogManager().readConfiguration(cfg.byteInputStream()) + + + val daemonJVMOptions = makeTestDaemonJvmOptions(logFile, xmx = -1) + + val daemon = kotlinCompilerClientInstance.connectToCompileService( + compilerId, + flagFile, + daemonJVMOptions, + daemonOptions, + DaemonReportingTargets(out = System.err), + autostart = true + ) + assertNotNull("failed to connect daemon", daemon) + TestCase.assertTrue("daemon is not new!", daemon !is CompileService) + + val resultCodes = arrayOfNulls(PARALLEL_THREADS_TO_COMPILE) + val localEndSignal = CountDownLatch(PARALLEL_THREADS_TO_COMPILE) + val outStreams = Array(PARALLEL_THREADS_TO_COMPILE, { ByteArrayOutputStream() }) + + val time = nowSeconds() + + (1..PARALLEL_THREADS_TO_COMPILE).forEach { + runCompile( + daemon!!, + resultCodes, + localEndSignal, + outStreams, + it - 1 + ) + } + + val succeeded = localEndSignal.await(PARALLEL_WAIT_TIMEOUT_S, TimeUnit.SECONDS) + + println("finished in ${nowSeconds() - time} seconds") + + assertTrue( + "parallel compilation failed to complete in $PARALLEL_WAIT_TIMEOUT_S s, ${localEndSignal.count} unfinished threads", + succeeded + ) + + (1..PARALLEL_THREADS_TO_COMPILE).forEach { + assertEquals("Compilation on thread $it failed:\n${outStreams[it - 1]}", 0, resultCodes[it - 1]) + println("result[$it] = ${resultCodes[it - 1]}") + } + } + } + } + } + + private object ParallelStartParams { + const val threads = 10 + const val performCompilation = false + const val connectionFailedErr = -100 + } + + fun testParallelDaemonStart() { + + val doneLatch = CountDownLatch(ParallelStartParams.threads) + + val resultCodes = arrayOfNulls(ParallelStartParams.threads) + val outStreams = Array(ParallelStartParams.threads, { ByteArrayOutputStream() }) + val logFiles = arrayOfNulls(ParallelStartParams.threads) + val daemonInfos = arrayOfNulls?, Int?>>(ParallelStartParams.threads) + + val daemonOptions = makeTestDaemonOptions(getTestName(true), 100) + + fun connectThread(threadNo: Int) = GlobalScope.async(newSingleThreadContext(name = "daemonConnect$threadNo")) { + // (name = "daemonConnect$threadNo") + try { + withFlagFile(getTestName(true), ".alive") { flagFile -> + withLogFile( + "kotlin-daemon-test", + printLogOnException = false + ) { logFile -> + logFiles[threadNo] = logFile + val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) +// println("-1 I'm working in thread ${Thread.currentThread().name}") + val compileServiceSession = + kotlinCompilerClientInstance.connectAndLease( + compilerId, flagFile, daemonJVMOptions, daemonOptions, + DaemonReportingTargets(out = PrintStream(outStreams[threadNo])), autostart = true, + leaseSession = true + ) +// println("0 I'm working in thread ${Thread.currentThread().name}") + daemonInfos[threadNo] = try { + compileServiceSession?.compileService?.getDaemonInfo() + } catch (e: ClosedChannelException) { + null + } to compileServiceSession?.sessionId + + resultCodes[threadNo] = when { + compileServiceSession?.compileService == null -> { + println("[$threadNo] not-compile!") + ParallelStartParams.connectionFailedErr + } + ParallelStartParams.performCompilation -> { + println("[$threadNo] compile!") + val jar = tmpdir.absolutePath + File.separator + "hello.$threadNo.jar" + kotlinCompilerClientInstance.compile( + compileServiceSession.compileService, + compileServiceSession.sessionId, + CompileService.TargetPlatform.JVM, + arrayOf(File(getHelloAppBaseDir(), "hello.kt").absolutePath, "-d", jar), + PrintingMessageCollector(PrintStream(outStreams[threadNo]), MessageRenderer.WITHOUT_PATHS, true) + ) + } + else -> { + println("[$threadNo] compilation skipped, assuming - successful!") + 0 // compilation skipped, assuming - successful + } + } + } + } + } finally { + doneLatch.countDown() + } + + } + + runBlocking { + System.setProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY, "true") + System.setProperty(COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY, "100000") + + val succeeded = try { + (1..ParallelStartParams.threads).forEach { connectThread(it - 1) } + doneLatch.await(PARALLEL_WAIT_TIMEOUT_S, TimeUnit.SECONDS) + } finally { + System.clearProperty(COMPILE_DAEMON_STARTUP_TIMEOUT_PROPERTY) + System.clearProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) + } + + delay(1000) // Wait for processes to finish and close log files + + val electionLogs = arrayOfNulls(ParallelStartParams.threads) + val port2logs = arrayOfNulls>(ParallelStartParams.threads) + + for (i in 0..(ParallelStartParams.threads - 1)) { + val logContents = logFiles[i]?.readLines() + port2logs[i] = logContents?.find { it.contains("daemon is listening on port") }?.split(" ")?.last()?.toIntOrNull() to + logFiles[i] + electionLogs[i] = logContents?.find { it.contains(LOG_PREFIX_ASSUMING_OTHER_DAEMONS_HAVE) } + } + + val electionsSuccess = electionLogs.any { it != null && (it.contains("lower prio") || it.contains("equal prio")) } + val resultsFailures = resultCodes.count { it != null && it == 0 } + + if (!succeeded || !electionsSuccess || resultsFailures > 0) { + val msg = buildString { + for (i in 0..ParallelStartParams.threads - 1) { + val daemonInfoRes = daemonInfos[i]?.first + val daemonInfo = when (daemonInfoRes) { + is CompileService.CallResult.Good -> daemonInfoRes.get() + is CompileService.CallResult.Dying -> "" + is CompileService.CallResult.Error -> "" + else -> "?" + } + val compiledPort: Int? = daemonInfo.trim().split(" ").last().toIntOrNull() + appendln("#$i\tcompiled on $daemonInfo, session ${daemonInfos[i]?.second}, result ${resultCodes[i]}; started daemon on port ${port2logs[i]?.first}, log: ${logFiles[i]?.canonicalPath}") + if (resultCodes[i] != 0 || electionLogs[i] == null) { + appendln("--- out $i, result ${resultCodes[i]}:\n${outStreams[i].toByteArray().toString(Charset.defaultCharset())}\n---") + compiledPort?.let { port -> port2logs.find { it?.first == port } }?.second?.let { logFile -> + appendln("--- log file ${logFile.name}:\n${logFile.readText()}\n---") + } + ?: appendln("--- log not found (port: $compiledPort)") + } + } + } + assertTrue( + "parallel daemons start failed to complete in $PARALLEL_WAIT_TIMEOUT_S s, ${doneLatch.count} unfinished threads:\n\n$msg", + succeeded + ) + assertTrue("No daemon elected:\n\n$msg\n--- elections:\n${electionLogs.joinToString("\n")}\n---", electionsSuccess) + assertTrue("Compilations failed: $resultsFailures of ${ParallelStartParams.threads}:\n\n$msg", resultsFailures > 0) + println("test passed (elected : $electionsSuccess, resultsFailures : $resultsFailures)") + } + } + } + + fun testDaemonConnectionProblems() { + withFlagFile(getTestName(true), ".alive") { flagFile -> + runBlocking { + val daemonOptions = makeTestDaemonOptions(getTestName(true)) + val daemonJVMOptions = makeTestDaemonJvmOptions() + val daemon = kotlinCompilerClientInstance.connectToCompileService( + compilerId, + flagFile, + daemonJVMOptions, + daemonOptions, + DaemonReportingTargets(out = System.err), + autostart = true + ) + assertNotNull("failed to connect daemon", daemon) + daemon?.registerClient(flagFile.absolutePath) + kotlinCompilerClientInstance.shutdownCompileService(compilerId, daemonOptions) + delay(2000L) + val exception: Exception? = try { + daemon!!.getUsedMemory() + null + } catch (e: ConnectException) { + e + } catch (e: UnmarshalException) { + e + } catch (e: ConnectIOException) { + e + } catch (e: IOException) { + e + } + + println("${(exception ?: Exception())::class.java.simpleName} : ${exception?.message}") + assertNotNull(exception) + } + } + } + + // TODO: fix this test + fun ignore_testDaemonCallbackConnectionProblems() { + withFlagFile(getTestName(true), ".alive") { flagFile -> + runBlocking { + val daemonOptions = makeTestDaemonOptions(getTestName(true)) + + withLogFile("kotlin-daemon-test") { logFile -> + val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) + + val daemon: CompileServiceAsync? = kotlinCompilerClientInstance.connectToCompileService( + compilerId, + flagFile, + daemonJVMOptions, + daemonOptions, + DaemonReportingTargets(out = System.err), + autostart = true + ) + assertNotNull("failed to connect daemon", daemon) + daemon?.registerClient(flagFile.absolutePath) + + val file = File(tmpdir, "largeKotlinFile.kt") + file.writeText(generateLargeKotlinFile(10)) + val jar = File(tmpdir, "largeKotlinFile.jar").absolutePath + + val strm = ByteArrayOutputStream() + var callbackServices: CompilerCallbackServicesFacadeServerServerSide? = null + callbackServices = CompilerCallbackServicesFacadeServerServerSide( + compilationCanceledStatus = object : CompilationCanceledStatus { + override suspend fun checkCanceled() { + thread { + Thread.sleep(10) + callbackServices!!.shutdownServer() + } + } + }, + messageCollector = PrintingMessageCollector(PrintStream(strm), MessageRenderer.WITHOUT_PATHS, true), + serverSocketWithPort = findCallbackServerSocket() + ) + callbackServices.runServer() + PrintingMessageCollector(PrintStream(strm), MessageRenderer.WITHOUT_PATHS, true) + val code = daemon!!.compile( + CompileService.NO_SESSION, + arrayOf("-include-runtime", file.absolutePath, "-d", jar), + CompilationOptions( + CompilerMode.JPS_COMPILER, + CompileService.TargetPlatform.JVM, + arrayOf( + ReportCategory.COMPILER_MESSAGE.code, + ReportCategory.DAEMON_MESSAGE.code, + ReportCategory.EXCEPTION.code, + ReportCategory.OUTPUT_MESSAGE.code + ), + ReportSeverity.DEBUG.code, + emptyArray() + ), + callbackServices.clientSide, + kotlinCompilerClientInstance.createCompResults().clientSide + ).get() + + TestCase.assertEquals(0, code) + + val compilerOutput = strm.toString() + assertTrue("Expecting cancellation message in:\n$compilerOutput", compilerOutput.contains("Compilation was canceled")) + logFile.assertLogContainsSequence("error communicating with host, assuming compilation canceled") + + } + } + } + } + + fun testDaemonReplScriptingNotInClasspathError() { + withDaemon(compilerId) { daemon -> + var repl: KotlinRemoteReplCompilerClientAsync? = null + var isErrorThrown = false + try { + repl = KotlinRemoteReplCompilerClientAsync( + daemon, null, CompileService.TargetPlatform.JVM, emptyArray(), TestMessageCollector(), + classpathFromClassloader(), ScriptWithNoParam::class.qualifiedName!! + ) + } catch (e: Exception) { + TestCase.assertEquals( + "Unable to use scripting/REPL in the daemon, no kotlin-scripting-compiler.jar or its dependencies are found in the compiler classpath", + e.message + ) + isErrorThrown = true + } finally { + repl?.dispose() + } + TestCase.assertTrue("Expecting exception that kotlin-scripting-plugin is not found in the classpath", isErrorThrown) + } + } + + fun testDaemonReplLocalEvalNoParams() { + withDaemon(compilerWithScriptingId) { daemon -> + val repl = KotlinRemoteReplCompilerClientAsync( + daemon, null, CompileService.TargetPlatform.JVM, + emptyArray(), + TestMessageCollector(), + classpathFromClassloader(), + ScriptWithNoParam::class.qualifiedName!! + ) + println("repl = $repl") + println("sessionId : ${repl.sessionId}") + + val localEvaluator = GenericReplEvaluator(emptyList(), Thread.currentThread().contextClassLoader) + + doReplTestWithLocalEval(repl, localEvaluator) + repl.dispose() + } + } + + // TODO: fix "Front-end Internal error: Failed to analyze declaration Line_1" + fun testDaemonReplLocalEvalStandardTemplate() { + withDaemon(compilerWithScriptingId) { daemon -> + val repl = KotlinRemoteReplCompilerClientAsync( + daemon, null, CompileService.TargetPlatform.JVM, emptyArray(), + TestMessageCollector(), + classpathFromClassloader(), + "kotlin.script.templates.standard.ScriptTemplateWithArgs" + ) + + val localEvaluator = GenericReplEvaluator( + emptyList(), Thread.currentThread().contextClassLoader, + ScriptArgsWithTypes(arrayOf(emptyArray()), arrayOf(Array::class)) + ) + + doReplTestWithLocalEval(repl, localEvaluator) + repl.dispose() + } + } + + fun testDaemonReplLocalEvalStandardTemplate_OldDaemon_NewClient() { + withOldDaemon { daemon -> + val repl = KotlinRemoteReplCompilerClientAsync( + daemon, null, CompileService.TargetPlatform.JVM, emptyArray(), + TestMessageCollector(), + classpathFromClassloader(), + "kotlin.script.templates.standard.ScriptTemplateWithArgs" + ) + + val localEvaluator = GenericReplEvaluator( + emptyList(), Thread.currentThread().contextClassLoader, + ScriptArgsWithTypes(arrayOf(emptyArray()), arrayOf(Array::class)) + ) + + doReplTestWithLocalEval(repl, localEvaluator) + repl.dispose() + } + } + + private suspend fun doReplTestWithLocalEval(replCompiler: KotlinRemoteReplCompilerClientAsync, localEvaluator: ReplEvaluator) { + println("doReplTestWithLocalEval...") + val compilerState = replCompiler.createState() + println("compilerState = $compilerState") + val evaluatorState = localEvaluator.createState() + println("evaluatorState = $evaluatorState") + + val res0 = replCompiler.check(compilerState, ReplCodeLine(0, 0, "val x =")) + TestCase.assertTrue("Unexpected check results: $res0", res0 is ReplCheckResult.Incomplete) + + val codeLine1 = ReplCodeLine(1, 0, "val lst = listOf(1)\nval x = 5") + val res1 = replCompiler.compile(compilerState, codeLine1) + val res1c = res1 as? ReplCompileResult.CompiledClasses + TestCase.assertNotNull("Unexpected compile result: $res1", res1c) + + val res11 = localEvaluator.eval(evaluatorState, res1c!!) + val res11e = res11 as? ReplEvalResult.UnitResult + TestCase.assertNotNull("Unexpected eval result: $res11", res11e) + + val codeLine2 = ReplCodeLine(2, 0, "x + 2") + val res2 = replCompiler.compile(compilerState, codeLine2) + val res2c = res2 as? ReplCompileResult.CompiledClasses + TestCase.assertNotNull("Unexpected compile result: $res2", res2c) + + val res21 = localEvaluator.eval(evaluatorState, res2c!!) + val res21e = res21 as? ReplEvalResult.ValueResult + TestCase.assertNotNull("Unexpected eval result: $res21", res21e) + TestCase.assertEquals(7, res21e!!.value) + } + + fun testDaemonReplAutoshutdownOnIdle() { + withFlagFile(getTestName(true), ".alive") { flagFile -> + runBlocking { + val daemonOptions = DaemonOptions( + autoshutdownIdleSeconds = 1, + autoshutdownUnusedSeconds = 1, + shutdownDelayMilliseconds = 1, + runFilesPath = File(tmpdir, getTestName(true)).absolutePath + ) + + withLogFile("kotlin-daemon-test") { logFile -> + val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) + + val daemon = kotlinCompilerClientInstance.connectToCompileService( + compilerWithScriptingId, + flagFile, + daemonJVMOptions, + daemonOptions, + DaemonReportingTargets(out = System.err), + autostart = true + ) + + println("daemon : $daemon") + assertNotNull("failed to connect daemon", daemon) + + val replCompiler = KotlinRemoteReplCompilerClientAsync( + daemon!!, null, CompileService.TargetPlatform.JVM, + emptyArray(), + TestMessageCollector(), + classpathFromClassloader(), + ScriptWithNoParam::class.qualifiedName!! + ) + + val compilerState = replCompiler.createState() + + // use repl compiler for >> 1s, making sure that idle/unused timeouts are not firing + for (attempts in 1..10) { + val codeLine1 = ReplCodeLine(attempts, 0, "3 + 5") + val res1 = replCompiler.compile(compilerState, codeLine1) + val res1c = res1 as? ReplCompileResult.CompiledClasses + TestCase.assertNotNull("Unexpected compile result: $res1", res1c) + delay(500) + } + + // wait up to 4s (more than 1s idle timeout) + for (attempts in 1..20) { + if (logFile.isLogContainsSequence("Idle timeout exceeded 1s")) break + delay(500) + } + replCompiler.dispose() + + delay(500) + logFile.assertLogContainsSequence( + "Idle timeout exceeded 1s", + "Shutdown started" + ) + } + } + } + } + + internal fun withDaemon(compilerId: CompilerId = this.compilerId, body: suspend (CompileServiceAsync) -> Unit) { + withFlagFile(getTestName(true), ".alive") { flagFile -> + val daemonOptions = makeTestDaemonOptions(getTestName(true)) + withLogFile("kotlin-daemon-test") { logFile -> + runBlocking { + val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) + val daemon: CompileServiceAsync? = kotlinCompilerClientInstance.connectToCompileService( + compilerId, + flagFile, + daemonJVMOptions, + daemonOptions, + DaemonReportingTargets(out = System.err), + autostart = true + ) + println("daemon : $daemon") + assertNotNull("failed to connect daemon", daemon) + assertTrue("daemon is not New", daemon !is CompileServiceAsyncWrapper) + + body(daemon!!) + } + } + } + } + + internal fun withOldDaemon(body: suspend (CompileServiceAsync) -> Unit) { + withFlagFile(getTestName(true), ".alive") { flagFile -> + val daemonOptions = makeTestDaemonOptions(getTestName(true)) + withLogFile("kotlin-daemon-test") { logFile -> + runBlocking { + val daemonJVMOptions = makeTestDaemonJvmOptions(logFile) + runOldServer(daemonOptions, daemonJVMOptions) + val daemon: CompileServiceAsync? = kotlinCompilerClientInstance.connectToCompileService( + compilerId, + flagFile, + daemonJVMOptions, + daemonOptions, + DaemonReportingTargets(out = System.err), + autostart = true + ) + println("daemon : $daemon, port : ${daemon?.serverPort}") + TestCase.assertTrue(daemon is CompileServiceAsyncWrapper) + assertNotNull("failed to connect daemon", daemon) + + body(daemon!!) + } + } + } + } + + private fun runOldServer( + daemonOptions: DaemonOptions, + daemonJVMOptions: DaemonJVMOptions, + timer: Timer = Timer(), + onShutdown: () -> Unit = {} + ) { + val (registry, serverPort) = findPortAndCreateRegistry( + COMPILE_DAEMON_FIND_PORT_ATTEMPTS, + COMPILE_DAEMON_PORTS_RANGE_START, + COMPILE_DAEMON_PORTS_RANGE_END + ) + val compilerSelector = object : CompilerSelector { + private val jvm by lazy { K2JVMCompiler() } + private val js by lazy { K2JSCompiler() } + private val metadata by lazy { K2MetadataCompiler() } + override fun get(targetPlatform: CompileService.TargetPlatform): CLICompiler<*> = when (targetPlatform) { + CompileService.TargetPlatform.JVM -> jvm + CompileService.TargetPlatform.JS -> js + CompileService.TargetPlatform.METADATA -> metadata + } + } + CompileServiceImpl( + registry = registry, + compiler = compilerSelector, + compilerId = compilerId, + daemonOptions = daemonOptions, + daemonJVMOptions = daemonJVMOptions, + port = serverPort, + timer = timer, + onShutdown = onShutdown + ).startDaemonLife() + println("old daemon init: port = $serverPort") + } +} + + +// stolen from CompilerFileLimitTest +internal fun generateLargeKotlinFile(size: Int): String { + return buildString { + append("package large\n\n") + (0..size).forEach { + appendln("class Class$it") + appendln("{") + appendln("\tfun foo(): Long = $it") + appendln("}") + appendln("\n") + repeat(2000) { + appendln("// kotlin rules ... and stuff") + } + } + appendln("fun main(args: Array)") + appendln("{") + appendln("\tval result = Class5().foo() + Class$size().foo()") + appendln("\tprintln(result)") + appendln("}") + } + +} + + +internal fun File.ifLogNotContainsSequence(vararg patterns: String, body: (LinePattern, Int) -> Unit) { + reader().useLines { + it.asSequence().ifNotContainsSequence(patterns.map { LinePattern(it) }, body) + } +} + +internal fun File.assertLogContainsSequence(vararg patterns: String) = assertLogContainsSequence(patterns.map { LinePattern(it) }) + +internal fun File.assertLogContainsSequence(vararg patterns: LinePattern) = assertLogContainsSequence(patterns.asIterable()) + +internal fun File.assertLogContainsSequence(patterns: Iterable) { + val lines = reader().readLines() + lines.asSequence().ifNotContainsSequence(patterns.iterator()) + { unmatchedPattern, lineNo -> + fail( + "pattern not found in the input: ${unmatchedPattern.regex}\nunmatched part of the log file ($absolutePath) from line $lineNo:\n\n${lines.asSequence().drop( + lineNo + ).joinToString("\n")}\n-------" + ) + } +} + +internal fun File.isLogContainsSequence(vararg patterns: String): Boolean { + var res = true + ifLogNotContainsSequence(*patterns) { _, _ -> res = false } + return res +} + +fun restoreSystemProperty(propertyName: String, backupValue: String?) { + if (backupValue == null) { + System.clearProperty(propertyName) + } else { + System.setProperty(propertyName, backupValue) + } +} + +internal inline fun withFlagFile(prefix: String, suffix: String? = null, body: (File) -> Unit) { + val file = createTempFile(prefix, suffix) + try { + body(file) + } finally { + file.delete() + } +} + +internal inline fun withLogFile(prefix: String, suffix: String = ".log", printLogOnException: Boolean = true, body: (File) -> Unit) { + val logFile = createTempFile(prefix, suffix) + println("LOG FILE : ${logFile.path}") + try { + body(logFile) + } catch (e: Exception) { + if (printLogOnException) { + Thread.sleep(100) // trying to wait log flushing + System.err.println("--- log file ${logFile.name}:\n${logFile.readText()}\n---") + } + throw e + } +} + +// java.util.Logger used in the daemon silently forgets to log into a file specified in the config on Windows, +// if file path is given in windows form (using backslash as a separator); the reason is unknown +// this function makes a path with forward slashed, that works on windows too +internal val File.loggerCompatiblePath: String + get() = + if (OSKind.current == OSKind.Windows) absolutePath.replace('\\', '/') + else absolutePath + +open class TestKotlinScriptDummyDependenciesResolver : DependenciesResolver { + + override fun resolve(scriptContents: ScriptContents, environment: Environment): ResolveResult { + return ScriptDependencies( + classpath = classpathFromClassloader(), + imports = listOf("org.jetbrains.kotlin.scripts.DependsOn", "org.jetbrains.kotlin.scripts.DependsOnTwo") + ).asSuccess() + } +} + +@ScriptTemplateDefinition(resolver = TestKotlinScriptDummyDependenciesResolver::class) +abstract class ScriptWithNoParam + +internal fun classpathFromClassloader(): List { + val additionalClasspath = System.getProperty("kotlin.test.script.classpath")?.split(File.pathSeparator) + ?.map { File(it) }.orEmpty() + return ((TestKotlinScriptDummyDependenciesResolver::class.java.classLoader as? URLClassLoader)?.urLs + ?.mapNotNull(URL::toFile) + ?.filter { it.path.contains("out") && it.path.contains("") } + ?: emptyList() + ) + additionalClasspath +} + +internal fun URL.toFile() = + try { + File(toURI().schemeSpecificPart) + } catch (e: java.net.URISyntaxException) { + if (protocol != "file") null + else File(file) + } + diff --git a/compiler/tests/org/jetbrains/kotlin/daemon/experimental/unit/ClientSerializationTest.kt b/compiler/tests/org/jetbrains/kotlin/daemon/experimental/unit/ClientSerializationTest.kt new file mode 100644 index 00000000000..e7f51db1314 --- /dev/null +++ b/compiler/tests/org/jetbrains/kotlin/daemon/experimental/unit/ClientSerializationTest.kt @@ -0,0 +1,91 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.experimental.unit + +import io.ktor.network.selector.ActorSelectorManager +import io.ktor.network.selector.SelectorManager +import io.ktor.network.sockets.aSocket +import kotlinx.coroutines.* +import org.jetbrains.kotlin.daemon.common.experimental.CompilerServicesFacadeBaseClientSideImpl +import org.jetbrains.kotlin.daemon.common.experimental.LoopbackNetworkInterface.selectorMgr +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.DefaultClient +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.ServerBase +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.openIO +import org.jetbrains.kotlin.daemon.common.experimental.socketInfrastructure.runWithTimeout +import org.jetbrains.kotlin.daemon.common.toRMI +import org.jetbrains.kotlin.integration.KotlinIntegrationTestBase +import java.io.ObjectInputStream +import java.io.ObjectOutputStream +import java.net.InetSocketAddress +import java.util.logging.Logger + +class TestServer(val serverPort: Int = 6999) { + private val serverSocket = aSocket(selectorMgr).tcp().bind(InetSocketAddress(serverPort)) + private val log = Logger.getLogger("TestServer") + + fun awaitClient() = GlobalScope.async { + log.info("accepting clientSocket...") + val client = serverSocket.accept() + log.info("client accepted! (${client.remoteAddress})") + val (input, output) = client.openIO(log) + + true + } +} + +val testServer = TestServer() + +@Suppress("UNCHECKED_CAST") +class ClientSerializationTest : KotlinIntegrationTestBase() { + + val file = createTempFile() + + val log = Logger.getLogger("ClientSerializationTest") + + private inline fun abstractSerializationTest(initClient: () -> T, vararg additionalTests: (T, T) -> Unit) { + val client = initClient() + log.info("created") + file.outputStream().use { + ObjectOutputStream(it).use { + it.writeObject(client) + } + } + log.info("printed") + var client2: T? = null + var connected = false + runBlocking { + val clientAwait = testServer.awaitClient() + client2 = file.inputStream().use { + ObjectInputStream(it).use { + it.readObject() as T + } + } + connected = runWithTimeout { clientAwait.await() } ?: false + } + assert(connected) + log.info("read") + assert(client2 != null) + additionalTests.forEach { it(client, client2!!) } + log.info("test passed") + } + + fun testDefaultClient() = abstractSerializationTest( + { DefaultClient(testServer.serverPort) }, + { client, client2 -> assert(client.serverPort == client2.serverPort) }, + { client, client2 -> + client2.log.info("abacaba (2)") + log.info("test passed") + } + ) + + fun testCompilerServicesFacadeBaseClientSide() = abstractSerializationTest( + { CompilerServicesFacadeBaseClientSideImpl(testServer.serverPort) }, + { client, client2 -> assert(client.serverPort == client2.serverPort) } + ) + + fun testRMIWrapper() = abstractSerializationTest({ CompilerServicesFacadeBaseClientSideImpl(testServer.serverPort).toRMI() }) + +} \ No newline at end of file diff --git a/compiler/tests/org/jetbrains/kotlin/daemon/experimental/unit/ConnectionsTest.kt b/compiler/tests/org/jetbrains/kotlin/daemon/experimental/unit/ConnectionsTest.kt new file mode 100644 index 00000000000..eb1d62711a4 --- /dev/null +++ b/compiler/tests/org/jetbrains/kotlin/daemon/experimental/unit/ConnectionsTest.kt @@ -0,0 +1,380 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.daemon.experimental.unit + +import kotlinx.coroutines.* +import org.jetbrains.kotlin.cli.common.CLICompiler +import org.jetbrains.kotlin.cli.common.messages.MessageRenderer +import org.jetbrains.kotlin.cli.common.messages.PrintingMessageCollector +import org.jetbrains.kotlin.cli.js.K2JSCompiler +import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler +import org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler +import org.jetbrains.kotlin.daemon.CompileServiceImpl +import org.jetbrains.kotlin.daemon.CompilerSelector +import org.jetbrains.kotlin.daemon.client.KotlinCompilerDaemonClient +import org.jetbrains.kotlin.daemon.client.experimental.BasicCompilerServicesWithResultsFacadeServerServerSide +import org.jetbrains.kotlin.daemon.common.* +import org.jetbrains.kotlin.daemon.common.experimental.* +import org.jetbrains.kotlin.daemon.experimental.CompileServiceServerSideImpl +import org.jetbrains.kotlin.daemon.loggerCompatiblePath +import org.jetbrains.kotlin.integration.KotlinIntegrationTestBase +import org.jetbrains.kotlin.test.KotlinTestUtils +import java.io.ByteArrayOutputStream +import java.io.File +import java.io.IOException +import java.io.PrintStream +import java.util.* +import java.util.logging.LogManager +import java.util.logging.Logger +import kotlin.concurrent.schedule + +class ConnectionsTest : KotlinIntegrationTestBase() { + + val kotlinCompilerClient = KotlinCompilerDaemonClient.instantiate(DaemonProtocolVariant.SOCKETS) + + private val logFile = createTempFile("/Users/jetbrains/Documents/kotlin/my_fork/kotlin", ".txt") + + private val cfg = "handlers = java.util.logging.FileHandler\n" + + "java.util.logging.FileHandler.level = ALL\n" + + "java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter\n" + + "java.util.logging.FileHandler.encoding = UTF-8\n" + + "java.util.logging.FileHandler.limit = 0\n" + // if file is provided - disabled, else - 1Mb + "java.util.logging.FileHandler.count = 1\n" + + "java.util.logging.FileHandler.append = true\n" + + "java.util.logging.FileHandler.pattern = ${logFile.loggerCompatiblePath}\n" + + "java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s%n\n" + + init { + LogManager.getLogManager().readConfiguration(cfg.byteInputStream()) + } + + val log by lazy { Logger.getLogger("ConnectionsTest") } + + private val daemonJVMOptions by lazy { + configureDaemonJVMOptions( + inheritMemoryLimits = true, + inheritOtherJvmOptions = true, + inheritAdditionalProperties = true + ) + } + + private val compilerId by lazy { CompilerId() } + + private val daemonOptions by lazy { DaemonOptions() } + + private val port + get() = findPortForSocket( + COMPILE_DAEMON_FIND_PORT_ATTEMPTS, + COMPILE_DAEMON_PORTS_RANGE_START, + COMPILE_DAEMON_PORTS_RANGE_END + ) + + private val timer by lazy { Timer(true) } + + private val runFile by lazy { + val runFileDir = File(daemonOptions.runFilesPathOrDefault) + runFileDir.mkdirs() + File( + runFileDir, + makeRunFilenameString( + timestamp = "%tFT% Unit = { + if (daemonOptions.forceShutdownTimeoutMilliseconds != COMPILE_DAEMON_TIMEOUT_INFINITE_MS) { + // running a watcher thread that ensures that if the daemon is not exited normally (may be due to RMI leftovers), it's forced to exit + timer.schedule(daemonOptions.forceShutdownTimeoutMilliseconds) { + cancel() + org.jetbrains.kotlin.daemon.KotlinCompileDaemon.log.info("force JVM shutdown") + System.exit(0) + } + } else { + timer.cancel() + } + } + + + private fun getNewDaemonsOrAsyncWrappers() = runBlocking { + walkDaemonsAsync( + File(daemonOptions.runFilesPathOrDefault), + compilerId, + runFile, + filter = { _, _ -> true }, + report = { _, msg -> log.info(msg) }, + useRMI = true, + useSockets = true + ).toList() + } + + private fun getOldDaemonsOrRMIWrappers() = runBlocking { + walkDaemons( + File(daemonOptions.runFilesPathOrDefault), + compilerId, + runFile, + filter = { _, _ -> true }, + report = { _, msg -> log.info(msg) } + ).toList() + } + + companion object { + + } + + private fun runNewServer(): Deferred = + port.let { serverPort -> + CompileServiceServerSideImpl( + serverPort, + compilerId, + daemonOptions, + daemonJVMOptions, + serverPort.port, + timer, + onShutdown + ).let { + log.info("service created") + it.startDaemonLife() + it.runServer() + } + } + + private fun runOldServer() { + val (registry, serverPort) = findPortAndCreateRegistry( + COMPILE_DAEMON_FIND_PORT_ATTEMPTS, + COMPILE_DAEMON_PORTS_RANGE_START, + COMPILE_DAEMON_PORTS_RANGE_END + ) + val compilerSelector = object : CompilerSelector { + private val jvm by lazy { K2JVMCompiler() } + private val js by lazy { K2JSCompiler() } + private val metadata by lazy { K2MetadataCompiler() } + override fun get(targetPlatform: CompileService.TargetPlatform): CLICompiler<*> = when (targetPlatform) { + CompileService.TargetPlatform.JVM -> jvm + CompileService.TargetPlatform.JS -> js + CompileService.TargetPlatform.METADATA -> metadata + } + } + log.info("old server run: (port= $serverPort, reg= $registry)") + CompileServiceImpl( + registry = registry, + compiler = compilerSelector, + compilerId = compilerId, + daemonOptions = daemonOptions, + daemonJVMOptions = daemonJVMOptions, + port = serverPort, + timer = timer, + onShutdown = onShutdown + ).startDaemonLife() + } + + val comparator = compareByDescending( + DaemonJVMOptionsMemoryComparator(), + { it.jvmOptions } + ) + .thenBy { + when (it.daemon) { + is CompileServiceAsyncWrapper -> 0 + else -> 1 + } + } + .thenBy(FileAgeComparator()) { it.runFile } + .thenBy { it.daemon.serverPort } + + private fun expectDaemon( + getDaemons: () -> List, + chooseDaemon: (List) -> Daemon, + getInfo: (Daemon) -> CompileService.CallResult, + registerClient: (Daemon) -> Unit, + port: (Daemon) -> Int, + expectedDaemonCount: Int?, + extraAction: (Daemon) -> Unit = {} + ) { + val daemons = getDaemons() + log.info("daemons (${daemons.size}) : ${daemons.map { (it ?: 0)::class.java.name }.toList()}\n\n") + expectedDaemonCount?.let { + log.info("expected $it daemons, found ${daemons.size}") + assertTrue( + "daemons.size : ${daemons.size}, but expected : $expectedDaemonCount", + daemons.size == it + ) + } + val daemon = chooseDaemon(daemons) + log.info("chosen : $daemon (port = ${port(daemon)})") + val info = getInfo(daemon) + log.info("info : $info") + assertTrue("bad info", info.isGood) + registerClient(daemon) + extraAction(daemon) + } + + private enum class ServerType(val instancesNumber: Int?) { + OLD(1), NEW(2), ANY(null) + } + + private fun expectNewDaemon(serverType: ServerType, extraAction: (CompileServiceAsync) -> Unit = {}) = expectDaemon( + getDaemons = ::getNewDaemonsOrAsyncWrappers, + chooseDaemon = { daemons -> daemons.maxWith(comparator)!!.daemon }, + getInfo = { d -> runBlocking { d.getDaemonInfo() } }, + registerClient = { d -> runBlocking { d.registerClient(generateClient()) } }, + port = { d -> d.serverPort }, + expectedDaemonCount = serverType.instancesNumber, + extraAction = extraAction + ) + + private fun expectOldDaemon(shouldCheckNumber: Boolean = true, extraAction: (CompileService) -> Unit = {}) = expectDaemon( + ::getOldDaemonsOrRMIWrappers, + { daemons -> daemons[0].daemon }, + { d -> d.getDaemonInfo() }, + { d -> d.registerClient(generateClient()) }, + { d -> -1 }, + 1.takeIf { shouldCheckNumber }, + extraAction + ) + + private val clientFiles = arrayListOf() + private fun generateClient(): String { + val file = createTempFile(getTestName(true), ".alive") + clientFiles.add(file) + return file.absolutePath + } + + private fun deleteClients() { + clientFiles.forEach { it.delete() } + } + + private fun endTest() { + deleteClients() + } + + fun testConnectionMechanism_OldClient_OldServer() { + runOldServer() + expectOldDaemon() + endTest() + } + + + fun testConnectionMechanism_NewClient_NewServer() { + runNewServer() + expectNewDaemon(ServerType.NEW) + endTest() + } + + fun testConnectionMechanism_OldClient_NewServer() { + runNewServer() + expectOldDaemon() + endTest() + } + + fun testConnectionMechanism_NewClient_OldServer() { + runOldServer() + expectNewDaemon(ServerType.OLD) + endTest() + } + + + fun testConnections_OldDaemon_DifferentClients() { + runOldServer() + (0..20).forEach { + expectNewDaemon(ServerType.OLD) + expectOldDaemon() + } + endTest() + } + + fun testConnections_NewDaemon_DifferentClients() { + runNewServer() + (0..4).forEach { + expectNewDaemon(ServerType.NEW) + expectOldDaemon() + } + endTest() + } + + fun testConnections_MultipleDaemons_MultipleClients() { + (0..3).forEach { + runNewServer() + runOldServer() + } + (0..4).forEach { + expectNewDaemon(ServerType.ANY) + expectOldDaemon(shouldCheckNumber = false) + } + endTest() + } + + fun testShutdown() { + runNewServer() + expectNewDaemon(ServerType.NEW) { daemon -> + runBlocking { + daemon.shutdown() + delay(1000L) + val mem: Long = try { + daemon.getUsedMemory().get() + } catch (e: IOException) { + -100500L + } + assertTrue(mem == -100500L) + } + } + } + + + fun testCompile() { + runNewServer() + expectNewDaemon(ServerType.NEW) { daemon -> + runBlocking { + assertTrue("daemon is wrapper", daemon !is CompileServiceAsyncWrapper) + val outStream = ByteArrayOutputStream() + val msgCollector = PrintingMessageCollector(PrintStream(outStream), MessageRenderer.WITHOUT_PATHS, true) + val codes = (0 until 10).toMutableList() + val services = BasicCompilerServicesWithResultsFacadeServerServerSide( + msgCollector, + { _, _ -> }, + findCallbackServerSocket() + ) + services.runServer() + val servicesClient = services.clientSide + val compResultsClient = kotlinCompilerClient.createCompResults().clientSide + val threadCount = 10 + fun runThread(i: Int) = + async(newSingleThreadContext("thread_$i")) { + val jar = tmpdir.absolutePath + File.separator + "hello.$i.jar" + val code = + daemon.compile( + CompileService.NO_SESSION, + arrayOf( + "-include-runtime", + File(KotlinTestUtils.getTestDataPathBase() + "/integration/smoke/helloApp", "hello.kt").absolutePath, + "-d", + jar + ), + CompilationOptions( + CompilerMode.NON_INCREMENTAL_COMPILER, + CompileService.TargetPlatform.JVM, + arrayOf( + ReportCategory.COMPILER_MESSAGE.code, + ReportCategory.DAEMON_MESSAGE.code, + ReportCategory.EXCEPTION.code, + ReportCategory.OUTPUT_MESSAGE.code + ), + ReportSeverity.INFO.code, + emptyArray() + ), + servicesClient, + compResultsClient + ).get() + codes[i] = code + } + (0 until threadCount).map(::runThread).map { it.await() } + assertTrue("not-null code", codes.all { it == 0 }) + } + } + } + +} \ No newline at end of file diff --git a/compiler/util/src/org/jetbrains/kotlin/progress/experimental/CancelationStatus.kt b/compiler/util/src/org/jetbrains/kotlin/progress/experimental/CancelationStatus.kt new file mode 100644 index 00000000000..340a66b3401 --- /dev/null +++ b/compiler/util/src/org/jetbrains/kotlin/progress/experimental/CancelationStatus.kt @@ -0,0 +1,10 @@ +/* + * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.progress.experimental + +interface CompilationCanceledStatus { + suspend fun checkCanceled() +} \ No newline at end of file diff --git a/idea/build.gradle.kts b/idea/build.gradle.kts index 75ff4aa73a1..88c107a4101 100644 --- a/idea/build.gradle.kts +++ b/idea/build.gradle.kts @@ -27,6 +27,7 @@ dependencies { compile(project(":compiler:util")) compile(project(":kotlin-build-common")) compile(project(":compiler:daemon-common")) + compile(project(":compiler:daemon-common-new")) compile(projectRuntimeJar(":kotlin-daemon-client")) compile(project(":kotlin-compiler-runner")) { isTransitive = false } compile(project(":compiler:plugin-api")) diff --git a/jps-plugin/build.gradle.kts b/jps-plugin/build.gradle.kts index 44281be3e25..ab25685df11 100644 --- a/jps-plugin/build.gradle.kts +++ b/jps-plugin/build.gradle.kts @@ -11,6 +11,7 @@ dependencies { compile(project(":core:descriptors.jvm")) compile(project(":kotlin-compiler-runner")) compile(project(":compiler:daemon-common")) + compile(project(":compiler:daemon-common-new")) compile(projectRuntimeJar(":kotlin-daemon-client")) compile(project(":compiler:frontend.java")) compile(project(":js:js.frontend")) diff --git a/license/third_party/testdata/findbugs_license.txt b/license/third_party/testdata/findbugs_license.txt index 40a0ab7d3b1..6789d151d7c 100644 --- a/license/third_party/testdata/findbugs_license.txt +++ b/license/third_party/testdata/findbugs_license.txt @@ -7,7 +7,7 @@ of this license document, but changing it is not allowed. [This is the first released version of the Lesser GPL. It also counts - as the successor of the GNU Library Public License, version 2, hence + as the successor of the GNU Library Public License, Version 2, hence the version number 2.1.] Preamble @@ -227,7 +227,7 @@ the scope of this License. 3. You may opt to apply the terms of the ordinary GNU General Public License instead of this License to a given copy of the Library. To do this, you must alter all the notices that refer to this License, so -that they refer to the ordinary GNU General Public License, version 2, +that they refer to the ordinary GNU General Public License, Version 2, instead of to this License. (If a newer version than version 2 of the ordinary GNU General Public License has appeared, then you can specify that version instead if you wish.) Do not make any other change in diff --git a/prepare/compiler-client-embeddable/build.gradle.kts b/prepare/compiler-client-embeddable/build.gradle.kts index 53cf6b2be7e..4a206c6c8df 100644 --- a/prepare/compiler-client-embeddable/build.gradle.kts +++ b/prepare/compiler-client-embeddable/build.gradle.kts @@ -16,9 +16,11 @@ val archives by configurations dependencies { jarContents(project(":compiler:cli-common")) { isTransitive = false } jarContents(project(":compiler:daemon-common")) { isTransitive = false } + jarContents(project(":compiler:daemon-common-new")) { isTransitive = false } jarContents(projectRuntimeJar(":kotlin-daemon-client")) testCompile(project(":compiler:cli-common")) testCompile(project(":compiler:daemon-common")) + testCompile(project(":compiler:daemon-common-new")) testCompile(projectRuntimeJar(":kotlin-daemon-client")) testCompile(commonDep("junit:junit")) testCompile(project(":kotlin-test:kotlin-test-jvm")) diff --git a/prepare/compiler/build.gradle.kts b/prepare/compiler/build.gradle.kts index 70bbae988dc..802eee32806 100644 --- a/prepare/compiler/build.gradle.kts +++ b/prepare/compiler/build.gradle.kts @@ -32,6 +32,8 @@ default.extendsFrom(runtimeJar) val compilerBaseName = name +val ktorExcludesForDaemon : List> by rootProject.extra + val outputJar = fileFrom(buildDir, "libs", "$compilerBaseName.jar") val compilerModules: Array by rootProject.extra @@ -58,6 +60,9 @@ dependencies { trove4jJar(intellijDep()) { includeIntellijCoreJarDependencies(project) { it.startsWith("trove4j") } } fatJarContents(kotlinBuiltins()) + fatJarContents(project(":kotlin-daemon-client-new")) { + isTransitive = false + } fatJarContents(commonDep("javax.inject")) fatJarContents(commonDep("org.jline", "jline")) fatJarContents(commonDep("org.fusesource.jansi", "jansi")) @@ -65,6 +70,11 @@ dependencies { fatJarContents(commonDep("com.google.code.findbugs", "jsr305")) fatJarContents(commonDep("io.javaslang", "javaslang")) fatJarContents(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-core")) { isTransitive = false } + fatJarContents(commonDep("io.ktor", "ktor-network")) { + ktorExcludesForDaemon.forEach { (group, module) -> + exclude(group = group, module = module) + } + } fatJarContents(intellijCoreDep()) { includeJars("intellij-core", "java-compatibility-1.0.1") } fatJarContents(intellijDep()) { diff --git a/prepare/compiler/build.gradle.kts.182 b/prepare/compiler/build.gradle.kts.182 index 0f3b6f62c32..50bc3b848ac 100644 --- a/prepare/compiler/build.gradle.kts.182 +++ b/prepare/compiler/build.gradle.kts.182 @@ -36,6 +36,8 @@ val outputJar = fileFrom(buildDir, "libs", "$compilerBaseName.jar") val compilerModules: Array by rootProject.extra +val ktorExcludesForDaemon : List> by rootProject.extra + dependencies { compile(kotlinStdlib()) compile(project(":kotlin-script-runtime")) @@ -58,6 +60,9 @@ dependencies { trove4jJar(intellijDep()) { includeIntellijCoreJarDependencies(project) { it.startsWith("trove4j") } } fatJarContents(kotlinBuiltins()) + fatJarContents(project(":kotlin-daemon-client-new")) { + isTransitive = false + } fatJarContents(commonDep("javax.inject")) fatJarContents(commonDep("org.jline", "jline")) fatJarContents(commonDep("org.fusesource.jansi", "jansi")) @@ -65,6 +70,11 @@ dependencies { fatJarContents(commonDep("com.google.code.findbugs", "jsr305")) fatJarContents(commonDep("io.javaslang", "javaslang")) fatJarContents(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-core")) { isTransitive = false } + fatJarContents(commonDep("io.ktor", "ktor-network")) { + ktorExcludesForDaemon.forEach { (group, module) -> + exclude(group = group, module = module) + } + } fatJarContents(intellijCoreDep()) { includeJars("intellij-core") } fatJarContents(intellijDep()) { diff --git a/prepare/compiler/build.gradle.kts.as34 b/prepare/compiler/build.gradle.kts.as34 index 0f3b6f62c32..e8cc420a03f 100644 --- a/prepare/compiler/build.gradle.kts.as34 +++ b/prepare/compiler/build.gradle.kts.as34 @@ -36,6 +36,8 @@ val outputJar = fileFrom(buildDir, "libs", "$compilerBaseName.jar") val compilerModules: Array by rootProject.extra +val ktorExcludesForDaemon : List> by rootProject.extra + dependencies { compile(kotlinStdlib()) compile(project(":kotlin-script-runtime")) @@ -58,6 +60,9 @@ dependencies { trove4jJar(intellijDep()) { includeIntellijCoreJarDependencies(project) { it.startsWith("trove4j") } } fatJarContents(kotlinBuiltins()) + fatJarContents(project(":kotlin-daemon-client-new")) { + isTransitive = false + } fatJarContents(commonDep("javax.inject")) fatJarContents(commonDep("org.jline", "jline")) fatJarContents(commonDep("org.fusesource.jansi", "jansi")) @@ -65,6 +70,11 @@ dependencies { fatJarContents(commonDep("com.google.code.findbugs", "jsr305")) fatJarContents(commonDep("io.javaslang", "javaslang")) fatJarContents(commonDep("org.jetbrains.kotlinx", "kotlinx-coroutines-core")) { isTransitive = false } + fatJarContents(commonDep("io.ktor", "ktor-network")) { + ktorExcludesForDaemon.forEach { (group, module) -> + exclude(group = group, module = module) + } + } fatJarContents(intellijCoreDep()) { includeJars("intellij-core") } fatJarContents(intellijDep()) { diff --git a/prepare/idea-plugin/build.gradle.kts b/prepare/idea-plugin/build.gradle.kts index 8428d6970f8..baae9f151da 100644 --- a/prepare/idea-plugin/build.gradle.kts +++ b/prepare/idea-plugin/build.gradle.kts @@ -22,6 +22,7 @@ val projectsToShadow by extra(listOf( ":compiler:cli-common", ":compiler:container", ":compiler:daemon-common", + ":compiler:daemon-common-new", ":core:metadata", ":core:metadata.jvm", ":core:descriptors", diff --git a/prepare/jps-plugin/build.gradle.kts b/prepare/jps-plugin/build.gradle.kts index 3f76b7999dd..649d8151f09 100644 --- a/prepare/jps-plugin/build.gradle.kts +++ b/prepare/jps-plugin/build.gradle.kts @@ -11,6 +11,7 @@ val projectsToShadow = listOf( ":compiler:cli-common", ":kotlin-compiler-runner", ":compiler:daemon-common", + ":compiler:daemon-common-new", ":core:descriptors", ":core:descriptors.jvm", ":idea:idea-jps-common", diff --git a/settings.gradle b/settings.gradle index 2bd1ddbc241..eac040cf236 100644 --- a/settings.gradle +++ b/settings.gradle @@ -20,8 +20,10 @@ include ":kotlin-build-common", ":compiler", ":compiler:util", ":compiler:daemon-common", + ":compiler:daemon-common-new", ":compiler:daemon", ":kotlin-daemon-client", + ":kotlin-daemon-client-new", ":kotlin-preloader", ":kotlin-runner", ":compiler:container", @@ -293,7 +295,9 @@ project(':compiler:cli-common').projectDir = "$rootDir/compiler/cli/cli-common" project(':compiler:cli-js').projectDir = "$rootDir/compiler/cli/cli-js" as File project(':kotlin-runner').projectDir = "$rootDir/compiler/cli/cli-runner" as File project(':compiler:daemon-common').projectDir = "$rootDir/compiler/daemon/daemon-common" as File +project(':compiler:daemon-common-new').projectDir = "$rootDir/compiler/daemon/daemon-common-new" as File project(':kotlin-daemon-client').projectDir = "$rootDir/compiler/daemon/daemon-client" as File +project(':kotlin-daemon-client-new').projectDir = "$rootDir/compiler/daemon/daemon-client-new" as File project(':kotlin-compiler-runner').projectDir = "$rootDir/compiler/compiler-runner" as File project(':kotlin-ant').projectDir = "$rootDir/ant" as File project(':compiler:ir.tree').projectDir = "$rootDir/compiler/ir/ir.tree" as File