Use connect-and-lease when using daemon in JPS & Gradle

Relates to KT-15562 "Service is dying". This commit includes multiple changes: 1. JPS & Gradle daemon clients are refactored to use `connectAndLease` from `KotlinCompilerClient`. `connectAndLease` was introduced in previous commits 2. `withKotlin` was removed because `connectAndLease` already covers retrying on connection error 3. Gradle flag files creation is changed: * client-alive flag file lives as long as Gradle instance lives, * session-alive flag file lives until the end of a build.
2017-02-22 07:57:47 +03:00
parent 9819de1abd
commit 9d95c841a6
9 changed files with 279 additions and 130 deletions
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.gradle
+
+import org.jetbrains.kotlin.compilerRunner.COULD_NOT_CONNECT_TO_DAEMON_MESSAGE
+import org.jetbrains.kotlin.compilerRunner.CREATED_SESSION_FILE_PREFIX
+import org.jetbrains.kotlin.compilerRunner.DELETED_SESSION_FILE_PREFIX
+import org.jetbrains.kotlin.compilerRunner.EXISTING_SESSION_FILE_PREFIX
+import org.junit.Assert
+import org.junit.Test
+
+// todo: test client file creation/deletion
+// todo: test daemon start (does not start every build)
+// todo: test daemon shutdown when gradle daemon dies
+class KotlinDaemonAdvaced : BaseGradleIT() {
+    companion object {
+        private const val GRADLE_VERSION = "2.10"
+    }
+
+    @Test
+    fun testDaemonMultiproject() {
+        fun String.findAllStringsPrefixed(prefix: String): Array<String> {
+            val result = arrayListOf<String>()
+            val regex = ("$prefix([^\\r\\n]*)").toRegex()
+            for (match in regex.findAll(this)) {
+                result.add(match.groupValues[1])
+            }
+            return result.toTypedArray()
+        }
+
+        val project = Project("multiprojectWithDependency", GRADLE_VERSION)
+        val strategyCLIArg = "-Dkotlin.compiler.execution.strategy=daemon"
+
+        fun checkAfterNonIncrementalBuild(output: String) {
+            val createdSessions = output.findAllStringsPrefixed(CREATED_SESSION_FILE_PREFIX)
+            assert(createdSessions.size == 1) { "Created multiple sessions per build ${createdSessions.joinToString()}" }
+
+            val existingSessions = output.findAllStringsPrefixed(EXISTING_SESSION_FILE_PREFIX)
+            Assert.assertArrayEquals("Existing sessions don't match created sessions for two module projects", createdSessions, existingSessions)
+
+            val deletedSessions = output.findAllStringsPrefixed(DELETED_SESSION_FILE_PREFIX)
+            Assert.assertArrayEquals("Deleted sessions don't match created sessions", createdSessions, deletedSessions)
+        }
+
+        project.build("build", strategyCLIArg) {
+            assertNotContains(COULD_NOT_CONNECT_TO_DAEMON_MESSAGE)
+            checkAfterNonIncrementalBuild(output)
+        }
+
+        project.build("clean", "build", strategyCLIArg) {
+            assertNotContains(COULD_NOT_CONNECT_TO_DAEMON_MESSAGE)
+            checkAfterNonIncrementalBuild(output)
+        }
+
+        project.build("build", strategyCLIArg) {
+            val createdSessions = output.findAllStringsPrefixed(CREATED_SESSION_FILE_PREFIX)
+            Assert.assertArrayEquals("Sessions should not be created (incremental build)", createdSessions, emptyArray())
+
+            val existingSessions = output.findAllStringsPrefixed(EXISTING_SESSION_FILE_PREFIX)
+            Assert.assertArrayEquals("Sessions should not be existing (incremental build)", existingSessions, emptyArray())
+
+            val deletedSessions = output.findAllStringsPrefixed(DELETED_SESSION_FILE_PREFIX)
+            Assert.assertArrayEquals("Sessions should not be deleted (incremental build)", deletedSessions, emptyArray())
+        }
+    }
+}
@@ -24,33 +24,30 @@ import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
 import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
 import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
 import org.jetbrains.kotlin.cli.common.arguments.K2MetadataCompilerArguments
+import org.jetbrains.kotlin.com.intellij.openapi.util.io.FileUtil
 import org.jetbrains.kotlin.config.Services
+import org.jetbrains.kotlin.daemon.client.CompileServiceSession
 import org.jetbrains.kotlin.daemon.common.*
-import org.jetbrains.kotlin.daemon.common.CompilationResultCategory
 import org.jetbrains.kotlin.gradle.plugin.ParentLastURLClassLoader
 import org.jetbrains.kotlin.gradle.plugin.kotlinDebug
 import org.jetbrains.kotlin.incremental.*
-import java.io.ByteArrayOutputStream
-import java.io.File
-import java.io.PrintStream
+
+import java.io.*
 import kotlin.concurrent.thread

 internal const val KOTLIN_COMPILER_EXECUTION_STRATEGY_PROPERTY = "kotlin.compiler.execution.strategy"
 internal const val DAEMON_EXECUTION_STRATEGY = "daemon"
 internal const val IN_PROCESS_EXECUTION_STRATEGY = "in-process"
 internal const val OUT_OF_PROCESS_EXECUTION_STRATEGY = "out-of-process"
+const val CREATED_CLIENT_FILE_PREFIX = "Created client-is-alive flag file: "
+const val EXISTING_CLIENT_FILE_PREFIX = "Existing client-is-alive flag file: "
+const val CREATED_SESSION_FILE_PREFIX = "Created session-is-alive flag file: "
+const val EXISTING_SESSION_FILE_PREFIX = "Existing session-is-alive flag file: "
+const val DELETED_SESSION_FILE_PREFIX = "Deleted session-is-alive flag file: "
+const val COULD_NOT_CONNECT_TO_DAEMON_MESSAGE = "Could not connect to Kotlin compile daemon"

 internal class GradleCompilerRunner(private val project: Project) : KotlinCompilerRunner<GradleCompilerEnvironment>() {
    override val log = GradleKotlinLogger(project.logger)
-    private val flagFile = run {
-        val dir = File(project.rootProject.buildDir, "kotlin").apply { mkdirs() }
-        File(dir, "daemon-is-alive").apply {
-            if (!exists()) {
-                createNewFile()
-                deleteOnExit()
-            }
-        }
-    }

    fun runJvmCompiler(
            sourcesToCompile: List<File>,
@@ -132,25 +129,43 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
    }

    override fun compileWithDaemon(compilerClassName: String, compilerArgs: CommonCompilerArguments, environment: GradleCompilerEnvironment): ExitCode? {
-        val exitCode = if (environment is GradleIncrementalCompilerEnvironment) {
-            incrementalCompilationWithDaemon(environment)
-        }
-        else {
-            nonIncrementalCompilationWithDaemon(compilerClassName, environment)
-        }
-        exitCode?.let {
-            withDaemon(environment, retryOnConnectionError = true) { daemon, sessionId ->
-                daemon.clearJarCache()
+        val connection = getDaemonConnection(environment)
+        if (connection == null) {
+            if (environment is GradleIncrementalCompilerEnvironment) {
+                log.warn("Could not perform incremental compilation: $COULD_NOT_CONNECT_TO_DAEMON_MESSAGE")
            }
-            logFinish(DAEMON_EXECUTION_STRATEGY)
+            else {
+                log.warn(COULD_NOT_CONNECT_TO_DAEMON_MESSAGE)
+            }
+            return null
        }
+
+        val (daemon, sessionId) = connection
+        val exitCode = try {
+            val res = if (environment is GradleIncrementalCompilerEnvironment) {
+                incrementalCompilationWithDaemon(daemon, sessionId, environment)
+            } else {
+                nonIncrementalCompilationWithDaemon(daemon, sessionId, compilerClassName, environment)
+            }
+            exitCodeFromProcessExitCode(res.get())
+        }
+        catch (e: Throwable) {
+            log.warn("Compilation with Kotlin compile daemon was not successful")
+            e.printStackTrace()
+            null
+        }
+        // todo: can we clear cache on the end of session?
+        daemon.clearJarCache()
+        logFinish(DAEMON_EXECUTION_STRATEGY)
        return exitCode
    }

    private fun nonIncrementalCompilationWithDaemon(
+            daemon: CompileService,
+            sessionId: Int,
            compilerClassName: String,
            environment: GradleCompilerEnvironment
-    ): ExitCode? {
+    ): CompileService.CallResult<Int> {
        val targetPlatform = when (compilerClassName) {
            K2JVM_COMPILER -> CompileService.TargetPlatform.JVM
            K2JS_COMPILER -> CompileService.TargetPlatform.JS
@@ -166,21 +181,15 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
                reportSeverity = reportSeverity(verbose),
                requestedCompilationResults = emptyArray())
        val servicesFacade = GradleCompilerServicesFacadeImpl(project, environment.messageCollector)
-
-        val res = withDaemon(environment, retryOnConnectionError = true) { daemon, sessionId ->
-            val argsArray = ArgumentUtils.convertArgumentsToStringList(environment.compilerArgs).toTypedArray()
-            daemon.compile(sessionId, argsArray, compilationOptions, servicesFacade, compilationResults = null)
-        }
-
-        val exitCode = res?.get()?.let { exitCodeFromProcessExitCode(it) }
-        if (exitCode == null) {
-            log.warn("Could not perform non-incremental compilation with Kotlin compile daemon, " +
-                    "non-incremental compilation in fallback mode will be performed")
-        }
-        return exitCode
+        val argsArray = ArgumentUtils.convertArgumentsToStringList(environment.compilerArgs).toTypedArray()
+        return daemon.compile(sessionId, argsArray, compilationOptions, servicesFacade, compilationResults = null)
    }

-    private fun incrementalCompilationWithDaemon(environment: GradleIncrementalCompilerEnvironment): ExitCode? {
+    private fun incrementalCompilationWithDaemon(
+            daemon: CompileService,
+            sessionId: Int,
+            environment: GradleIncrementalCompilerEnvironment
+    ): CompileService.CallResult<Int> {
        val knownChangedFiles = environment.changedFiles as? ChangedFiles.Known

        val verbose = environment.compilerArgs.verbose
@@ -198,18 +207,8 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
                targetPlatform = CompileService.TargetPlatform.JVM
        )
        val servicesFacade = GradleIncrementalCompilerServicesFacadeImpl(project, environment)
-
-        val res = withDaemon(environment, retryOnConnectionError = true) { daemon, sessionId ->
-            val argsArray = ArgumentUtils.convertArgumentsToStringList(environment.compilerArgs).toTypedArray()
-            daemon.compile(sessionId, argsArray, compilationOptions, servicesFacade, GradleCompilationResults(project))
-        }
-
-        val exitCode = res?.get()?.let { exitCodeFromProcessExitCode(it) }
-        if (exitCode == null) {
-            log.warn("Could not perform incremental compilation with Kotlin compile daemon, " +
-                    "non-incremental compilation in fallback mode will be performed")
-        }
-        return exitCode
+        val argsArray = ArgumentUtils.convertArgumentsToStringList(environment.compilerArgs).toTypedArray()
+        return daemon.compile(sessionId, argsArray, compilationOptions, servicesFacade, GradleCompilationResults(project))
    }

    private fun reportCategories(verbose: Boolean): Array<Int> =
@@ -286,10 +285,61 @@ internal class GradleCompilerRunner(private val project: Project) : KotlinCompil
        log.debug("Finished executing kotlin compiler using $strategy strategy")
    }

-    @Synchronized
-    override fun getDaemonConnection(environment: GradleCompilerEnvironment): DaemonConnection {
+    override fun getDaemonConnection(environment: GradleCompilerEnvironment): CompileServiceSession? {
        val compilerId = CompilerId.makeCompilerId(environment.compilerClasspath)
-        return newDaemonConnection(compilerId, flagFile, environment)
+        val clientIsAliveFlagFile = getOrCreateClientFlagFile(project)
+        val sessionIsAliveFlagFile = getOrCreateSessionFlagFile(project)
+        return newDaemonConnection(compilerId, clientIsAliveFlagFile, sessionIsAliveFlagFile, environment)
+    }
+
+    companion object {
+        // created once per gradle instance
+        // when gradle daemon dies, kotlin daemon should die too
+        // however kotlin daemon (if it idles enough) can die before gradle daemon dies
+        @Volatile
+        private var clientIsAliveFlagFile: File? = null
+
+        @Synchronized
+        private fun getOrCreateClientFlagFile(project: Project): File {
+            val log = project.logger
+            if (clientIsAliveFlagFile == null || !clientIsAliveFlagFile!!.exists()) {
+                val projectName = project.rootProject.name.normalizeForFlagFile()
+                clientIsAliveFlagFile =  FileUtil.createTempFile("kotlin-compiler-in-$projectName-", ".alive")
+                log.kotlinDebug { CREATED_CLIENT_FILE_PREFIX + clientIsAliveFlagFile!!.relativeToRoot(project) }
+            }
+            else {
+                log.kotlinDebug { EXISTING_CLIENT_FILE_PREFIX + clientIsAliveFlagFile!!.relativeToRoot(project) }
+            }
+
+            return clientIsAliveFlagFile!!
+        }
+
+        private fun String.normalizeForFlagFile(): String {
+            val validChars = ('a'..'z') + ('0'..'9') + "-_"
+            return filter { it in validChars }
+        }
+
+        // session is created per build
+        @Volatile
+        private var sessionFlagFile: File? = null
+
+        // session files are deleted at org.jetbrains.kotlin.gradle.plugin.KotlinGradleBuildServices.buildFinished
+        @Synchronized
+        private fun getOrCreateSessionFlagFile(project: Project): File {
+            val log = project.logger
+            if (sessionFlagFile == null || !sessionFlagFile!!.exists()) {
+                val sessionFilesDir = sessionsDir(project).apply { mkdirs() }
+                sessionFlagFile = FileUtil.createTempFile(sessionFilesDir, "kotlin-compiler-", ".salive")
+                log.kotlinDebug { CREATED_SESSION_FILE_PREFIX + sessionFlagFile!!.relativeToRoot(project) }
+            }
+            else {
+                log.kotlinDebug { EXISTING_SESSION_FILE_PREFIX + sessionFlagFile!!.relativeToRoot(project) }
+            }
+
+            return sessionFlagFile!!
+        }
+
+        internal fun sessionsDir(project: Project): File =
+                File(File(project.rootProject.buildDir, "kotlin"), "sessions")
    }
 }
-
@@ -24,9 +24,12 @@ import org.gradle.api.logging.Logging
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.com.intellij.openapi.vfs.impl.ZipHandler
 import org.jetbrains.kotlin.com.intellij.openapi.vfs.impl.jar.CoreJarFileSystem
+import org.jetbrains.kotlin.compilerRunner.DELETED_SESSION_FILE_PREFIX
+import org.jetbrains.kotlin.compilerRunner.GradleCompilerRunner
 import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompile
 import org.jetbrains.kotlin.incremental.BuildCacheStorage
 import org.jetbrains.kotlin.incremental.multiproject.ArtifactDifferenceRegistryProvider
+import org.jetbrains.kotlin.incremental.relativeToRoot
 import java.io.File

 internal class KotlinGradleBuildServices private constructor(gradle: Gradle): BuildAdapter() {
@@ -91,6 +94,22 @@ internal class KotlinGradleBuildServices private constructor(gradle: Gradle): Bu
            log.kotlinDebug("Skipping kotlin cleanup since compiler wasn't called")
        }

+        val rootProject = gradle.rootProject
+        val sessionsDir = GradleCompilerRunner.sessionsDir(rootProject)
+        if (sessionsDir.exists()) {
+            val sessionFiles = sessionsDir.listFiles()
+
+            // it is expected that only one session file per build exists
+            // afaik is is not possible to run multiple gradle builds in one project since gradle locks some dirs
+            if (sessionFiles.size > 1) {
+                log.warn("w: Detected multiple Kotlin daemon sessions at ${sessionsDir.relativeToRoot(rootProject)}")
+            }
+            for (file in sessionFiles) {
+                file.delete()
+                log.kotlinDebug { DELETED_SESSION_FILE_PREFIX + file.relativeToRoot(rootProject) }
+            }
+        }
+
        if (kotlinCompilerCalled) {
            startMemory?.let { startMemoryCopy ->
                getUsedMemoryKb()?.let { endMemory ->
@@ -16,6 +16,7 @@

 package org.jetbrains.kotlin.incremental

+import org.gradle.api.Project
 import java.io.File

 internal fun File.isJavaFile() =
@@ -38,3 +39,6 @@ internal fun File.relativeOrCanonical(base: File): String =

 internal fun Iterable<File>.pathsAsStringRelativeTo(base: File): String =
        map { it.relativeOrCanonical(base) }.sorted().joinToString()
+
+internal fun File.relativeToRoot(project: Project): String =
+        relativeOrCanonical(project.rootProject.rootDir)