Implement graceful connection failure behavior on daemon connect and cancellation check

Should fix (or actually - behave gracefully on) #EA-74003, #EA-74493, #EA-76032, #EA-76529, #EA-81295
Tests added

compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteCompilationCanceledStatusClient.kt

@@ -23,12 +23,23 @@ import org.jetbrains.kotlin.daemon.common.Profiler
 import org.jetbrains.kotlin.daemon.common.RmiFriendlyCompilationCancelledException
 import org.jetbrains.kotlin.progress.CompilationCanceledException
 import java.util.concurrent.TimeUnit
+import java.util.logging.Logger
 
 val CANCELED_STATUS_CHECK_THRESHOLD_NS = TimeUnit.MILLISECONDS.toNanos(100)
 
 class RemoteCompilationCanceledStatusClient(val facade: CompilerCallbackServicesFacade, val profiler: Profiler = DummyProfiler()): CompilationCanceledStatus {
+
+    private val log by lazy { Logger.getLogger("compiler") }
+
     @Volatile var lastChecked: Long = System.nanoTime()
+
     override fun checkCanceled() {
+
+        fun cancelOnError(e: Exception) {
+            log.warning("error communicating with host, assuming compilation cancelled (${e.message})")
+            throw CompilationCanceledException()
+        }
+
         val curNanos = System.nanoTime()
         if (curNanos - lastChecked > CANCELED_STATUS_CHECK_THRESHOLD_NS) {
             profiler.withMeasure(this) {
@@ -38,6 +49,19 @@ class RemoteCompilationCanceledStatusClient(val facade: CompilerCallbackServices
                 catch (e: RmiFriendlyCompilationCancelledException) {
                     throw CompilationCanceledException()
                 }
+                catch (e: java.rmi.ConnectIOException) {
+                    cancelOnError(e)
+                }
+                catch (e: java.rmi.ConnectException) {
+                    cancelOnError(e)
+                }
+                catch (e: java.rmi.NoSuchObjectException) {
+                    // this was added mostly for tests since others are more difficult to emulate
+                    cancelOnError(e)
+                }
+                catch (e: java.rmi.UnmarshalException) {
+                    cancelOnError(e)
+                }
             }
             lastChecked = curNanos
         }

compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt

@@ -17,13 +17,17 @@
 package org.jetbrains.kotlin.daemon
 
 import org.jetbrains.kotlin.cli.AbstractCliTest
+import org.jetbrains.kotlin.daemon.client.CompilerCallbackServicesFacadeServer
 import org.jetbrains.kotlin.daemon.client.DaemonReportingTargets
 import org.jetbrains.kotlin.daemon.client.KotlinCompilerClient
+import org.jetbrains.kotlin.daemon.client.RemoteOutputStreamServer
 import org.jetbrains.kotlin.daemon.common.*
 import org.jetbrains.kotlin.integration.KotlinIntegrationTestBase
+import org.jetbrains.kotlin.progress.CompilationCanceledStatus
 import org.jetbrains.kotlin.test.KotlinTestUtils
 import java.io.ByteArrayOutputStream
 import java.io.File
+import java.rmi.server.UnicastRemoteObject
 import java.util.concurrent.CountDownLatch
 import java.util.concurrent.TimeUnit
 import kotlin.concurrent.thread
@@ -389,6 +393,101 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
             }
         }
     }
+
+    fun testDaemonConnectionProblems() {
+        withFlagFile(getTestName(true), ".alive") { flagFile ->
+            val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
+            KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions)
+
+            val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false)
+            val daemon = KotlinCompilerClient.connectToCompileService(compilerId, flagFile, daemonJVMOptions, daemonOptions, DaemonReportingTargets(out = System.err), autostart = true)
+            assertNotNull("failed to connect daemon", daemon)
+            daemon?.registerClient(flagFile.absolutePath)
+
+            KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions)
+            Thread.sleep(200)
+
+            val exception: Exception? = try {
+                daemon!!.getUsedMemory()
+                null
+            }
+            catch (e: java.rmi.ConnectException) {
+                e
+            }
+            catch (e: java.rmi.UnmarshalException) {
+                e
+            }
+            assertNotNull(exception)
+        }
+    }
+
+    fun testDaemonCallbackConnectionProblems() {
+        withFlagFile(getTestName(true), ".alive") { flagFile ->
+            val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
+            KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions)
+
+            val logFile = createTempFile("kotlin-daemon-test", ".log")
+            val daemonJVMOptions =
+                    configureDaemonJVMOptions("D${COMPILE_DAEMON_LOG_PATH_PROPERTY}=\"${logFile.loggerCompatiblePath}\"",
+                                              inheritMemoryLimits = false, inheritAdditionalProperties = false)
+
+            val daemon = KotlinCompilerClient.connectToCompileService(compilerId, flagFile, daemonJVMOptions, daemonOptions, DaemonReportingTargets(out = System.err), autostart = true)
+            assertNotNull("failed to connect daemon", daemon)
+            daemon?.registerClient(flagFile.absolutePath)
+
+            val file = File(tmpdir, "largeKotlinFile.kt")
+            file.writeText(generateLargeKotlinFile(10))
+            val jar = tmpdir.absolutePath + File.separator + "largeKotlinFile.jar"
+
+            var callbackServices: CompilerCallbackServicesFacadeServer? = null
+            callbackServices = CompilerCallbackServicesFacadeServer(compilationCancelledStatus = object : CompilationCanceledStatus {
+                override fun checkCanceled() {
+                    thread {
+                        Thread.sleep(10)
+                        UnicastRemoteObject.unexportObject(callbackServices, true)
+                    }
+                }
+            }, port = SOCKET_ANY_FREE_PORT)
+            val strm = ByteArrayOutputStream()
+            val code = daemon!!.remoteCompile(CompileService.NO_SESSION,
+                                              CompileService.TargetPlatform.JVM,
+                                              arrayOf("-include-runtime", file.absolutePath, "-d", jar),
+                                              callbackServices,
+                                              RemoteOutputStreamServer(strm, SOCKET_ANY_FREE_PORT),
+                                              CompileService.OutputFormat.XML,
+                                              RemoteOutputStreamServer(strm, SOCKET_ANY_FREE_PORT),
+                                              null).get()
+
+            val compilerOutput = strm.toString()
+            assertTrue("Expecting cancelation message in:\n$compilerOutput", compilerOutput.contains("Compilation was canceled"))
+            logFile.assertLogContainsSequence("error communicating with host, assuming compilation cancelled")
+
+            logFile.delete()
+        }
+    }
+}
+
+// stolen from CompilerFileLimitTest
+internal fun generateLargeKotlinFile(size: Int): String {
+    return buildString {
+        append("package large\n\n")
+        (0..size).forEach {
+            appendln("class Class$it")
+            appendln("{")
+            appendln("\tfun foo(): Long = $it")
+            appendln("}")
+            appendln("\n")
+            repeat(2000) {
+                appendln("// kotlin rules ... and stuff")
+            }
+        }
+        appendln("fun main(args: Array<String>)")
+        appendln("{")
+        appendln("\tval result = Class5().foo() + Class$size().foo()")
+        appendln("\tprintln(result)")
+        appendln("}")
+    }
+
 }
 
 

jps-plugin/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.kt

@@ -38,9 +38,11 @@ import org.jetbrains.kotlin.daemon.common.*
 import org.jetbrains.kotlin.jps.build.KotlinBuilder
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
 import org.jetbrains.kotlin.progress.CompilationCanceledStatus
+import org.jetbrains.kotlin.utils.rethrow
 import java.io.*
 import java.lang.reflect.Field
 import java.lang.reflect.Modifier
+import java.rmi.ConnectException
 import java.util.*
 import java.util.concurrent.TimeUnit
 
@@ -180,7 +182,8 @@ object KotlinCompilerRunner {
                                      argsArray: Array<String>,
                                      environment: CompilerEnvironment,
                                      messageCollector: MessageCollector,
-                                     collector: OutputItemsCollector): Boolean {
+                                     collector: OutputItemsCollector,
+                                     retryOnConnectionError: Boolean = true): Boolean {
 
         if (isDaemonEnabled()) {
 
@@ -202,7 +205,25 @@ object KotlinCompilerRunner {
                     K2JS_COMPILER -> CompileService.TargetPlatform.JS
                     else -> throw IllegalArgumentException("Unknown compiler type $compilerClassName")
                 }
-                val res = KotlinCompilerClient.incrementalCompile(connection.daemon, connection.sessionId, targetPlatform, argsArray, services, compilerOut, daemonOut)
+
+                fun retryOrFalse(e: Exception): Boolean {
+                    if (retryOnConnectionError) {
+                        KotlinBuilder.LOG.debug("retrying once on daemon connection error: ${e.message}")
+                        return tryCompileWithDaemon(compilerClassName, argsArray, environment, messageCollector, collector, retryOnConnectionError = false)
+                    }
+                    KotlinBuilder.LOG.info("daemon connection error: ${e.message}")
+                    return false
+                }
+
+                val res: Int = try {
+                    KotlinCompilerClient.incrementalCompile(connection.daemon, connection.sessionId, targetPlatform, argsArray, services, compilerOut, daemonOut)
+                }
+                catch (e: java.rmi.ConnectException) {
+                    return retryOrFalse(e)
+                }
+                catch (e: java.rmi.UnmarshalException) {
+                    return retryOrFalse(e)
+                }
 
                 processCompilerOutput(messageCollector, collector, compilerOut, res.toString())
                 BufferedReader(StringReader(daemonOut.toString())).forEachLine {