From 6a1c82d1016f3c51d0c697addb6c355811c0a6a9 Mon Sep 17 00:00:00 2001
From: Ilya Chernikov <ilya.chernikov@jetbrains.com>
Date: Wed, 28 Oct 2015 17:27:12 +0100
Subject: [PATCH] hiding compiler digest management inside the compiler
 service, implement simple lazy checking mechanism, using it from clients

---
 .idea/runConfigurations/IDEA.xml              |   2 +-
 .../rmi/kotlinr/KotlinCompilerClient.kt       |  18 +--
 .../jetbrains/kotlin/rmi/CompileService.kt    |   2 +-
 .../org/jetbrains/kotlin/rmi/DaemonParams.kt  |  53 +-------
 .../kotlin/rmi/service/CompileDaemon.kt       |   4 +-
 .../kotlin/rmi/service/CompileServiceImpl.kt  |  12 +-
 .../rmi/service/LazyClasspathWatcher.kt       | 128 ++++++++++++++++++
 7 files changed, 152 insertions(+), 67 deletions(-)
 create mode 100644 compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/LazyClasspathWatcher.kt

diff --git a/.idea/runConfigurations/IDEA.xml b/.idea/runConfigurations/IDEA.xml
index f40593fb308..16f5fcdef1e 100644
--- a/.idea/runConfigurations/IDEA.xml
+++ b/.idea/runConfigurations/IDEA.xml
@@ -6,7 +6,7 @@
     <option name="PROGRAM_PARAMETERS" value="" />
     <option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/ideaSDK/bin" />
     <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
-    <option name="ALTERNATIVE_JRE_PATH" value="" />
+    <option name="ALTERNATIVE_JRE_PATH" />
     <option name="ENABLE_SWING_INSPECTOR" value="false" />
     <option name="ENV_VARIABLES" />
     <option name="PASS_PARENT_ENVS" value="true" />
diff --git a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt
index 998e45df1d4..db961e9b919 100644
--- a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt
+++ b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt
@@ -59,7 +59,7 @@ public object KotlinCompilerClient {
             while (attempts++ < DAEMON_CONNECT_CYCLE_ATTEMPTS) {
                 val service = tryFindDaemon(File(daemonOptions.runFilesPath), compilerId, reportingTargets)
                 if (service != null) {
-                    if (!checkId || checkCompilerId(service, compilerId)) {
+                    if (!checkId || service.checkCompilerId(compilerId)) {
                         reportingTargets.report(DaemonReportCategory.DEBUG, "connected to the daemon")
                         return service
                     }
@@ -186,8 +186,6 @@ public object KotlinCompilerClient {
                 throw IllegalArgumentException("Cannot find compiler jar")
             else
                 println("desired compiler classpath: " + compilerId.compilerClasspath.joinToString(File.pathSeparator))
-
-            compilerId.updateDigest()
         }
 
         val daemon = connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, DaemonReportingTargets(out = System.out), autostart = !clientOptions.stop, checkId = !clientOptions.stop)
@@ -206,7 +204,7 @@ public object KotlinCompilerClient {
             }
             filteredArgs.none() -> {
                 // so far used only in tests
-                println("Warning: empty arguments list, only daemon check is performed: checkCompilerId() returns ${checkCompilerId(daemon, compilerId)}")
+                println("Warning: empty arguments list, only daemon check is performed: checkCompilerId() returns ${daemon.checkCompilerId(compilerId)}")
             }
             else -> {
                 println("Executing daemon compilation with args: " + filteredArgs.joinToString(" "))
@@ -264,7 +262,7 @@ public object KotlinCompilerClient {
 
 
     private fun tryFindDaemon(registryDir: File, compilerId: CompilerId, reportingTargets: DaemonReportingTargets): CompileService? {
-        val classPathDigest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest()
+        val classPathDigest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest().toHexString()
         val daemons = registryDir.walk()
                 .map { Pair(it, it.name.extractPortFromRunFilename(classPathDigest)) }
                 .filter { it.second != 0 }
@@ -377,20 +375,12 @@ public object KotlinCompilerClient {
     }
 
 
-    private fun checkCompilerId(compiler: CompileService, localId: CompilerId): Boolean {
-        val remoteId = compiler.getCompilerId()
-        return (localId.compilerVersion.isEmpty() || localId.compilerVersion == remoteId.compilerVersion) &&
-               (localId.compilerClasspath.all { remoteId.compilerClasspath.contains(it) }) &&
-               (localId.compilerDigest.isEmpty() || remoteId.compilerDigest.isEmpty() || localId.compilerDigest == remoteId.compilerDigest)
-    }
-
-
     class FileBasedLock(compilerId: CompilerId, daemonOptions: DaemonOptions) {
 
         private val lockFile: File =
                 File(daemonOptions.runFilesPath,
                      makeRunFilenameString(timestamp = "lock",
-                                           digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest(),
+                                           digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest().toHexString(),
                                            port = "0"))
         @Volatile private var locked = acquireLockFile(lockFile)
         private val channel = if (locked) RandomAccessFile(lockFile, "rw").channel else null
diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt
index 7126838ad53..b2ba6d764b0 100644
--- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt
+++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt
@@ -32,7 +32,7 @@ public interface CompileService : Remote {
     }
 
     @Throws(RemoteException::class)
-    public fun getCompilerId(): CompilerId
+    public fun checkCompilerId(compilerId: CompilerId): Boolean
 
     @Throws(RemoteException::class)
     public fun getUsedMemory(): Long
diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt
index 257a85bccc3..7884e0eca60 100644
--- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt
+++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt
@@ -19,7 +19,6 @@ package org.jetbrains.kotlin.rmi
 import java.io.File
 import java.io.Serializable
 import java.lang.management.ManagementFactory
-import java.security.DigestInputStream
 import java.security.MessageDigest
 import kotlin.reflect.KMutableProperty1
 
@@ -51,8 +50,7 @@ public val COMPILE_DAEMON_FORCE_SHUTDOWN_TIMEOUT_INFINITE: Long = 0L
 public val COMPILE_DAEMON_DEFAULT_RUN_DIR_PATH: String get() =
     FileSystem.getRuntimeStateFilesPath("kotlin", "daemon")
 
-val COMPILER_ID_DIGEST = "MD5"
-
+val CLASSPATH_ID_DIGEST = "MD5"
 
 public fun makeRunFilenameString(timestamp: String, digest: String, port: String, escapeSequence: String = ""): String = "$COMPILE_DAEMON_DEFAULT_FILES_PREFIX$escapeSequence.$timestamp$escapeSequence.$digest$escapeSequence.$port$escapeSequence.run"
 
@@ -222,68 +220,29 @@ public data class DaemonOptions(
 }
 
 
-fun updateSingleFileDigest(file: File, md: MessageDigest) {
-    DigestInputStream(file.inputStream(), md).use {
-        val buf = ByteArray(1024)
-        while (it.read(buf) != -1) {}
-        it.close()
-    }
-}
-
-fun updateForAllClasses(dir: File, md: MessageDigest) {
-    dir.walk().forEach { updateEntryDigest(it, md) }
-}
-
-fun updateEntryDigest(entry: File, md: MessageDigest) {
-    when {
-        entry.isDirectory
-            -> updateForAllClasses(entry, md)
-        entry.isFile &&
-        (entry.extension.equals("class", ignoreCase = true) ||
-         entry.extension.equals("jar", ignoreCase = true))
-            -> updateSingleFileDigest(entry, md)
-    // else skip
-    }
-}
-
-@JvmName("getFilesClasspathDigest_Files")
-fun Iterable<File>.getFilesClasspathDigest(): String {
-    val md = MessageDigest.getInstance(COMPILER_ID_DIGEST)
-    this.forEach { updateEntryDigest(it, md) }
-    return md.digest().joinToString("", transform = { "%02x".format(it) })
-}
-
-@JvmName("getFilesClasspathDigest_Strings")
-fun Iterable<String>.getFilesClasspathDigest(): String = map { File(it) }.getFilesClasspathDigest()
-
-fun Iterable<String>.distinctStringsDigest(): String =
-        MessageDigest.getInstance(COMPILER_ID_DIGEST)
+fun Iterable<String>.distinctStringsDigest(): ByteArray =
+        MessageDigest.getInstance(CLASSPATH_ID_DIGEST)
                 .digest(this.distinct().sorted().joinToString("").toByteArray())
-                .joinToString("", transform = { "%02x".format(it) })
+
+fun ByteArray.toHexString(): String = joinToString("", transform = { "%02x".format(it) })
 
 
 public data class CompilerId(
         public var compilerClasspath: List<String> = listOf(),
-        public var compilerDigest: String = "",
         public var compilerVersion: String = ""
 ) : OptionsGroup {
 
     override val mappers: List<PropMapper<*, *, *>>
         get() = listOf(PropMapper(this, CompilerId::compilerClasspath, toString = { it.joinToString(File.pathSeparator) }, fromString = { it.trimQuotes().split(File.pathSeparator) }),
-                       StringPropMapper(this, CompilerId::compilerDigest),
                        StringPropMapper(this, CompilerId::compilerVersion))
 
-    public fun updateDigest() {
-        compilerDigest = compilerClasspath.getFilesClasspathDigest()
-    }
-
     companion object {
         @JvmStatic
         public fun makeCompilerId(vararg paths: File): CompilerId = makeCompilerId(paths.asIterable())
 
         @JvmStatic
         public fun makeCompilerId(paths: Iterable<File>): CompilerId =
-                CompilerId(compilerClasspath = paths.map { it.absolutePath }, compilerDigest = paths.getFilesClasspathDigest())
+                CompilerId(compilerClasspath = paths.map { it.absolutePath })
     }
 }
 
diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt
index e46614666d9..b790875717c 100644
--- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt
+++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt
@@ -116,14 +116,14 @@ public object CompileDaemon {
             //            if (System.getSecurityManager() == null)
             //                System.setSecurityManager (RMISecurityManager())
             //
-            //            setDaemonPpermissions(daemonOptions.port)
+            //            setDaemonPermissions(daemonOptions.port)
 
             val (registry, port) = findPortAndCreateRegistry(COMPILE_DAEMON_FIND_PORT_ATTEMPTS, COMPILE_DAEMON_PORTS_RANGE_START, COMPILE_DAEMON_PORTS_RANGE_END)
             val runFileDir = File(if (daemonOptions.runFilesPath.isBlank()) COMPILE_DAEMON_DEFAULT_RUN_DIR_PATH else daemonOptions.runFilesPath)
             runFileDir.mkdirs()
             val runFile = File(runFileDir,
                                makeRunFilenameString(timestamp = "%tFT%<tH-%<tM-%<tS.%<tLZ".format(Calendar.getInstance(TimeZone.getTimeZone("Z"))),
-                                                     digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest(),
+                                                     digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest().toHexString(),
                                                      port = port.toString()))
             try {
                 if (!runFile.createNewFile()) throw Exception("createNewFile returned false")
diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt
index d384e57bb15..d07c1847b85 100644
--- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt
+++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt
@@ -32,8 +32,8 @@ import java.util.concurrent.TimeUnit
 import java.util.concurrent.locks.ReentrantReadWriteLock
 import java.util.logging.Logger
 import kotlin.concurrent.read
-import kotlin.concurrent.write
 import kotlin.concurrent.schedule
+import kotlin.concurrent.write
 
 fun nowSeconds() = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime())
 
@@ -49,9 +49,16 @@ class CompileServiceImpl(
         port: Int
 ) : CompileService {
 
+    private val classpathWatcher = LazyClasspathWatcher(selfCompilerId.compilerClasspath)
+
     // RMI-exposed API
 
-    override fun getCompilerId(): CompilerId = ifAlive { selfCompilerId }
+    override fun checkCompilerId(compilerId: CompilerId): Boolean =
+        ifAlive {
+            (selfCompilerId.compilerVersion.isEmpty() || selfCompilerId.compilerVersion == compilerId.compilerVersion) &&
+            (selfCompilerId.compilerClasspath.all { compilerId.compilerClasspath.contains(it) }) &&
+            !classpathWatcher.isChanged
+        }
 
     override fun getUsedMemory(): Long = ifAlive { usedMemory(withGC = true) }
 
@@ -146,6 +153,7 @@ class CompileServiceImpl(
                           operationsTracer: RemoteOperationsTracer?,
                           body: (PrintStream, Profiler) -> ExitCode): Int =
             ifAlive {
+
                 operationsTracer?.before("compile")
                 val rpcProfiler = if (daemonOptions.reportPerf) WallAndThreadTotalProfiler() else DummyProfiler()
                 val compilerMessagesStream = PrintStream(BufferedOutputStream(RemoteOutputStreamClient(compilerMessagesStreamProxy, rpcProfiler), 4096))
diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/LazyClasspathWatcher.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/LazyClasspathWatcher.kt
new file mode 100644
index 00000000000..2adad29e6ab
--- /dev/null
+++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/LazyClasspathWatcher.kt
@@ -0,0 +1,128 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.rmi.service
+
+import java.io.File
+import java.io.IOException
+import java.security.DigestInputStream
+import java.security.MessageDigest
+import java.util.*
+import java.util.concurrent.Semaphore
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicBoolean
+import java.util.concurrent.atomic.AtomicLong
+import java.util.logging.Level
+import java.util.logging.Logger
+import kotlin.concurrent.thread
+
+
+val CLASSPATH_FILE_ID_DIGEST = "MD5"
+val DEFAULT_CLASSPATH_WATCH_PERIOD_MS = 1000L
+val DEFAULT_CLASSPATH_DIGEST_WATCH_PERIOD_MS = 300000L // 5 min
+
+
+/**
+ * Class for lazy (on demand) check if any relevant file in the classpath is changed
+ * poor-man watcher in the absence of NIO
+ * TODO: replace with NIO watching when switching to java 7+
+ */
+public class LazyClasspathWatcher(classpath: Iterable<String>,
+                                  val chackPeriod: Long = DEFAULT_CLASSPATH_WATCH_PERIOD_MS,
+                                  val digestCheckPeriod: Long = DEFAULT_CLASSPATH_DIGEST_WATCH_PERIOD_MS) {
+
+    private data class FileId(val file: File, val lastModified: Long, val digest: ByteArray)
+
+    private val fileIdsLock = Semaphore(1) // a barrier for ensuring ids are initialized, using semaphore to allow modifications from another thread
+    private var fileIds: ArrayList<FileId>? = null
+    private val lastChangedStatus = AtomicBoolean(false)
+    private val lastUpdate = AtomicLong(0)
+    private val lastDigestUpdate = AtomicLong(0)
+    private val log by lazy { Logger.getLogger("classpath watcher") }
+
+    init {
+        // locking before entering thread in order to avoid racing with isChanged
+        fileIdsLock.acquire()
+        thread(daemon = true, start = true) {
+            try {
+                fileIds = classpath
+                        .map { File(it) }
+                        .asSequence()
+                        .flatMap { FileTreeWalk(it, filter = ::isClasspathFile) }
+                        .map { FileId(it, it.lastModified(), it.md5Digest()) }
+                        .toArrayList()
+                val nowMs = TimeUnit.MILLISECONDS.toMillis(System.nanoTime())
+                lastUpdate.set(nowMs)
+                lastDigestUpdate.set(nowMs)
+            }
+            catch (e: IOException) {
+                log.log(Level.WARNING, "Error on walking classpath", e)
+                // ignoring it for now
+            }
+            finally {
+                fileIdsLock.release()
+            }
+        }
+    }
+
+    public val isChanged: Boolean get() {
+        if (lastChangedStatus.get()) return true
+        val nowMs = TimeUnit.MILLISECONDS.toMillis(System.nanoTime())
+        if (nowMs - lastUpdate.get() < chackPeriod) return false
+
+        val checkDigest = nowMs - lastDigestUpdate.get() > digestCheckPeriod
+        // making sure that fieldIds are initialized
+        fileIdsLock.acquire()
+        fileIdsLock.release()
+        val changed =
+            fileIds?.find {
+                try {
+                    if (!it.file.exists()) {
+                        log.info("cp changed: ${it.file} doesn't exist any more")
+                        true
+                    }
+                    // if last modified changed or if enforced by param - checking the digest
+                    else if ((it.file.lastModified() != it.lastModified || checkDigest) && !Arrays.equals(it.digest, it.file.md5Digest())) {
+                        log.info("cp changed: ${it.file} digests differ")
+                        true
+                    }
+                    else false
+                }
+                catch (e: IOException) {
+                    log.log(Level.INFO, "cp changed: ${it.file} access throws the exception", e)
+                    true // io error considered as change
+                }
+            } != null
+        lastUpdate.set(TimeUnit.MILLISECONDS.toMillis(System.nanoTime()))
+        if (checkDigest) lastDigestUpdate.set(lastUpdate.get())
+
+        return changed
+    }
+}
+
+
+fun isClasspathFile(file: File): Boolean = file.isFile && listOf("class", "jar").contains(file.extension.toLowerCase())
+
+fun File.md5Digest(): ByteArray {
+    val md = MessageDigest.getInstance(CLASSPATH_FILE_ID_DIGEST)
+    DigestInputStream(inputStream(), md).use {
+        val buf = ByteArray(1024)
+        while (it.read(buf) != -1) {}
+        it.close()
+    }
+    return md.digest()
+}
+