azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hiding compiler digest management inside the compiler service, implement simple lazy checking mechanism, using it from clients

Browse Source
This commit is contained in:
Ilya Chernikov
2015-10-28 17:27:12 +01:00
parent 2fa7b53d2c
commit 6a1c82d101
7 changed files with 152 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/runConfigurations/IDEA.xml
Generated
+1 -1
View File
@@ -6,7 +6,7 @@
<option name="PROGRAM_PARAMETERS" value="" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/ideaSDK/bin" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" value="" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="ENABLE_SWING_INSPECTOR" value="false" />
<option name="ENV_VARIABLES" />
<option name="PASS_PARENT_ENVS" value="true" />
compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt
+4 -14
View File
@@ -59,7 +59,7 @@ public object KotlinCompilerClient {
while (attempts++ < DAEMON_CONNECT_CYCLE_ATTEMPTS) {
val service = tryFindDaemon(File(daemonOptions.runFilesPath), compilerId, reportingTargets)
if (service != null) {
if (!checkId || checkCompilerId(service, compilerId)) {
if (!checkId || service.checkCompilerId(compilerId)) {
reportingTargets.report(DaemonReportCategory.DEBUG, "connected to the daemon")
return service
}
@@ -186,8 +186,6 @@ public object KotlinCompilerClient {
throw IllegalArgumentException("Cannot find compiler jar")
else
println("desired compiler classpath: " + compilerId.compilerClasspath.joinToString(File.pathSeparator))
compilerId.updateDigest()
}
val daemon = connectToCompileService(compilerId, daemonLaunchingOptions, daemonOptions, DaemonReportingTargets(out = System.out), autostart = !clientOptions.stop, checkId = !clientOptions.stop)
@@ -206,7 +204,7 @@ public object KotlinCompilerClient {
}
filteredArgs.none() -> {
// so far used only in tests
println("Warning: empty arguments list, only daemon check is performed: checkCompilerId() returns ${checkCompilerId(daemon, compilerId)}")
println("Warning: empty arguments list, only daemon check is performed: checkCompilerId() returns ${daemon.checkCompilerId(compilerId)}")
}
else -> {
println("Executing daemon compilation with args: " + filteredArgs.joinToString(" "))
@@ -264,7 +262,7 @@ public object KotlinCompilerClient {
private fun tryFindDaemon(registryDir: File, compilerId: CompilerId, reportingTargets: DaemonReportingTargets): CompileService? {
val classPathDigest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest()
val classPathDigest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest().toHexString()
val daemons = registryDir.walk()
.map { Pair(it, it.name.extractPortFromRunFilename(classPathDigest)) }
.filter { it.second != 0 }
@@ -377,20 +375,12 @@ public object KotlinCompilerClient {
}
private fun checkCompilerId(compiler: CompileService, localId: CompilerId): Boolean {
val remoteId = compiler.getCompilerId()
return (localId.compilerVersion.isEmpty() || localId.compilerVersion == remoteId.compilerVersion) &&
(localId.compilerClasspath.all { remoteId.compilerClasspath.contains(it) }) &&
(localId.compilerDigest.isEmpty() || remoteId.compilerDigest.isEmpty() || localId.compilerDigest == remoteId.compilerDigest)
}
class FileBasedLock(compilerId: CompilerId, daemonOptions: DaemonOptions) {
private val lockFile: File =
File(daemonOptions.runFilesPath,
makeRunFilenameString(timestamp = "lock",
digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest(),
digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest().toHexString(),
port = "0"))
@Volatile private var locked = acquireLockFile(lockFile)
private val channel = if (locked) RandomAccessFile(lockFile, "rw").channel else null
compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompileService.kt
+1 -1
View File
@@ -32,7 +32,7 @@ public interface CompileService : Remote {
}
@Throws(RemoteException::class)
public fun getCompilerId(): CompilerId
public fun checkCompilerId(compilerId: CompilerId): Boolean
@Throws(RemoteException::class)
public fun getUsedMemory(): Long
compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt
+6 -47
View File
@@ -19,7 +19,6 @@ package org.jetbrains.kotlin.rmi
import java.io.File
import java.io.Serializable
import java.lang.management.ManagementFactory
import java.security.DigestInputStream
import java.security.MessageDigest
import kotlin.reflect.KMutableProperty1
@@ -51,8 +50,7 @@ public val COMPILE_DAEMON_FORCE_SHUTDOWN_TIMEOUT_INFINITE: Long = 0L
public val COMPILE_DAEMON_DEFAULT_RUN_DIR_PATH: String get() =
FileSystem.getRuntimeStateFilesPath("kotlin", "daemon")
val COMPILER_ID_DIGEST = "MD5"
val CLASSPATH_ID_DIGEST = "MD5"
public fun makeRunFilenameString(timestamp: String, digest: String, port: String, escapeSequence: String = ""): String = "$COMPILE_DAEMON_DEFAULT_FILES_PREFIX$escapeSequence.$timestamp$escapeSequence.$digest$escapeSequence.$port$escapeSequence.run"
@@ -222,68 +220,29 @@ public data class DaemonOptions(
}
fun updateSingleFileDigest(file: File, md: MessageDigest) {
DigestInputStream(file.inputStream(), md).use {
val buf = ByteArray(1024)
while (it.read(buf) != -1) {}
it.close()
}
}
fun updateForAllClasses(dir: File, md: MessageDigest) {
dir.walk().forEach { updateEntryDigest(it, md) }
}
fun updateEntryDigest(entry: File, md: MessageDigest) {
when {
entry.isDirectory
-> updateForAllClasses(entry, md)
entry.isFile &&
(entry.extension.equals("class", ignoreCase = true) ||
entry.extension.equals("jar", ignoreCase = true))
-> updateSingleFileDigest(entry, md)
// else skip
}
}
@JvmName("getFilesClasspathDigest_Files")
fun Iterable<File>.getFilesClasspathDigest(): String {
val md = MessageDigest.getInstance(COMPILER_ID_DIGEST)
this.forEach { updateEntryDigest(it, md) }
return md.digest().joinToString("", transform = { "%02x".format(it) })
}
@JvmName("getFilesClasspathDigest_Strings")
fun Iterable<String>.getFilesClasspathDigest(): String = map { File(it) }.getFilesClasspathDigest()
fun Iterable<String>.distinctStringsDigest(): String =
MessageDigest.getInstance(COMPILER_ID_DIGEST)
fun Iterable<String>.distinctStringsDigest(): ByteArray =
MessageDigest.getInstance(CLASSPATH_ID_DIGEST)
.digest(this.distinct().sorted().joinToString("").toByteArray())
.joinToString("", transform = { "%02x".format(it) })
fun ByteArray.toHexString(): String = joinToString("", transform = { "%02x".format(it) })
public data class CompilerId(
public var compilerClasspath: List<String> = listOf(),
public var compilerDigest: String = "",
public var compilerVersion: String = ""
) : OptionsGroup {
override val mappers: List<PropMapper<*, *, *>>
get() = listOf(PropMapper(this, CompilerId::compilerClasspath, toString = { it.joinToString(File.pathSeparator) }, fromString = { it.trimQuotes().split(File.pathSeparator) }),
StringPropMapper(this, CompilerId::compilerDigest),
StringPropMapper(this, CompilerId::compilerVersion))
public fun updateDigest() {
compilerDigest = compilerClasspath.getFilesClasspathDigest()
}
companion object {
@JvmStatic
public fun makeCompilerId(vararg paths: File): CompilerId = makeCompilerId(paths.asIterable())
@JvmStatic
public fun makeCompilerId(paths: Iterable<File>): CompilerId =
CompilerId(compilerClasspath = paths.map { it.absolutePath }, compilerDigest = paths.getFilesClasspathDigest())
CompilerId(compilerClasspath = paths.map { it.absolutePath })
}
}
compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt
+2 -2
View File
@@ -116,14 +116,14 @@ public object CompileDaemon {
// if (System.getSecurityManager() == null)
// System.setSecurityManager (RMISecurityManager())
//
// setDaemonPpermissions(daemonOptions.port)
// setDaemonPermissions(daemonOptions.port)
val (registry, port) = findPortAndCreateRegistry(COMPILE_DAEMON_FIND_PORT_ATTEMPTS, COMPILE_DAEMON_PORTS_RANGE_START, COMPILE_DAEMON_PORTS_RANGE_END)
val runFileDir = File(if (daemonOptions.runFilesPath.isBlank()) COMPILE_DAEMON_DEFAULT_RUN_DIR_PATH else daemonOptions.runFilesPath)
runFileDir.mkdirs()
val runFile = File(runFileDir,
makeRunFilenameString(timestamp = "%tFT%<tH-%<tM-%<tS.%<tLZ".format(Calendar.getInstance(TimeZone.getTimeZone("Z"))),
digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest(),
digest = compilerId.compilerClasspath.map { File(it).absolutePath }.distinctStringsDigest().toHexString(),
port = port.toString()))
try {
if (!runFile.createNewFile()) throw Exception("createNewFile returned false")
compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt
+10 -2
View File
@@ -32,8 +32,8 @@ import java.util.concurrent.TimeUnit
import java.util.concurrent.locks.ReentrantReadWriteLock
import java.util.logging.Logger
import kotlin.concurrent.read
import kotlin.concurrent.write
import kotlin.concurrent.schedule
import kotlin.concurrent.write
fun nowSeconds() = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime())
@@ -49,9 +49,16 @@ class CompileServiceImpl(
port: Int
) : CompileService {
private val classpathWatcher = LazyClasspathWatcher(selfCompilerId.compilerClasspath)
// RMI-exposed API
override fun getCompilerId(): CompilerId = ifAlive { selfCompilerId }
override fun checkCompilerId(compilerId: CompilerId): Boolean =
ifAlive {
(selfCompilerId.compilerVersion.isEmpty() || selfCompilerId.compilerVersion == compilerId.compilerVersion) &&
(selfCompilerId.compilerClasspath.all { compilerId.compilerClasspath.contains(it) }) &&
!classpathWatcher.isChanged
}
override fun getUsedMemory(): Long = ifAlive { usedMemory(withGC = true) }
@@ -146,6 +153,7 @@ class CompileServiceImpl(
operationsTracer: RemoteOperationsTracer?,
body: (PrintStream, Profiler) -> ExitCode): Int =
ifAlive {
operationsTracer?.before("compile")
val rpcProfiler = if (daemonOptions.reportPerf) WallAndThreadTotalProfiler() else DummyProfiler()
val compilerMessagesStream = PrintStream(BufferedOutputStream(RemoteOutputStreamClient(compilerMessagesStreamProxy, rpcProfiler), 4096))
compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/LazyClasspathWatcher.kt
+128
View File
@@ -0,0 +1,128 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.rmi.service
import java.io.File
import java.io.IOException
import java.security.DigestInputStream
import java.security.MessageDigest
import java.util.*
import java.util.concurrent.Semaphore
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicLong
import java.util.logging.Level
import java.util.logging.Logger
import kotlin.concurrent.thread
val CLASSPATH_FILE_ID_DIGEST = "MD5"
val DEFAULT_CLASSPATH_WATCH_PERIOD_MS = 1000L
val DEFAULT_CLASSPATH_DIGEST_WATCH_PERIOD_MS = 300000L // 5 min
/**
* Class for lazy (on demand) check if any relevant file in the classpath is changed
* poor-man watcher in the absence of NIO
* TODO: replace with NIO watching when switching to java 7+
*/
public class LazyClasspathWatcher(classpath: Iterable<String>,
val chackPeriod: Long = DEFAULT_CLASSPATH_WATCH_PERIOD_MS,
val digestCheckPeriod: Long = DEFAULT_CLASSPATH_DIGEST_WATCH_PERIOD_MS) {
private data class FileId(val file: File, val lastModified: Long, val digest: ByteArray)
private val fileIdsLock = Semaphore(1) // a barrier for ensuring ids are initialized, using semaphore to allow modifications from another thread
private var fileIds: ArrayList<FileId>? = null
private val lastChangedStatus = AtomicBoolean(false)
private val lastUpdate = AtomicLong(0)
private val lastDigestUpdate = AtomicLong(0)
private val log by lazy { Logger.getLogger("classpath watcher") }
init {
// locking before entering thread in order to avoid racing with isChanged
fileIdsLock.acquire()
thread(daemon = true, start = true) {
try {
fileIds = classpath
.map { File(it) }
.asSequence()
.flatMap { FileTreeWalk(it, filter = ::isClasspathFile) }
.map { FileId(it, it.lastModified(), it.md5Digest()) }
.toArrayList()
val nowMs = TimeUnit.MILLISECONDS.toMillis(System.nanoTime())
lastUpdate.set(nowMs)
lastDigestUpdate.set(nowMs)
}
catch (e: IOException) {
log.log(Level.WARNING, "Error on walking classpath", e)
// ignoring it for now
}
finally {
fileIdsLock.release()
}
}
}
public val isChanged: Boolean get() {
if (lastChangedStatus.get()) return true
val nowMs = TimeUnit.MILLISECONDS.toMillis(System.nanoTime())
if (nowMs - lastUpdate.get() < chackPeriod) return false
val checkDigest = nowMs - lastDigestUpdate.get() > digestCheckPeriod
// making sure that fieldIds are initialized
fileIdsLock.acquire()
fileIdsLock.release()
val changed =
fileIds?.find {
try {
if (!it.file.exists()) {
log.info("cp changed: ${it.file} doesn't exist any more")
true
}
// if last modified changed or if enforced by param - checking the digest
else if ((it.file.lastModified() != it.lastModified || checkDigest) && !Arrays.equals(it.digest, it.file.md5Digest())) {
log.info("cp changed: ${it.file} digests differ")
true
}
else false
}
catch (e: IOException) {
log.log(Level.INFO, "cp changed: ${it.file} access throws the exception", e)
true // io error considered as change
}
} != null
lastUpdate.set(TimeUnit.MILLISECONDS.toMillis(System.nanoTime()))
if (checkDigest) lastDigestUpdate.set(lastUpdate.get())
return changed
}
}
fun isClasspathFile(file: File): Boolean = file.isFile && listOf("class", "jar").contains(file.extension.toLowerCase())
fun File.md5Digest(): ByteArray {
val md = MessageDigest.getInstance(CLASSPATH_FILE_ID_DIGEST)
DigestInputStream(inputStream(), md).use {
val buf = ByteArray(1024)
while (it.read(buf) != -1) {}
it.close()
}
return md.digest()
}