From 89791dc85c57e15f257a5f25eb58b12add9855f5 Mon Sep 17 00:00:00 2001 From: Zalim Bashorov Date: Fri, 13 Nov 2015 22:35:10 +0300 Subject: [PATCH] Batch recording lookups in daemon --- .../CompilerCallbackServicesFacadeServer.kt | 11 +++--- .../rmi/CompilerCallbackServicesFacade.kt | 8 ++--- .../kotlin/rmi/service/CompileServiceImpl.kt | 34 ++++++++++++++----- ...eIncrementalCompilationComponentsClient.kt | 5 +-- .../rmi/service/RemoteLookupTrackerClient.kt | 25 +++++++++++--- .../incremental/components/LookupTracker.kt | 10 ++++++ .../jps/build/AbstractLookupTrackerTest.kt | 19 +++-------- 7 files changed, 75 insertions(+), 37 deletions(-) diff --git a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/CompilerCallbackServicesFacadeServer.kt b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/CompilerCallbackServicesFacadeServer.kt index 43dce9b8964..a872da2bed1 100644 --- a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/CompilerCallbackServicesFacadeServer.kt +++ b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/CompilerCallbackServicesFacadeServer.kt @@ -16,9 +16,8 @@ package org.jetbrains.kotlin.rmi.kotlinr +import org.jetbrains.kotlin.incremental.components.LookupInfo import org.jetbrains.kotlin.incremental.components.LookupTracker -import org.jetbrains.kotlin.incremental.components.Position -import org.jetbrains.kotlin.incremental.components.ScopeKind import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents import org.jetbrains.kotlin.load.kotlin.incremental.components.JvmPackagePartProto import org.jetbrains.kotlin.modules.TargetId @@ -67,8 +66,12 @@ public class CompilerCallbackServicesFacadeServer( override fun lookupTracker_requiresPosition() = incrementalCompilationComponents!!.getLookupTracker().requiresPosition - override fun lookupTracker_record(filePath: String, position: Position, scopeFqName: String, scopeKind: ScopeKind, name: String) { - incrementalCompilationComponents!!.getLookupTracker().record(filePath, position, scopeFqName, scopeKind, name) + override fun lookupTracker_record(lookups: Collection) { + val lookupTracker = incrementalCompilationComponents!!.getLookupTracker() + + for (it in lookups) { + lookupTracker.record(it.filePath, it.position, it.scopeFqName, it.scopeKind, it.name) + } } private val lookupTracker_isDoNothing: Boolean = incrementalCompilationComponents != null && incrementalCompilationComponents.getLookupTracker() == LookupTracker.DO_NOTHING diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerCallbackServicesFacade.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerCallbackServicesFacade.kt index 97bcaedd617..a38641cb830 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerCallbackServicesFacade.kt +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/CompilerCallbackServicesFacade.kt @@ -16,8 +16,7 @@ package org.jetbrains.kotlin.rmi -import org.jetbrains.kotlin.incremental.components.Position -import org.jetbrains.kotlin.incremental.components.ScopeKind +import org.jetbrains.kotlin.incremental.components.LookupInfo import org.jetbrains.kotlin.load.kotlin.incremental.components.JvmPackagePartProto import org.jetbrains.kotlin.modules.TargetId import java.rmi.Remote @@ -74,11 +73,12 @@ public interface CompilerCallbackServicesFacade : Remote { @Throws(RemoteException::class) fun lookupTracker_requiresPosition(): Boolean - fun lookupTracker_record(filePath: String, position: Position, scopeFqName: String, scopeKind: ScopeKind, name: String) + @Throws(RemoteException::class) + fun lookupTracker_record(lookups: Collection) @Throws(RemoteException::class) fun lookupTracker_isDoNothing(): Boolean - + // ---------------------------------------------------- // CompilationCanceledStatus @Throws(RemoteException::class) diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt index 41c5c0cdbbf..16ee62e0dfb 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileServiceImpl.kt @@ -39,6 +39,22 @@ interface CompilerSelector { operator fun get(targetPlatform: CompileService.TargetPlatform): CLICompiler<*> } +interface EventManger { + fun onCompilationFinished(f : () -> Unit) +} + +private class EventMangerImpl : EventManger { + private val onCompilationFinished = arrayListOf<() -> Unit>() + + override fun onCompilationFinished(f: () -> Unit) { + onCompilationFinished.add(f) + } + + fun fireCompilationFinished() { + onCompilationFinished.forEach { it() } + } +} + class CompileServiceImpl( val registry: Registry, val compiler: CompilerSelector, @@ -79,10 +95,10 @@ class CompileServiceImpl( serviceOutputStream: RemoteOutputStream, operationsTracer: RemoteOperationsTracer? ): Int = - doCompile(args, compilerOutputStream, serviceOutputStream, operationsTracer) { printStream, profiler -> + doCompile(args, compilerOutputStream, serviceOutputStream, operationsTracer) { printStream, eventManager, profiler -> when (outputFormat) { CompileService.OutputFormat.PLAIN -> compiler[targetPlatform].exec(printStream, *args) - CompileService.OutputFormat.XML -> compiler[targetPlatform].execAndOutputXml(printStream, createCompileServices(servicesFacade, profiler), *args) + CompileService.OutputFormat.XML -> compiler[targetPlatform].execAndOutputXml(printStream, createCompileServices(servicesFacade, eventManager, profiler), *args) } } @@ -94,10 +110,10 @@ class CompileServiceImpl( serviceOutputStream: RemoteOutputStream, operationsTracer: RemoteOperationsTracer? ): Int = - doCompile(args, compilerOutputStream, serviceOutputStream, operationsTracer) { printStream, profiler -> + doCompile(args, compilerOutputStream, serviceOutputStream, operationsTracer) { printStream, eventManager, profiler -> when (compilerOutputFormat) { CompileService.OutputFormat.PLAIN -> throw NotImplementedError("Only XML output is supported in remote incremental compilation") - CompileService.OutputFormat.XML -> compiler[targetPlatform].execAndOutputXml(printStream, createCompileServices(servicesFacade, profiler), *args) + CompileService.OutputFormat.XML -> compiler[targetPlatform].execAndOutputXml(printStream, createCompileServices(servicesFacade, eventManager, profiler), *args) } } @@ -143,16 +159,17 @@ class CompileServiceImpl( compilerMessagesStreamProxy: RemoteOutputStream, serviceOutputStreamProxy: RemoteOutputStream, operationsTracer: RemoteOperationsTracer?, - body: (PrintStream, Profiler) -> ExitCode): Int = + body: (PrintStream, EventManger, Profiler) -> ExitCode): Int = ifAlive { operationsTracer?.before("compile") val rpcProfiler = if (daemonOptions.reportPerf) WallAndThreadTotalProfiler() else DummyProfiler() + val eventManger = EventMangerImpl() val compilerMessagesStream = PrintStream(BufferedOutputStream(RemoteOutputStreamClient(compilerMessagesStreamProxy, rpcProfiler), 4096)) val serviceOutputStream = PrintStream(BufferedOutputStream(RemoteOutputStreamClient(serviceOutputStreamProxy, rpcProfiler), 4096)) try { checkedCompile(args, serviceOutputStream, rpcProfiler) { - val res = body(compilerMessagesStream, rpcProfiler).code + val res = body(compilerMessagesStream, eventManger, rpcProfiler).code _lastUsedSeconds = nowSeconds() res } @@ -160,14 +177,15 @@ class CompileServiceImpl( finally { serviceOutputStream.flush() compilerMessagesStream.flush() + eventManger.fireCompilationFinished() operationsTracer?.after("compile") } } - private fun createCompileServices(facade: CompilerCallbackServicesFacade, rpcProfiler: Profiler): Services { + private fun createCompileServices(facade: CompilerCallbackServicesFacade, eventManger: EventManger, rpcProfiler: Profiler): Services { val builder = Services.Builder() if (facade.hasIncrementalCaches() || facade.hasLookupTracker()) { - builder.register(IncrementalCompilationComponents::class.java, RemoteIncrementalCompilationComponentsClient(facade, rpcProfiler)) + builder.register(IncrementalCompilationComponents::class.java, RemoteIncrementalCompilationComponentsClient(facade, eventManger, rpcProfiler)) } if (facade.hasCompilationCanceledStatus()) { builder.register(CompilationCanceledStatus::class.java, RemoteCompilationCanceledStatusClient(facade, rpcProfiler)) diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCompilationComponentsClient.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCompilationComponentsClient.kt index 7af286c8327..771c3ce900c 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCompilationComponentsClient.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteIncrementalCompilationComponentsClient.kt @@ -25,9 +25,10 @@ import org.jetbrains.kotlin.rmi.DummyProfiler import org.jetbrains.kotlin.rmi.Profiler -class RemoteIncrementalCompilationComponentsClient(val facade: CompilerCallbackServicesFacade, val profiler: Profiler = DummyProfiler()) : IncrementalCompilationComponents { +class RemoteIncrementalCompilationComponentsClient(val facade: CompilerCallbackServicesFacade, eventManger: EventManger, val profiler: Profiler = DummyProfiler()) : IncrementalCompilationComponents { + val remoteLookupTrackerClient = RemoteLookupTrackerClient(facade, eventManger, profiler) override fun getIncrementalCache(target: TargetId): IncrementalCache = RemoteIncrementalCacheClient(facade, target, profiler) - override fun getLookupTracker(): LookupTracker = RemoteLookupTrackerClient(facade, profiler) + override fun getLookupTracker(): LookupTracker = remoteLookupTrackerClient } diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteLookupTrackerClient.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteLookupTrackerClient.kt index 6d94fb3c59a..eb11eddf7fa 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteLookupTrackerClient.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/RemoteLookupTrackerClient.kt @@ -16,6 +16,7 @@ package org.jetbrains.kotlin.rmi.service +import org.jetbrains.kotlin.incremental.components.LookupInfo import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.components.Position import org.jetbrains.kotlin.incremental.components.ScopeKind @@ -24,16 +25,30 @@ import org.jetbrains.kotlin.rmi.DummyProfiler import org.jetbrains.kotlin.rmi.Profiler -class RemoteLookupTrackerClient(val facade: CompilerCallbackServicesFacade, val profiler: Profiler = DummyProfiler()) : LookupTracker { +class RemoteLookupTrackerClient(val facade: CompilerCallbackServicesFacade, val eventManger: EventManger, val profiler: Profiler = DummyProfiler()) : LookupTracker { private val isDoNothing = profiler.withMeasure(this) { facade.lookupTracker_isDoNothing() } + private val lookups = hashSetOf() + override val requiresPosition: Boolean = profiler.withMeasure(this) { facade.lookupTracker_requiresPosition() } override fun record(filePath: String, position: Position, scopeFqName: String, scopeKind: ScopeKind, name: String) { - if (!isDoNothing) { - profiler.withMeasure(this) { - facade.lookupTracker_record(filePath, position, scopeFqName, scopeKind, name) - } + if (isDoNothing) return + + lookups.add(LookupInfo(filePath, position, scopeFqName, scopeKind, name)) + } + + init { + eventManger.onCompilationFinished { flush() } + } + + private fun flush() { + if (isDoNothing || lookups.isEmpty()) return + + profiler.withMeasure(this) { + facade.lookupTracker_record(lookups) } + + lookups.clear() } } diff --git a/core/descriptors/src/org/jetbrains/kotlin/incremental/components/LookupTracker.kt b/core/descriptors/src/org/jetbrains/kotlin/incremental/components/LookupTracker.kt index 8ee0b7ec97a..09c11e5468f 100644 --- a/core/descriptors/src/org/jetbrains/kotlin/incremental/components/LookupTracker.kt +++ b/core/descriptors/src/org/jetbrains/kotlin/incremental/components/LookupTracker.kt @@ -16,6 +16,8 @@ package org.jetbrains.kotlin.incremental.components +import java.io.Serializable + public interface LookupTracker { // used in tests for more accurate checks val requiresPosition: Boolean @@ -43,3 +45,11 @@ public enum class ScopeKind { PACKAGE, CLASSIFIER } + +data class LookupInfo( + val filePath: String, + val position: Position, + val scopeFqName: String, + val scopeKind: ScopeKind, + val name: String +) : Serializable diff --git a/jps-plugin/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt b/jps-plugin/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt index 597c04cf891..f9f40951b8a 100644 --- a/jps-plugin/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt +++ b/jps-plugin/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt @@ -17,6 +17,7 @@ package org.jetbrains.kotlin.jps.build import com.intellij.openapi.util.io.FileUtil +import org.jetbrains.kotlin.incremental.components.LookupInfo import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.components.Position import org.jetbrains.kotlin.incremental.components.ScopeKind @@ -37,7 +38,7 @@ abstract class AbstractLookupTrackerTest : AbstractIncrementalJpsTest( override fun checkLookups(lookupTracker: LookupTracker, compiledFiles: Set) { if (lookupTracker !is TestLookupTracker) throw AssertionError("Expected TestLookupTracker, but: ${lookupTracker.javaClass}") - val fileToLookups = lookupTracker.lookups.groupBy { it.lookupContainingFile } + val fileToLookups = lookupTracker.lookups.groupBy { it.filePath } fun checkLookupsInFile(expectedFile: File, actualFile: File) { @@ -53,8 +54,8 @@ abstract class AbstractLookupTrackerTest : AbstractIncrementalJpsTest( val lines = text.lines().toArrayList() - for ((line, lookupsFromLine) in lookupsFromFile.groupBy { it.lookupLine }) { - val columnToLookups = lookupsFromLine.groupBy { it.lookupColumn }.toList().sortedBy { it.first } + for ((line, lookupsFromLine) in lookupsFromFile.groupBy { it.position.line }) { + val columnToLookups = lookupsFromLine.groupBy { it.position.column }.toList().sortedBy { it.first } val lineContent = lines[line - 1] val parts = ArrayList(columnToLookups.size * 2) @@ -123,18 +124,8 @@ class TestLookupTracker : LookupTracker { get() = true override fun record(filePath: String, position: Position, scopeFqName: String, scopeKind: ScopeKind, name: String) { - val (line, column) = position - lookups.add(LookupInfo(filePath, line, column, scopeFqName, scopeKind, name)) + lookups.add(LookupInfo(filePath, position, scopeFqName, scopeKind, name)) } - - data class LookupInfo( - val lookupContainingFile: String, - val lookupLine: Int, - val lookupColumn: Int, - val scopeFqName: String, - val scopeKind: ScopeKind, - val name: String - ) }