Fix multiproject Gradle IC with daemon

compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/IncrementalCompilationServicesFacade.kt

@@ -17,6 +17,7 @@
 package org.jetbrains.kotlin.daemon.common
 
 import java.io.File
+import java.io.Serializable
 import java.rmi.Remote
 import java.rmi.RemoteException
 
@@ -53,4 +54,19 @@ interface IncrementalCompilationServicesFacade : Remote {
 
     @Throws(RemoteException::class)
     fun revert()
+
+    // ChangesRegistry
+    @Throws(RemoteException::class)
+    fun registerChanges(timestamp: Long, dirtyData: SimpleDirtyData)
+
+    @Throws(RemoteException::class)
+    fun unknownChanges(timestamp: Long)
+
+    @Throws(RemoteException::class)
+    fun getChanges(artifact: File, sinceTS: Long): Iterable<SimpleDirtyData>?
 }
+
+class SimpleDirtyData(
+        val dirtyLookupSymbols: List<String>,
+        val dirtyClassesFqNames: List<String>
+) : Serializable

compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt

@@ -36,7 +36,10 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.config.IncrementalCompilation
 import org.jetbrains.kotlin.config.Services
 import org.jetbrains.kotlin.daemon.common.*
+import org.jetbrains.kotlin.daemon.incremental.*
 import org.jetbrains.kotlin.incremental.*
+import org.jetbrains.kotlin.incremental.multiproject.ArtifactChangesProvider
+import org.jetbrains.kotlin.incremental.multiproject.ChangesRegistry
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
 import org.jetbrains.kotlin.modules.Module
 import org.jetbrains.kotlin.name.FqName
@@ -355,12 +358,16 @@ class CompileServiceImpl(
                 ChangedFiles.Unknown()
             }
 
+            val artifactChanges = RemoteArtifactChangesProvider(servicesFacade)
+            val changesRegistry = RemoteChangesRegostry(servicesFacade)
+
             val workingDir = servicesFacade.workingDir()
             val versions = commonCacheVersions(workingDir) + standaloneCacheVersion(workingDir, forceEnable = true)
 
             try {
                 printStream.print(renderer.renderPreamble())
-                IncrementalJvmCompilerRunner(workingDir, javaSourceRoots, versions, reporter, annotationFileUpdater)
+                IncrementalJvmCompilerRunner(workingDir, javaSourceRoots, versions, reporter, annotationFileUpdater,
+                                             artifactChanges, changesRegistry)
                         .compile(allKotlinFiles, k2jvmArgs, messageCollector, { changedFiles })
             }
             finally {

compiler/daemon/src/org/jetbrains/kotlin/daemon/incremental/RemoteAnnotationsFileUpdater.kt

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.jetbrains.kotlin.daemon
+package org.jetbrains.kotlin.daemon.incremental
 
 import org.jetbrains.kotlin.annotation.AnnotationFileUpdater
 import org.jetbrains.kotlin.cli.common.ExitCode

compiler/daemon/src/org/jetbrains/kotlin/daemon/incremental/RemoteArtifactChangesProvider.kt

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.daemon.incremental
+
+import org.jetbrains.kotlin.daemon.common.IncrementalCompilationServicesFacade
+import org.jetbrains.kotlin.incremental.DirtyData
+import org.jetbrains.kotlin.incremental.multiproject.ArtifactChangesProvider
+import java.io.File
+
+class RemoteArtifactChangesProvider(private val servicesFacade: IncrementalCompilationServicesFacade) : ArtifactChangesProvider {
+    override fun getChanges(artifact: File, sinceTS: Long): Iterable<DirtyData>? =
+            servicesFacade.getChanges(artifact, sinceTS)?.map { it.toDirtyData() }
+}

compiler/daemon/src/org/jetbrains/kotlin/daemon/incremental/RemoteChangesRegostry.kt

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.daemon.incremental
+
+import org.jetbrains.kotlin.daemon.common.IncrementalCompilationServicesFacade
+import org.jetbrains.kotlin.incremental.DirtyData
+import org.jetbrains.kotlin.incremental.multiproject.ChangesRegistry
+
+internal class RemoteChangesRegostry(private val servicesFacade: IncrementalCompilationServicesFacade) : ChangesRegistry {
+    override fun unknownChanges(timestamp: Long) {
+        servicesFacade.unknownChanges(timestamp)
+    }
+
+    override fun registerChanges(timestamp: Long, dirtyData: DirtyData) {
+        servicesFacade.registerChanges(timestamp, dirtyData.toSimpleDirtyData())
+    }
+}

compiler/daemon/src/org/jetbrains/kotlin/daemon/incremental/RemoteICReporter.kt

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.jetbrains.kotlin.daemon
+package org.jetbrains.kotlin.daemon.incremental
 
 import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.daemon.common.IncrementalCompilationServicesFacade

compiler/daemon/src/org/jetbrains/kotlin/daemon/incremental/dirtyDataUtils.kt

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.daemon.incremental
+
+import org.jetbrains.kotlin.daemon.common.SimpleDirtyData
+import org.jetbrains.kotlin.incremental.DirtyData
+import org.jetbrains.kotlin.incremental.LookupSymbol
+import org.jetbrains.kotlin.name.FqName
+
+fun SimpleDirtyData.toDirtyData(): DirtyData {
+    val dirtyClassesFqNames = dirtyClassesFqNames.map(::FqName)
+    val dirtyLookupSymbols = dirtyLookupSymbols.map {
+        LookupSymbol(scope = it.substringBeforeLast("."), name = it.substringAfterLast("."))
+    }
+    return DirtyData(dirtyLookupSymbols, dirtyClassesFqNames)
+}
+
+fun DirtyData.toSimpleDirtyData(): SimpleDirtyData {
+    return SimpleDirtyData(dirtyClassesFqNames = dirtyClassesFqNames.map(FqName::asString),
+                           dirtyLookupSymbols = dirtyLookupSymbols.map { "${it.scope}.${it.name}" })
+}

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt

@@ -32,8 +32,8 @@ import org.jetbrains.kotlin.compilerRunner.OutputItemsCollector
 import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl
 import org.jetbrains.kotlin.config.IncrementalCompilation
 import org.jetbrains.kotlin.incremental.components.LookupTracker
-import org.jetbrains.kotlin.incremental.multiproject.ArtifactDifference
-import org.jetbrains.kotlin.incremental.multiproject.ArtifactDifferenceRegistryProvider
+import org.jetbrains.kotlin.incremental.multiproject.ArtifactChangesProvider
+import org.jetbrains.kotlin.incremental.multiproject.ChangesRegistry
 import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
 import org.jetbrains.kotlin.modules.TargetId
 import org.jetbrains.kotlin.name.FqName
@@ -92,8 +92,8 @@ class IncrementalJvmCompilerRunner(
         private val cacheVersions: List<CacheVersion>,
         private val reporter: ICReporter,
         private var kaptAnnotationsFileUpdater: AnnotationFileUpdater? = null,
-        private val artifactDifferenceRegistryProvider: ArtifactDifferenceRegistryProvider? = null,
-        private val artifactFile: File? = null
+        private val artifactChangesProvider: ArtifactChangesProvider? = null,
+        private val changesRegistry: ChangesRegistry? = null
 ) {
     var anyClassesCompiled: Boolean = false
             private set
@@ -112,7 +112,6 @@ class IncrementalJvmCompilerRunner(
 
         fun onError(e: Exception): ExitCode {
             caches.clean()
-            artifactDifferenceRegistryProvider?.clean()
 
             // todo: warn?
             reporter.report { "Possible cache corruption. Rebuilding. $e" }
@@ -135,9 +134,6 @@ class IncrementalJvmCompilerRunner(
             onError(e)
         }
         finally {
-            artifactDifferenceRegistryProvider?.withRegistry(reporter) {
-                it.flush(memoryCachesOnly = true)
-            }
             caches.close(flush = true)
             reporter.report { "flushed incremental caches" }
         }
@@ -227,10 +223,6 @@ class IncrementalJvmCompilerRunner(
             reporter.report {"No classpath changes"}
             return ChangesEither.Known()
         }
-        if (artifactDifferenceRegistryProvider == null) {
-            reporter.report {"No artifact history provider"}
-            return ChangesEither.Unknown()
-        }
 
         val lastBuildTS = lastBuildInfo?.startTS
         if (lastBuildTS == null) {
@@ -241,23 +233,16 @@ class IncrementalJvmCompilerRunner(
         val symbols = HashSet<LookupSymbol>()
         val fqNames = HashSet<FqName>()
         for (file in modifiedClasspath) {
-            val diffs = artifactDifferenceRegistryProvider.withRegistry(reporter) {artifactDifferenceRegistry ->
-                artifactDifferenceRegistry[file]
-            }
+            val diffs = artifactChangesProvider?.getChanges(file, lastBuildTS)
+
             if (diffs == null) {
                 reporter.report {"Could not get changes for file: $file"}
                 return ChangesEither.Unknown()
             }
 
-            val (beforeLastBuild, afterLastBuild) = diffs.partition {it.buildTS < lastBuildTS}
-            if (beforeLastBuild.isEmpty()) {
-                reporter.report {"No known build preceding timestamp $lastBuildTS for file $file"}
-                return ChangesEither.Unknown()
-            }
-
-            afterLastBuild.forEach {
-                symbols.addAll(it.dirtyData.dirtyLookupSymbols)
-                fqNames.addAll(it.dirtyData.dirtyClassesFqNames)
+            diffs.forEach {
+                symbols.addAll(it.dirtyLookupSymbols)
+                fqNames.addAll(it.dirtyClassesFqNames)
             }
         }
 
@@ -295,7 +280,6 @@ class IncrementalJvmCompilerRunner(
         @Suppress("NAME_SHADOWING")
         var compilationMode = compilationMode
 
-        reporter.report { "Artifact to register difference for: $artifactFile" }
         val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis())
         BuildInfo.write(currentBuildInfo, lastBuildInfoFile)
         val buildDirtyLookupSymbols = HashSet<LookupSymbol>()
@@ -347,11 +331,6 @@ class IncrementalJvmCompilerRunner(
             caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources)
 
             if (compilationMode is CompilationMode.Rebuild) {
-                artifactFile?.let { artifactFile ->
-                    artifactDifferenceRegistryProvider?.withRegistry(reporter) { registry ->
-                        registry.remove(artifactFile)
-                    }
-                }
                 break
             }
 
@@ -371,16 +350,14 @@ class IncrementalJvmCompilerRunner(
         if (exitCode == ExitCode.OK && compilationMode is CompilationMode.Incremental) {
             buildDirtyLookupSymbols.addAll(javaFilesProcessor.allChangedSymbols)
         }
-        if (artifactFile != null && artifactDifferenceRegistryProvider != null) {
-            val dirtyData = DirtyData(buildDirtyLookupSymbols, buildDirtyFqNames)
-            val artifactDifference = ArtifactDifference(currentBuildInfo.startTS, dirtyData)
-            artifactDifferenceRegistryProvider.withRegistry(reporter) {registry ->
-                registry.add(artifactFile, artifactDifference)
+        if (changesRegistry != null) {
+            if (compilationMode is CompilationMode.Incremental) {
+                val dirtyData = DirtyData(buildDirtyLookupSymbols, buildDirtyFqNames)
+                changesRegistry.registerChanges(currentBuildInfo.startTS, dirtyData)
             }
-            reporter.report {
-                val dirtySymbolsSorted = buildDirtyLookupSymbols.map { it.scope + "#" + it.name }.sorted()
-                "Added artifact difference for $artifactFile (ts: ${currentBuildInfo.startTS}): " +
-                        "[\n\t${dirtySymbolsSorted.joinToString(",\n\t")}]"
+            else {
+                assert(compilationMode is CompilationMode.Rebuild) { "Unexpected compilation mode: ${compilationMode.javaClass}" }
+                changesRegistry.unknownChanges(currentBuildInfo.startTS)
             }
         }
 

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/multiproject/ArtifactChangesProvider.kt

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental.multiproject
+
+import org.jetbrains.kotlin.incremental.DirtyData
+import java.io.File
+
+interface ArtifactChangesProvider {
+    fun getChanges(artifact: File, sinceTS: Long): Iterable<DirtyData>?
+}

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/multiproject/ChangesRegistry.kt

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental.multiproject
+
+import org.jetbrains.kotlin.incremental.DirtyData
+
+interface ChangesRegistry {
+    fun registerChanges(timestamp: Long, dirtyData: DirtyData)
+    fun unknownChanges(timestamp: Long)
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerEnvironment.kt

@@ -1,10 +1,13 @@
 package org.jetbrains.kotlin.compilerRunner
 
+import org.gradle.api.internal.artifacts.dsl.ArtifactFile
 import org.jetbrains.kotlin.annotation.AnnotationFileUpdater
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.config.Services
 import org.jetbrains.kotlin.incremental.ChangedFiles
 import org.jetbrains.kotlin.incremental.GradleICReporter
+import org.jetbrains.kotlin.incremental.multiproject.ArtifactDifferenceRegistry
+import org.jetbrains.kotlin.incremental.multiproject.ArtifactDifferenceRegistryProvider
 import java.io.File
 import java.net.URL
 
@@ -27,5 +30,7 @@ internal class GradleIncrementalCompilerEnvironment(
         val workingDir: File,
         messageCollector: MessageCollector,
         outputItemsCollector: OutputItemsCollector,
-        val kaptAnnotationsFileUpdater: AnnotationFileUpdater?
+        val kaptAnnotationsFileUpdater: AnnotationFileUpdater?,
+        val artifactDifferenceRegistryProvider: ArtifactDifferenceRegistryProvider?,
+        val artifactFile: File?
 ) : GradleCompilerEnvironment(compilerJar, messageCollector, outputItemsCollector)

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/IncrementalCompilationFacadeImpl.kt

@@ -20,7 +20,12 @@ import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.daemon.common.IncrementalCompilationServicesFacade
 import org.jetbrains.kotlin.daemon.common.LoopbackNetworkInterface
 import org.jetbrains.kotlin.daemon.common.SOCKET_ANY_FREE_PORT
+import org.jetbrains.kotlin.daemon.common.SimpleDirtyData
+import org.jetbrains.kotlin.daemon.incremental.toDirtyData
+import org.jetbrains.kotlin.daemon.incremental.toSimpleDirtyData
 import org.jetbrains.kotlin.incremental.ChangedFiles
+import org.jetbrains.kotlin.incremental.DirtyData
+import org.jetbrains.kotlin.incremental.multiproject.ArtifactDifference
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName
 import java.io.File
 import java.rmi.server.UnicastRemoteObject
@@ -71,4 +76,32 @@ internal class IncrementalCompilationFacadeImpl(
         val jvmNames = outdatedClassesJvmNames.map { JvmClassName.byInternalName(it) }
         environment.kaptAnnotationsFileUpdater!!.updateAnnotations(jvmNames)
     }
+
+    override fun getChanges(artifact: File, sinceTS: Long): Iterable<SimpleDirtyData>? {
+        val artifactChanges = environment.artifactDifferenceRegistryProvider?.withRegistry(environment.reporter) { registry ->
+            registry[artifact]
+        } ?: return null
+
+        val (beforeLastBuild, afterLastBuild) = artifactChanges.partition { it.buildTS < sinceTS }
+        if (beforeLastBuild.isEmpty()) return null
+
+        return afterLastBuild.map { it.dirtyData.toSimpleDirtyData() }
+    }
+
+    override fun registerChanges(timestamp: Long, dirtyData: SimpleDirtyData) {
+        val artifactFile = environment.artifactFile ?: return
+
+        environment.artifactDifferenceRegistryProvider?.withRegistry(environment.reporter) { registry ->
+            registry.add(artifactFile, ArtifactDifference(timestamp, dirtyData.toDirtyData()))
+        }
+    }
+
+    override fun unknownChanges(timestamp: Long) {
+        val artifactFile = environment.artifactFile ?: return
+
+        environment.artifactDifferenceRegistryProvider?.withRegistry(environment.reporter) { registry ->
+            registry.remove(artifactFile)
+            registry.add(artifactFile, ArtifactDifference(timestamp, DirtyData()))
+        }
+    }
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt

@@ -45,6 +45,7 @@ import org.jetbrains.kotlin.gradle.plugin.kotlinDebug
 import org.jetbrains.kotlin.gradle.plugin.kotlinInfo
 import org.jetbrains.kotlin.gradle.utils.ParsedGradleVersion
 import org.jetbrains.kotlin.incremental.*
+import org.jetbrains.kotlin.incremental.multiproject.ArtifactDifferenceRegistry
 import org.jetbrains.kotlin.incremental.multiproject.ArtifactDifferenceRegistryProvider
 import org.jetbrains.kotlin.utils.LibraryUtils
 import java.io.File
@@ -204,25 +205,34 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
         args.destinationAsFile = destinationDir
         val outputItemCollector = OutputItemsCollectorImpl()
         val compilerRunner = GradleCompilerRunner(project)
+        val reporter = GradleICReporter(project.rootProject.projectDir)
 
         val environment = when {
             !incremental -> GradleCompilerEnvironment(compilerJar, messageCollector, outputItemCollector)
             else -> {
                 logger.warn(USING_EXPERIMENTAL_INCREMENTAL_MESSAGE)
-                val reporter = GradleICReporter(project.rootProject.projectDir)
                 GradleIncrementalCompilerEnvironment(compilerJar, changedFiles, reporter, taskBuildDirectory,
-                        messageCollector, outputItemCollector, kaptAnnotationsFileUpdater)
+                        messageCollector, outputItemCollector, kaptAnnotationsFileUpdater,
+                        artifactDifferenceRegistryProvider,
+                        artifactFile)
             }
         }
 
         try {
             val exitCode = compilerRunner.runJvmCompiler(sourceRoots.kotlinSourceFiles, sourceRoots.javaSourceRoots, args, environment)
             processCompilerExitCode(exitCode)
+            artifactDifferenceRegistryProvider?.withRegistry(reporter) {
+                it.flush(true)
+            }
         }
         catch (e: Throwable) {
             cleanupOnError()
+            artifactDifferenceRegistryProvider?.clean()
             throw e
         }
+        finally {
+            artifactDifferenceRegistryProvider?.withRegistry(reporter, ArtifactDifferenceRegistry::close)
+        }
         anyClassesCompiled = true
     }
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/multiproject/ArtifactDifferenceRegistry.kt

@@ -25,6 +25,7 @@ interface ArtifactDifferenceRegistry {
     fun add(artifact: File, difference: ArtifactDifference)
     fun remove(artifact: File)
     fun flush(memoryCachesOnly: Boolean)
+    fun close()
 }
 
 class ArtifactDifference(val buildTS: Long, val dirtyData: DirtyData)