Remove global artifact history cache in Gradle

Each Kotlin task now writes build history to separate file. A map of output directories to history files is used to get changes for modified files. #KT-22623 fixed
2018-05-16 16:18:45 +03:00
parent 598e89c03d
commit 6a45310830
31 changed files with 304 additions and 843 deletions
@@ -15,6 +15,7 @@ dependencies {
    compile(project(":compiler:frontend.java"))
    compile(project(":compiler:cli"))
    compile(project(":kotlin-build-common"))
+    compile(project(":compiler:daemon-common"))
    compileOnly(intellijCoreDep()) { includeJars("intellij-core") }
    compileOnly(intellijDep()) { includeJars("annotations") }

@@ -29,6 +29,11 @@ data class BuildDifference(val ts: Long, val isIncremental: Boolean, val dirtyDa
 data class BuildDiffsStorage(val buildDiffs: List<BuildDifference>) {
    companion object {
        fun readFromFile(file: File, reporter: ICReporter?): BuildDiffsStorage? {
+            val diffs = readDiffsFromFile(file, reporter)
+            return diffs?.let { BuildDiffsStorage(it) }
+        }
+
+        fun readDiffsFromFile(file: File, reporter: ICReporter?): MutableList<BuildDifference>? {
            fun reportFail(reason: String) {
                reporter?.report { "Could not read diff from file $file: $reason" }
            }
@@ -48,7 +53,7 @@ data class BuildDiffsStorage(val buildDiffs: List<BuildDifference>) {
                    repeat(size) {
                        result.add(input.readBuildDifference())
                    }
-                    return BuildDiffsStorage(result)
+                    return result
                }
            }
            catch (e: IOException) {
@@ -23,5 +23,5 @@ internal sealed class ChangesEither {
            val lookupSymbols: Collection<LookupSymbol> = emptyList(),
            val fqNames: Collection<FqName> = emptyList()
    ) : ChangesEither()
-    internal class Unknown : ChangesEither()
+    internal class Unknown(val reason: String? = null) : ChangesEither()
 }
@@ -27,8 +27,6 @@ import org.jetbrains.kotlin.compilerRunner.toGeneratedFile
 import org.jetbrains.kotlin.config.Services
 import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
 import org.jetbrains.kotlin.incremental.components.LookupTracker
-import org.jetbrains.kotlin.incremental.multiproject.ArtifactChangesProvider
-import org.jetbrains.kotlin.incremental.multiproject.ChangesRegistry
 import org.jetbrains.kotlin.incremental.parsing.classesFqNames
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.progress.CompilationCanceledStatus
@@ -43,8 +41,6 @@ abstract class IncrementalCompilerRunner<
        cacheDirName: String,
        protected val cacheVersions: List<CacheVersion>,
        protected val reporter: ICReporter,
-        protected val artifactChangesProvider: ArtifactChangesProvider?,
-        protected val changesRegistry: ChangesRegistry?,
        private val localStateDirs: Collection<File> = emptyList()
 ) {

@@ -307,16 +303,11 @@ abstract class IncrementalCompilerRunner<

    open fun runWithNoDirtyKotlinSources(caches: CacheManager): Boolean = false

-    protected open fun processChangesAfterBuild(compilationMode: CompilationMode, currentBuildInfo: BuildInfo, dirtyData: DirtyData) {
-        if (changesRegistry == null) return
-
-        if (compilationMode is CompilationMode.Incremental) {
-            changesRegistry.registerChanges(currentBuildInfo.startTS, dirtyData)
-        }
-        else {
-            assert(compilationMode is CompilationMode.Rebuild) { "Unexpected compilation mode: ${compilationMode::class.java}" }
-            changesRegistry.unknownChanges(currentBuildInfo.startTS)
-        }
+    protected open fun processChangesAfterBuild(
+        compilationMode: CompilationMode,
+        currentBuildInfo: BuildInfo,
+        dirtyData: DirtyData
+    ) {
    }

    companion object {
@@ -67,9 +67,7 @@ class IncrementalJsCompilerRunner(
        workingDir,
        "caches-js",
        cacheVersions,
-        reporter,
-        artifactChangesProvider = null,
-        changesRegistry = null
+        reporter
 ) {
    override fun isICEnabled(): Boolean =
        IncrementalCompilation.isEnabled() && IncrementalCompilation.isEnabledForJs()
@@ -18,7 +18,6 @@ package org.jetbrains.kotlin.incremental

 import com.intellij.lang.java.JavaLanguage
 import com.intellij.openapi.util.Disposer
-import com.intellij.openapi.util.io.FileUtil
 import com.intellij.psi.PsiClass
 import com.intellij.psi.PsiFile
 import com.intellij.psi.PsiFileFactory
@@ -39,8 +38,9 @@ import org.jetbrains.kotlin.config.IncrementalCompilation
 import org.jetbrains.kotlin.config.Services
 import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
 import org.jetbrains.kotlin.incremental.components.LookupTracker
-import org.jetbrains.kotlin.incremental.multiproject.ArtifactChangesProvider
-import org.jetbrains.kotlin.incremental.multiproject.ChangesRegistry
+import org.jetbrains.kotlin.incremental.multiproject.EmptyModulesApiHistory
+import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory
+import org.jetbrains.kotlin.incremental.util.Either
 import org.jetbrains.kotlin.load.java.JavaClassesTracker
 import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
@@ -64,6 +64,7 @@ fun makeIncrementally(
    val rootsWalk = sourceRoots.asSequence().flatMap { it.walk() }
    val files = rootsWalk.filter(File::isFile)
    val sourceFiles = files.filter { it.extension.toLowerCase() in allExtensions }.toList()
+    val buildHistoryFile = File(cachesDir, "build-history.bin")

    withIC {
        val compiler = IncrementalJvmCompilerRunner(
@@ -72,7 +73,9 @@ fun makeIncrementally(
                versions, reporter,
                // Use precise setting in case of non-Gradle build
                usePreciseJavaTracking = true,
-                localStateDirs = emptyList()
+                localStateDirs = emptyList(),
+                buildHistoryFile = buildHistoryFile,
+                modulesApiHistory = EmptyModulesApiHistory
        )
        compiler.compile(sourceFiles, args, messageCollector, providedChangedFiles = null)
    }
@@ -101,19 +104,15 @@ class IncrementalJvmCompilerRunner(
        cacheVersions: List<CacheVersion>,
        reporter: ICReporter,
        private var kaptAnnotationsFileUpdater: AnnotationFileUpdater? = null,
-        artifactChangesProvider: ArtifactChangesProvider? = null,
-        changesRegistry: ChangesRegistry? = null,
-        private val buildHistoryFile: File? = null,
-        private val friendBuildHistoryFile: File? = null,
        private val usePreciseJavaTracking: Boolean,
-        localStateDirs: Collection<File>
+        private val buildHistoryFile: File,
+        localStateDirs: Collection<File>,
+        private val modulesApiHistory: ModulesApiHistory
 ) : IncrementalCompilerRunner<K2JVMCompilerArguments, IncrementalJvmCachesManager>(
        workingDir,
        "caches-jvm",
        cacheVersions,
        reporter,
-        artifactChangesProvider,
-        changesRegistry,
        localStateDirs = localStateDirs
 ) {
    override fun isICEnabled(): Boolean =
@@ -166,41 +165,18 @@ class IncrementalJvmCompilerRunner(
        val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) ?: return CompilationMode.Rebuild { "No information on previous build" }
        reporter.report { "Last Kotlin Build info -- $lastBuildInfo" }

-        val changesFromFriend by lazy {
-            val myLastTS = lastBuildInfo.startTS 
-            val storage = friendBuildHistoryFile?.let { BuildDiffsStorage.readFromFile(it, reporter) } ?: return@lazy ChangesEither.Unknown()
+        val classpathChanges = getClasspathChanges(args.classpathAsList, changedFiles, lastBuildInfo)

-            val (prevDiffs, newDiffs) = storage.buildDiffs.partition { it.ts < myLastTS }
-            if (prevDiffs.isEmpty()) return@lazy ChangesEither.Unknown()
-
-            val dirtyLookupSymbols = HashSet<LookupSymbol>()
-            val dirtyClassesFqNames = HashSet<FqName>()
-            for ((_, isIncremental, dirtyData) in newDiffs) {
-                if (!isIncremental) return@lazy ChangesEither.Unknown()
-
-                dirtyLookupSymbols.addAll(dirtyData.dirtyLookupSymbols)
-                dirtyClassesFqNames.addAll(dirtyData.dirtyClassesFqNames)
+        @Suppress("UNUSED_VARIABLE") // for sealed when
+        val unused = when (classpathChanges) {
+            is ChangesEither.Unknown -> return CompilationMode.Rebuild {
+                // todo: we can recompile all files incrementally (not cleaning caches), so rebuild won't propagate
+                "Could not get classpath's changes${classpathChanges.reason?.let { ": $it" }}"
+            }
+            is ChangesEither.Known -> {
+                markDirtyBy(classpathChanges.lookupSymbols)
+                markDirtyBy(classpathChanges.fqNames)
            }
-
-            markDirtyBy(dirtyLookupSymbols)
-            markDirtyBy(dirtyClassesFqNames)
-            ChangesEither.Known(dirtyLookupSymbols, dirtyClassesFqNames)
-        }
-        val friendDirs = args.friendPaths?.map { File(it) } ?: emptyList()
-        for (file in changedFiles.removed.asSequence() + changedFiles.modified.asSequence()) {
-            if (!file.isClassFile()) continue
-
-            val isFriendClassFile = friendDirs.any { FileUtil.isAncestor(it, file, false) }
-            if (isFriendClassFile && changesFromFriend is ChangesEither.Known) continue
-
-            return CompilationMode.Rebuild { "Cannot get changes from modified or removed class file: ${reporter.pathsAsString(file)}" }
-        }
-
-        val classpathSet = args.classpathAsList.toHashSet()
-        val modifiedClasspathEntries = changedFiles.modified.filter { it in classpathSet }
-        val classpathChanges = getClasspathChanges(modifiedClasspathEntries, lastBuildInfo)
-        if (classpathChanges !is ChangesEither.Known) {
-            return CompilationMode.Rebuild { "could not get changes from modified classpath entries: ${reporter.pathsAsString(modifiedClasspathEntries)}" }
        }

        if (!usePreciseJavaTracking) {
@@ -221,8 +197,6 @@ class IncrementalJvmCompilerRunner(
        val removedClassesChanges = getRemovedClassesChanges(caches, changedFiles)

        markDirtyBy(androidLayoutChanges)
-        markDirtyBy(classpathChanges.lookupSymbols)
-        markDirtyBy(classpathChanges.fqNames)
        markDirtyBy(removedClassesChanges.dirtyLookupSymbols)
        markDirtyBy(removedClassesChanges.dirtyClassesFqNames)

@@ -285,33 +259,49 @@ class IncrementalJvmCompilerRunner(
    }

    private fun getClasspathChanges(
-            modifiedClasspath: List<File>,
-            lastBuildInfo: BuildInfo?
+        classpath: List<File>,
+        changedFiles: ChangedFiles.Known,
+        lastBuildInfo: BuildInfo
    ): ChangesEither {
-        if (modifiedClasspath.isEmpty()) {
-            reporter.report {"No classpath changes"}
-            return ChangesEither.Known()
+        val classpathSet = HashSet<File>()
+        for (file in classpath) {
+            when {
+                file.isFile -> classpathSet.add(file)
+                file.isDirectory -> file.walk().filterTo(classpathSet) { it.isFile }
+            }
        }

-        val lastBuildTS = lastBuildInfo?.startTS
-        if (lastBuildTS == null) {
-            reporter.report {"Could not determine last build timestamp"}
-            return ChangesEither.Unknown()
-        }
+        val modifiedClasspath = changedFiles.modified.filterTo(HashSet()) { it in classpathSet }
+        val removedClasspath = changedFiles.removed.filterTo(HashSet()) { it in classpathSet }
+
+        // todo: removed classes could be processed normally
+        if (removedClasspath.isNotEmpty()) return ChangesEither.Unknown("Some files are removed from classpath $removedClasspath")
+
+        if (modifiedClasspath.isEmpty()) return ChangesEither.Known()
+
+        val lastBuildTS = lastBuildInfo.startTS

        val symbols = HashSet<LookupSymbol>()
        val fqNames = HashSet<FqName>()
-        for (file in modifiedClasspath) {
-            val diffs = artifactChangesProvider?.getChanges(file, lastBuildTS)

-            if (diffs == null) {
-                reporter.report {"Could not get changes for file: $file"}
-                return ChangesEither.Unknown()
+        val historyFilesEither = modulesApiHistory.historyFilesForChangedFiles(modifiedClasspath)
+        val historyFiles = when (historyFilesEither) {
+            is Either.Success<Set<File>> -> historyFilesEither.value
+            is Either.Error -> return ChangesEither.Unknown(historyFilesEither.reason)
+        }
+
+        for (historyFile in historyFiles) {
+            val allBuilds = BuildDiffsStorage.readDiffsFromFile(historyFile, reporter = reporter)
+                    ?: return ChangesEither.Unknown("Could not read diffs from $historyFile")
+            val (knownBuilds, newBuilds) = allBuilds.partition { it.ts <= lastBuildTS }
+            if (knownBuilds.isEmpty()) {
+                return ChangesEither.Unknown("No previously known builds for $historyFile")
            }

-            diffs.forEach {
-                symbols.addAll(it.dirtyLookupSymbols)
-                fqNames.addAll(it.dirtyClassesFqNames)
+            for (buildDiff in newBuilds) {
+                val dirtyData = buildDiff.dirtyData
+                symbols.addAll(dirtyData.dirtyLookupSymbols)
+                fqNames.addAll(dirtyData.dirtyClassesFqNames)
            }
        }

@@ -400,16 +390,15 @@ class IncrementalJvmCompilerRunner(
    override fun additionalDirtyLookupSymbols(): Iterable<LookupSymbol> =
            javaFilesProcessor?.allChangedSymbols ?: emptyList()

-    override fun processChangesAfterBuild(compilationMode: CompilationMode, currentBuildInfo: BuildInfo, dirtyData: DirtyData) {
-        super.processChangesAfterBuild(compilationMode, currentBuildInfo, dirtyData)
-
-        if (buildHistoryFile == null) return
-
+    override fun processChangesAfterBuild(
+        compilationMode: CompilationMode,
+        currentBuildInfo: BuildInfo,
+        dirtyData: DirtyData
+    ) {
        val prevDiffs = BuildDiffsStorage.readFromFile(buildHistoryFile, reporter)?.buildDiffs ?: emptyList()
        val newDiff = if (compilationMode is CompilationMode.Incremental) {
            BuildDifference(currentBuildInfo.startTS, true, dirtyData)
-        }
-        else {
+        } else {
            val emptyDirtyData = DirtyData()
            BuildDifference(currentBuildInfo.startTS, false, emptyDirtyData)
        }
@@ -1,24 +0,0 @@
-/*
- * Copyright 2010-2016 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.jetbrains.kotlin.incremental.multiproject
-
-import org.jetbrains.kotlin.incremental.DirtyData
-import java.io.File
-
-interface ArtifactChangesProvider {
-    fun getChanges(artifact: File, sinceTS: Long): Iterable<DirtyData>?
-}
@@ -1,24 +0,0 @@
-/*
- * Copyright 2010-2016 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.jetbrains.kotlin.incremental.multiproject
-
-import org.jetbrains.kotlin.incremental.DirtyData
-
-interface ChangesRegistry {
-    fun registerChanges(timestamp: Long, dirtyData: DirtyData)
-    fun unknownChanges(timestamp: Long)
-}
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.incremental.multiproject
+
+import org.jetbrains.kotlin.daemon.common.IncrementalModuleInfo
+import org.jetbrains.kotlin.incremental.util.Either
+import java.io.File
+import java.nio.file.Path
+import java.nio.file.Paths
+import java.util.zip.ZipFile
+
+interface ModulesApiHistory {
+    fun historyFilesForChangedFiles(changedFiles: Set<File>): Either<Set<File>>
+}
+
+object EmptyModulesApiHistory : ModulesApiHistory {
+    override fun historyFilesForChangedFiles(changedFiles: Set<File>): Either<Set<File>> =
+        Either.Error("Multi-module IC is not configured")
+}
+
+class ModulesApiHistoryImpl(
+    private val modulesInfo: IncrementalModuleInfo
+) : ModulesApiHistory {
+    private val projectRootPath = Paths.get(modulesInfo.projectRoot.absolutePath)
+    private val dirToHistoryFileCache = HashMap<File, File?>()
+
+    override fun historyFilesForChangedFiles(changedFiles: Set<File>): Either<Set<File>> {
+        val result = HashSet<File>()
+        val jarFiles = ArrayList<File>()
+        val classFiles = ArrayList<File>()
+
+        for (file in changedFiles) {
+            val extension = file.extension
+
+            when {
+                extension.equals("class", ignoreCase = true) -> {
+                    classFiles.add(file)
+                }
+                extension.equals("jar", ignoreCase = true) -> {
+                    jarFiles.add(file)
+                }
+            }
+        }
+
+        for (jar in jarFiles) {
+            val historyEither = getBuildHistoryForJar(jar)
+            when (historyEither) {
+                is Either.Success<File> -> result.add(historyEither.value)
+                is Either.Error -> return historyEither
+            }
+        }
+
+        val classFileDirs = classFiles.groupBy { it.parentFile }
+        for ((dir, files) in classFileDirs) {
+            val buildHistory = getBuildHistoryForDir(dir)
+                    ?: return Either.Error("Could not get build history for class files: ${files.joinToString()}")
+            result.add(buildHistory)
+        }
+
+        return Either.Success(result)
+    }
+
+    private fun getBuildHistoryForDir(file: File): File? =
+        dirToHistoryFileCache.getOrPut(file) {
+            val module = modulesInfo.dirToModule[file]
+            val parent = file.parentFile
+
+            when {
+                module != null ->
+                    module.buildHistoryFile
+                parent != null && projectRootPath.isParentOf(parent) ->
+                    getBuildHistoryForDir(parent)
+                else ->
+                    null
+            }
+        }
+
+    private fun getBuildHistoryForJar(jar: File): Either<File> =
+        Either.Error("Cannot get changes for jar $jar")
+
+    private fun Path.isParentOf(path: Path) = path.startsWith(this)
+    private fun Path.isParentOf(file: File) = this.isParentOf(Paths.get(file.absolutePath))
+}
@@ -0,0 +1,11 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.incremental.util
+
+sealed class Either<out T> {
+    class Success<T>(val value: T) : Either<T>()
+    class Error(val reason: String) : Either<Nothing>()
+}