Gradle: compile tests incrementally when main is changed

#KT-17674 fixed
2017-09-15 16:35:30 +03:00
parent 779b9c6fcc
commit 62f293280c
12 changed files with 371 additions and 66 deletions
@@ -0,0 +1,132 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import org.jetbrains.annotations.TestOnly
+import org.jetbrains.kotlin.name.FqName
+import java.io.File
+import java.io.IOException
+import java.io.ObjectInputStream
+import java.io.ObjectOutputStream
+
+data class BuildDifference(val ts: Long, val isIncremental: Boolean, val dirtyData: DirtyData)
+
+// todo: storage format can be optimized by compressing fq-names
+data class BuildDiffsStorage(val buildDiffs: List<BuildDifference>) {
+    companion object {
+        fun readFromFile(file: File, reporter: ICReporter?): BuildDiffsStorage? {
+            fun reportFail(reason: String) {
+                reporter?.report { "Could not read diff from file $file: $reason" }
+            }
+
+            if (!file.exists()) return null
+
+            try {
+                ObjectInputStream(file.inputStream().buffered()).use { input ->
+                    val version = input.readInt()
+                    if (version != CURRENT_VERSION) {
+                        reportFail("incompatible version $version, actual version is $CURRENT_VERSION")
+                        return null
+                    }
+
+                    val size = input.readInt()
+                    val result = ArrayList<BuildDifference>(size)
+                    repeat(size) {
+                        result.add(input.readBuildDifference())
+                    }
+                    return BuildDiffsStorage(result)
+                }
+            }
+            catch (e: IOException) {
+                reportFail(e.toString())
+            }
+
+            return null
+        }
+
+        fun writeToFile(file: File, storage: BuildDiffsStorage, reporter: ICReporter?) {
+            file.parentFile.mkdirs()
+
+            try {
+                ObjectOutputStream(file.outputStream().buffered()).use { output ->
+                    output.writeInt(CURRENT_VERSION)
+
+                    val diffsToWrite = storage.buildDiffs.sortedBy { it.ts }.takeLast(MAX_DIFFS_ENTRIES)
+                    output.writeInt(diffsToWrite.size)
+                    for (diff in diffsToWrite) {
+                        output.writeBuildDifference(diff)
+                    }
+                }
+            }
+            catch (e: IOException) {
+                reporter?.report { "Could not write diff to file $file: $e" }
+            }
+        }
+
+        private fun ObjectInputStream.readBuildDifference(): BuildDifference {
+            val ts = readLong()
+            val isIncremental = readBoolean()
+            val dirtyData = readDirtyData()
+            return BuildDifference(ts, isIncremental, dirtyData)
+        }
+
+        private fun ObjectOutputStream.writeBuildDifference(diff: BuildDifference) {
+            writeLong(diff.ts)
+            writeBoolean(diff.isIncremental)
+            writeDirtyData(diff.dirtyData)
+        }
+
+        private fun ObjectInputStream.readDirtyData(): DirtyData {
+            val lookupSymbolSize = readInt()
+            val lookupSymbols = ArrayList<LookupSymbol>(lookupSymbolSize)
+            repeat(lookupSymbolSize) {
+                val name = readUTF()
+                val scope = readUTF()
+                lookupSymbols.add(LookupSymbol(name = name, scope = scope))
+            }
+
+            val dirtyClassesSize = readInt()
+            val dirtyClassesFqNames = ArrayList<FqName>(dirtyClassesSize)
+            repeat(dirtyClassesSize) {
+                val fqNameString = readUTF()
+                dirtyClassesFqNames.add(FqName(fqNameString))
+            }
+
+            return DirtyData(lookupSymbols, dirtyClassesFqNames)
+        }
+
+        private fun ObjectOutputStream.writeDirtyData(dirtyData: DirtyData) {
+            val lookupSymbols = dirtyData.dirtyLookupSymbols
+            writeInt(lookupSymbols.size)
+            for ((name, scope) in lookupSymbols) {
+                writeUTF(name)
+                writeUTF(scope)
+            }
+
+            val dirtyClassesFqNames = dirtyData.dirtyClassesFqNames
+            writeInt(dirtyClassesFqNames.size)
+            for (fqName in dirtyClassesFqNames) {
+                writeUTF(fqName.asString())
+            }
+        }
+
+        internal val MAX_DIFFS_ENTRIES: Int = 10
+
+        @set:TestOnly
+        var CURRENT_VERSION: Int = 0
+    }
+}
@@ -18,7 +18,7 @@ package org.jetbrains.kotlin.incremental

 import java.io.*

-internal data class BuildInfo(val startTS: Long) : Serializable {
+data class BuildInfo(val startTS: Long) : Serializable {
    companion object {
        fun read(file: File): BuildInfo? =
                try {
@@ -255,16 +255,9 @@ abstract class IncrementalCompilerRunner<
        if (exitCode == ExitCode.OK && compilationMode is CompilationMode.Incremental) {
            buildDirtyLookupSymbols.addAll(additionalDirtyLookupSymbols())
        }
-        if (changesRegistry != null) {
-            if (compilationMode is CompilationMode.Incremental) {
-                val dirtyData = DirtyData(buildDirtyLookupSymbols, buildDirtyFqNames)
-                changesRegistry.registerChanges(currentBuildInfo.startTS, dirtyData)
-            }
-            else {
-                assert(compilationMode is CompilationMode.Rebuild) { "Unexpected compilation mode: ${compilationMode::class.java}" }
-                changesRegistry.unknownChanges(currentBuildInfo.startTS)
-            }
-        }
+
+        val dirtyData = DirtyData(buildDirtyLookupSymbols, buildDirtyFqNames)
+        processChangesAfterBuild(compilationMode, currentBuildInfo, dirtyData)

        if (exitCode == ExitCode.OK) {
            cacheVersions.forEach { it.saveIfNeeded() }
@@ -273,6 +266,18 @@ abstract class IncrementalCompilerRunner<
        return exitCode
    }

+    protected open fun processChangesAfterBuild(compilationMode: CompilationMode, currentBuildInfo: BuildInfo, dirtyData: DirtyData) {
+        if (changesRegistry == null) return
+
+        if (compilationMode is CompilationMode.Incremental) {
+            changesRegistry.registerChanges(currentBuildInfo.startTS, dirtyData)
+        }
+        else {
+            assert(compilationMode is CompilationMode.Rebuild) { "Unexpected compilation mode: ${compilationMode::class.java}" }
+            changesRegistry.unknownChanges(currentBuildInfo.startTS)
+        }
+    }
+
    companion object {
        const val DIRTY_SOURCES_FILE_NAME = "dirty-sources.txt"
        const val LAST_BUILD_INFO_FILE_NAME = "last-build.bin"
@@ -16,10 +16,10 @@

 package org.jetbrains.kotlin.incremental

+import com.intellij.openapi.util.io.FileUtil
 import org.jetbrains.kotlin.annotation.AnnotationFileUpdater
 import org.jetbrains.kotlin.build.GeneratedFile
 import org.jetbrains.kotlin.build.GeneratedJvmClass
-import org.jetbrains.kotlin.build.isModuleMappingFile
 import org.jetbrains.kotlin.build.JvmSourceRoot
 import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
@@ -38,6 +38,7 @@ import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName
 import java.io.File
 import java.util.*
+import kotlin.collections.HashSet

 fun makeIncrementally(
        cachesDir: File,
@@ -89,7 +90,9 @@ class IncrementalJvmCompilerRunner(
        reporter: ICReporter,
        private var kaptAnnotationsFileUpdater: AnnotationFileUpdater? = null,
        artifactChangesProvider: ArtifactChangesProvider? = null,
-        changesRegistry: ChangesRegistry? = null
+        changesRegistry: ChangesRegistry? = null,
+        private val buildHistoryFile: File? = null,
+        private val friendBuildHistoryFile: File? = null
 ) : IncrementalCompilerRunner<K2JVMCompilerArguments, IncrementalJvmCachesManager>(
        workingDir,
        "caches-jvm",
@@ -110,16 +113,58 @@ class IncrementalJvmCompilerRunner(
    private var javaFilesProcessor = ChangedJavaFilesProcessor(reporter)

    override fun calculateSourcesToCompile(caches: IncrementalJvmCachesManager, changedFiles: ChangedFiles.Known, args: K2JVMCompilerArguments): CompilationMode {
-        val removedClassFiles = changedFiles.removed.filter(File::isClassFile)
-        if (removedClassFiles.any()) return CompilationMode.Rebuild { "Removed class files: ${reporter.pathsAsString(removedClassFiles)}" }
+        val dirtyFiles = getDirtyFiles(changedFiles)

-        val modifiedClassFiles = changedFiles.modified.filter(File::isClassFile)
-        if (modifiedClassFiles.any()) return CompilationMode.Rebuild { "Modified class files: ${reporter.pathsAsString(modifiedClassFiles)}" }
+        fun markDirtyBy(lookupSymbols: Collection<LookupSymbol>) {
+            if (lookupSymbols.isEmpty()) return
+
+            val dirtyFilesFromLookups = mapLookupSymbolsToFiles(caches.lookupCache, lookupSymbols, reporter)
+            dirtyFiles.addAll(dirtyFilesFromLookups)
+        }
+
+        fun markDirtyBy(dirtyClassesFqNames: Collection<FqName>) {
+            if (dirtyClassesFqNames.isEmpty()) return
+
+            val fqNamesWithSubtypes = dirtyClassesFqNames.flatMap { withSubtypes(it, listOf(caches.platformCache)) }
+            val dirtyFilesFromFqNames = mapClassesFqNamesToFiles(listOf(caches.platformCache), fqNamesWithSubtypes, reporter)
+            dirtyFiles.addAll(dirtyFilesFromFqNames)
+        }
+
+        val lastBuildInfo = BuildInfo.read(lastBuildInfoFile)
+        reporter.report { "Last Kotlin Build info -- $lastBuildInfo" }
+
+        val changesFromFriend by lazy {
+            val myLastTS = lastBuildInfo?.startTS ?: return@lazy ChangesEither.Unknown()
+            val storage = friendBuildHistoryFile?.let { BuildDiffsStorage.readFromFile(it, reporter) } ?: return@lazy ChangesEither.Unknown()
+
+            val (prevDiffs, newDiffs) = storage.buildDiffs.partition { it.ts < myLastTS }
+            if (prevDiffs.isEmpty()) return@lazy ChangesEither.Unknown()
+
+            val dirtyLookupSymbols = HashSet<LookupSymbol>()
+            val dirtyClassesFqNames = HashSet<FqName>()
+            for ((_, isIncremental, dirtyData) in newDiffs) {
+                if (!isIncremental) return@lazy ChangesEither.Unknown()
+
+                dirtyLookupSymbols.addAll(dirtyData.dirtyLookupSymbols)
+                dirtyClassesFqNames.addAll(dirtyData.dirtyClassesFqNames)
+            }
+
+            markDirtyBy(dirtyLookupSymbols)
+            markDirtyBy(dirtyClassesFqNames)
+            ChangesEither.Known(dirtyLookupSymbols, dirtyClassesFqNames)
+        }
+        val friendDirs = args.friendPaths?.map { File(it) } ?: emptyList()
+        for (file in changedFiles.removed.asSequence() + changedFiles.modified.asSequence()) {
+            if (!file.isClassFile()) continue
+
+            val isFriendClassFile = friendDirs.any { FileUtil.isAncestor(it, file, false) }
+            if (isFriendClassFile && changesFromFriend is ChangesEither.Known) continue
+
+            return CompilationMode.Rebuild { "Cannot get changes from modified or removed class file: ${reporter.pathsAsString(file)}" }
+        }

        val classpathSet = args.classpathAsList.toHashSet()
        val modifiedClasspathEntries = changedFiles.modified.filter { it in classpathSet }
-        val lastBuildInfo = BuildInfo.read(lastBuildInfoFile)
-        reporter.report { "Last Kotlin Build info -- $lastBuildInfo" }
        val classpathChanges = getClasspathChanges(modifiedClasspathEntries, lastBuildInfo)
        if (classpathChanges !is ChangesEither.Known) {
            return CompilationMode.Rebuild { "could not get changes from modified classpath entries: ${reporter.pathsAsString(modifiedClasspathEntries)}" }
@@ -131,21 +176,9 @@ class IncrementalJvmCompilerRunner(
            is ChangesEither.Unknown -> return CompilationMode.Rebuild { "Could not get changes for java files" }
        }

-        val dirtyFiles = getDirtyFiles(changedFiles)
-        val lookupSymbols = HashSet<LookupSymbol>()
-        lookupSymbols.addAll(affectedJavaSymbols)
-        lookupSymbols.addAll(classpathChanges.lookupSymbols)
-
-        if (lookupSymbols.any()) {
-            val dirtyFilesFromLookups = mapLookupSymbolsToFiles(caches.lookupCache, lookupSymbols, reporter)
-            dirtyFiles.addAll(dirtyFilesFromLookups)
-        }
-
-        val dirtyClassesFqNames = classpathChanges.fqNames.flatMap { withSubtypes(it, listOf(caches.platformCache)) }
-        if (dirtyClassesFqNames.any()) {
-            val dirtyFilesFromFqNames = mapClassesFqNamesToFiles(listOf(caches.platformCache), dirtyClassesFqNames, reporter)
-            dirtyFiles.addAll(dirtyFilesFromFqNames)
-        }
+        markDirtyBy(affectedJavaSymbols)
+        markDirtyBy(classpathChanges.lookupSymbols)
+        markDirtyBy(classpathChanges.fqNames)

        return CompilationMode.Incremental(dirtyFiles)
    }
@@ -263,6 +296,23 @@ class IncrementalJvmCompilerRunner(
    override fun additionalDirtyLookupSymbols(): Iterable<LookupSymbol> =
            javaFilesProcessor.allChangedSymbols

+    override fun processChangesAfterBuild(compilationMode: CompilationMode, currentBuildInfo: BuildInfo, dirtyData: DirtyData) {
+        super.processChangesAfterBuild(compilationMode, currentBuildInfo, dirtyData)
+
+        if (buildHistoryFile == null) return
+
+        val prevDiffs = BuildDiffsStorage.readFromFile(buildHistoryFile, reporter)?.buildDiffs ?: emptyList()
+        val newDiff = if (compilationMode is CompilationMode.Incremental) {
+            BuildDifference(currentBuildInfo.startTS, true, dirtyData)
+        }
+        else {
+            val emptyDirtyData = DirtyData()
+            BuildDifference(currentBuildInfo.startTS, false, emptyDirtyData)
+        }
+
+        BuildDiffsStorage.writeToFile(buildHistoryFile, BuildDiffsStorage(prevDiffs + newDiff), reporter)
+    }
+
    override fun makeServices(
            args: K2JVMCompilerArguments,
            lookupTracker: LookupTracker,
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import org.jetbrains.kotlin.name.FqName
+import org.junit.After
+import org.junit.Assert
+import org.junit.Before
+import org.junit.Test
+import java.io.File
+import java.io.OutputStream
+import java.util.*
+
+class BuildDiffsStorageTest {
+    lateinit var storageFile: File
+    private val random = Random(System.currentTimeMillis())
+
+    @Before
+    fun setUp() {
+        storageFile = File.createTempFile("BuildDiffsStorageTest", "storage")
+    }
+
+    @After
+    fun tearDown() {
+        storageFile.delete()
+    }
+
+    @Test
+    fun testToString() {
+        val lookupSymbols = listOf(LookupSymbol("foo", "bar"))
+        val fqNames = listOf(FqName("fizz.Buzz"))
+        val diff = BuildDifference(100, true, DirtyData(lookupSymbols, fqNames))
+        val diffs = BuildDiffsStorage(listOf(diff))
+        Assert.assertEquals("BuildDiffsStorage(buildDiffs=[BuildDifference(ts=100, isIncremental=true, dirtyData=DirtyData(dirtyLookupSymbols=[LookupSymbol(name=foo, scope=bar)], dirtyClassesFqNames=[fizz.Buzz]))])",
+                            diffs.toString())
+    }
+
+    @Test
+    fun writeReadSimple() {
+        val diffs = BuildDiffsStorage(listOf(getRandomDiff()))
+        BuildDiffsStorage.writeToFile(storageFile, diffs, reporter = null)
+
+        val diffsDeserialized = BuildDiffsStorage.readFromFile(storageFile, reporter = null)
+        Assert.assertEquals(diffs.toString(), diffsDeserialized.toString())
+    }
+
+    @Test
+    fun writeReadMany() {
+        val generated = Array(20) { getRandomDiff() }.toList()
+        val diffs = BuildDiffsStorage(generated)
+        BuildDiffsStorage.writeToFile(storageFile, diffs, reporter = null)
+
+        val diffsDeserialized = BuildDiffsStorage.readFromFile(storageFile, reporter = null)
+        val expected = generated.sortedBy { it.ts }.takeLast(BuildDiffsStorage.MAX_DIFFS_ENTRIES).toTypedArray()
+        Assert.assertArrayEquals(expected, diffsDeserialized?.buildDiffs?.toTypedArray())
+    }
+
+    @Test
+    fun readFileNotExist() {
+        storageFile.delete()
+
+        val diffsDeserialized = BuildDiffsStorage.readFromFile(storageFile, reporter = null)
+        Assert.assertEquals(null, diffsDeserialized)
+    }
+
+    @Test
+    fun versionChanged() {
+        val diffs = BuildDiffsStorage(listOf(getRandomDiff()))
+        BuildDiffsStorage.writeToFile(storageFile, diffs, reporter = null)
+
+        val versionBackup = BuildDiffsStorage.CURRENT_VERSION
+        try {
+            BuildDiffsStorage.CURRENT_VERSION++
+            val diffsDeserialized = BuildDiffsStorage.readFromFile(storageFile, reporter = null)
+            Assert.assertEquals(null, diffsDeserialized)
+        }
+        finally {
+            BuildDiffsStorage.CURRENT_VERSION = versionBackup
+        }
+    }
+
+    private fun getRandomDiff(): BuildDifference {
+        val ts = random.nextLong()
+        val lookupSymbols = listOf(LookupSymbol("foo", "bar"))
+        val fqNames = listOf(FqName("fizz.Buzz"))
+        return BuildDifference(ts, true, DirtyData(lookupSymbols, fqNames))
+    }
+}