Implement inter-project JS IC with Gradle

#KT-25025 fixed
2018-07-10 15:18:32 +03:00
parent 02af631e14
commit 30f3622b27
30 changed files with 408 additions and 271 deletions
@@ -41,6 +41,7 @@ abstract class IncrementalCompilerRunner<
        cacheDirName: String,
        protected val cacheVersions: List<CacheVersion>,
        protected val reporter: ICReporter,
+        private val buildHistoryFile: File,
        private val localStateDirs: Collection<File> = emptyList()
 ) {

@@ -299,11 +300,20 @@ abstract class IncrementalCompilerRunner<

    open fun runWithNoDirtyKotlinSources(caches: CacheManager): Boolean = false

-    protected open fun processChangesAfterBuild(
+    private fun processChangesAfterBuild(
        compilationMode: CompilationMode,
        currentBuildInfo: BuildInfo,
        dirtyData: DirtyData
    ) {
+        val prevDiffs = BuildDiffsStorage.readFromFile(buildHistoryFile, reporter)?.buildDiffs ?: emptyList()
+        val newDiff = if (compilationMode is CompilationMode.Incremental) {
+            BuildDifference(currentBuildInfo.startTS, true, dirtyData)
+        } else {
+            val emptyDirtyData = DirtyData()
+            BuildDifference(currentBuildInfo.startTS, false, emptyDirtyData)
+        }
+
+        BuildDiffsStorage.writeToFile(buildHistoryFile, BuildDiffsStorage(prevDiffs + newDiff), reporter)
    }

    companion object {
@@ -16,7 +16,6 @@

 package org.jetbrains.kotlin.incremental

-import com.intellij.openapi.util.io.FileUtil
 import org.jetbrains.kotlin.build.GeneratedFile
 import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
@@ -27,6 +26,8 @@ import org.jetbrains.kotlin.config.Services
 import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.js.*
+import org.jetbrains.kotlin.incremental.multiproject.EmptyModulesApiHistory
+import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory
 import java.io.File

 fun makeJsIncrementally(
@@ -40,9 +41,13 @@ fun makeJsIncrementally(
    val versions = commonCacheVersions(cachesDir, isIncremental) + standaloneCacheVersion(cachesDir, isIncremental)
    val allKotlinFiles = sourceRoots.asSequence().flatMap { it.walk() }
            .filter { it.isFile && it.extension.equals("kt", ignoreCase = true) }.toList()
+    val buildHistoryFile = File(cachesDir, "build-history.bin")

    withJsIC {
-        val compiler = IncrementalJsCompilerRunner(cachesDir, versions, reporter)
+        val compiler = IncrementalJsCompilerRunner(
+            cachesDir, versions, reporter,
+            buildHistoryFile = buildHistoryFile,
+            modulesApiHistory = EmptyModulesApiHistory)
        compiler.compile(allKotlinFiles, args, messageCollector, providedChangedFiles = null)
    }
 }
@@ -61,12 +66,15 @@ inline fun <R> withJsIC(fn: () -> R): R {
 class IncrementalJsCompilerRunner(
        workingDir: File,
        cacheVersions: List<CacheVersion>,
-        reporter: ICReporter
+        reporter: ICReporter,
+        buildHistoryFile: File,
+        private val modulesApiHistory: ModulesApiHistory
 ) : IncrementalCompilerRunner<K2JSCompilerArguments, IncrementalJsCachesManager>(
        workingDir,
        "caches-js",
        cacheVersions,
-        reporter
+        reporter,
+        buildHistoryFile = buildHistoryFile
 ) {
    override fun isICEnabled(): Boolean =
        IncrementalCompilation.isEnabledForJs()
@@ -78,21 +86,28 @@ class IncrementalJsCompilerRunner(
        File(args.outputFile).parentFile

    override fun calculateSourcesToCompile(caches: IncrementalJsCachesManager, changedFiles: ChangedFiles.Known, args: K2JSCompilerArguments): CompilationMode {
-        if (BuildInfo.read(lastBuildInfoFile) == null) return CompilationMode.Rebuild { "No information on previous build" }
-
-        val libs = (args.libraries ?: "").split(File.pathSeparator).mapTo(HashSet()) { File(it) }
-        val libsDirs = libs.filter { it.isDirectory }
-
-        val changedLib = changedFiles.allAsSequence.find { it in libs }
-                         ?: changedFiles.allAsSequence.find { changedFile ->
-                                libsDirs.any { libDir -> FileUtil.isAncestor(libDir, changedFile, true) }
-                            }
-
-        if (changedLib != null) return CompilationMode.Rebuild { "Library has been changed: $changedLib" }
+        val lastBuildInfo = BuildInfo.read(lastBuildInfoFile)
+            ?: return CompilationMode.Rebuild { "No information on previous build" }

        val dirtyFiles = DirtyFilesContainer(caches, reporter)
        initDirtyFiles(dirtyFiles, changedFiles)

+        val libs = (args.libraries ?: "").split(File.pathSeparator).map { File(it) }
+        val classpathChanges = getClasspathChanges(libs, changedFiles, lastBuildInfo, modulesApiHistory, reporter)
+
+        @Suppress("UNUSED_VARIABLE") // for sealed when
+        val unused = when (classpathChanges) {
+            is ChangesEither.Unknown -> return CompilationMode.Rebuild {
+                // todo: we can recompile all files incrementally (not cleaning caches), so rebuild won't propagate
+                "Could not get classpath's changes${classpathChanges.reason?.let { ": $it" }}"
+            }
+            is ChangesEither.Known -> {
+                dirtyFiles.addByDirtySymbols(classpathChanges.lookupSymbols)
+                dirtyFiles.addByDirtyClasses(classpathChanges.fqNames)
+            }
+        }
+
+
        val removedClassesChanges = getRemovedClassesChanges(caches, changedFiles)
        dirtyFiles.addByDirtySymbols(removedClassesChanges.dirtyLookupSymbols)
        dirtyFiles.addByDirtyClasses(removedClassesChanges.dirtyClassesFqNames)
@@ -104,7 +104,7 @@ class IncrementalJvmCompilerRunner(
        cacheVersions: List<CacheVersion>,
        reporter: ICReporter,
        private val usePreciseJavaTracking: Boolean,
-        private val buildHistoryFile: File,
+        buildHistoryFile: File,
        localStateDirs: Collection<File>,
        private val modulesApiHistory: ModulesApiHistory
 ) : IncrementalCompilerRunner<K2JVMCompilerArguments, IncrementalJvmCachesManager>(
@@ -112,7 +112,8 @@ class IncrementalJvmCompilerRunner(
        "caches-jvm",
        cacheVersions,
        reporter,
-        localStateDirs = localStateDirs
+        localStateDirs = localStateDirs,
+        buildHistoryFile = buildHistoryFile
 ) {
    override fun isICEnabled(): Boolean =
            IncrementalCompilation.isEnabledForJvm()
@@ -310,22 +311,6 @@ class IncrementalJvmCompilerRunner(
    override fun additionalDirtyLookupSymbols(): Iterable<LookupSymbol> =
            javaFilesProcessor?.allChangedSymbols ?: emptyList()

-    override fun processChangesAfterBuild(
-        compilationMode: CompilationMode,
-        currentBuildInfo: BuildInfo,
-        dirtyData: DirtyData
-    ) {
-        val prevDiffs = BuildDiffsStorage.readFromFile(buildHistoryFile, reporter)?.buildDiffs ?: emptyList()
-        val newDiff = if (compilationMode is CompilationMode.Incremental) {
-            BuildDifference(currentBuildInfo.startTS, true, dirtyData)
-        } else {
-            val emptyDirtyData = DirtyData()
-            BuildDifference(currentBuildInfo.startTS, false, emptyDirtyData)
-        }
-
-        BuildDiffsStorage.writeToFile(buildHistoryFile, BuildDiffsStorage(prevDiffs + newDiff), reporter)
-    }
-
    override fun makeServices(
            args: K2JVMCompilerArguments,
            lookupTracker: LookupTracker,
@@ -387,57 +372,3 @@ var K2JVMCompilerArguments.destinationAsFile: File
 var K2JVMCompilerArguments.classpathAsList: List<File>
    get() = classpath.orEmpty().split(File.pathSeparator).map(::File)
    set(value) { classpath = value.joinToString(separator = File.pathSeparator, transform = { it.path }) }
-
-private fun getClasspathChanges(
-    classpath: List<File>,
-    changedFiles: ChangedFiles.Known,
-    lastBuildInfo: BuildInfo,
-    modulesApiHistory: ModulesApiHistory,
-    reporter: ICReporter?
-): ChangesEither {
-    val classpathSet = HashSet<File>()
-    for (file in classpath) {
-        when {
-            file.isFile -> classpathSet.add(file)
-            file.isDirectory -> file.walk().filterTo(classpathSet) { it.isFile }
-        }
-    }
-
-    val modifiedClasspath = changedFiles.modified.filterTo(HashSet()) { it in classpathSet }
-    val removedClasspath = changedFiles.removed.filterTo(HashSet()) { it in classpathSet }
-
-    // todo: removed classes could be processed normally
-    if (removedClasspath.isNotEmpty()) return ChangesEither.Unknown("Some files are removed from classpath $removedClasspath")
-
-    if (modifiedClasspath.isEmpty()) return ChangesEither.Known()
-
-    val lastBuildTS = lastBuildInfo.startTS
-
-    val symbols = HashSet<LookupSymbol>()
-    val fqNames = HashSet<FqName>()
-
-    val historyFilesEither = modulesApiHistory.historyFilesForChangedFiles(modifiedClasspath)
-    val historyFiles = when (historyFilesEither) {
-        is Either.Success<Set<File>> -> historyFilesEither.value
-        is Either.Error -> return ChangesEither.Unknown(historyFilesEither.reason)
-    }
-
-    for (historyFile in historyFiles) {
-        val allBuilds = BuildDiffsStorage.readDiffsFromFile(historyFile, reporter = reporter)
-            ?: return ChangesEither.Unknown("Could not read diffs from $historyFile")
-        val (knownBuilds, newBuilds) = allBuilds.partition { it.ts <= lastBuildTS }
-        if (knownBuilds.isEmpty()) {
-            return ChangesEither.Unknown("No previously known builds for $historyFile")
-        }
-
-        for (buildDiff in newBuilds) {
-            if (!buildDiff.isIncremental) return ChangesEither.Unknown("Non-incremental build from dependency $historyFile")
-
-            val dirtyData = buildDiff.dirtyData
-            symbols.addAll(dirtyData.dirtyLookupSymbols)
-            fqNames.addAll(dirtyData.dirtyClassesFqNames)
-        }
-    }
-
-    return ChangesEither.Known(symbols, fqNames)
-}
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory
+import org.jetbrains.kotlin.incremental.util.Either
+import org.jetbrains.kotlin.name.FqName
+import java.io.File
+
+internal fun getClasspathChanges(
+    classpath: List<File>,
+    changedFiles: ChangedFiles.Known,
+    lastBuildInfo: BuildInfo,
+    modulesApiHistory: ModulesApiHistory,
+    reporter: ICReporter?
+): ChangesEither {
+    val classpathSet = HashSet<File>()
+    for (file in classpath) {
+        when {
+            file.isFile -> classpathSet.add(file)
+            file.isDirectory -> file.walk().filterTo(classpathSet) { it.isFile }
+        }
+    }
+
+    val modifiedClasspath = changedFiles.modified.filterTo(HashSet()) { it in classpathSet }
+    val removedClasspath = changedFiles.removed.filterTo(HashSet()) { it in classpathSet }
+
+    // todo: removed classes could be processed normally
+    if (removedClasspath.isNotEmpty()) return ChangesEither.Unknown("Some files are removed from classpath $removedClasspath")
+
+    if (modifiedClasspath.isEmpty()) return ChangesEither.Known()
+
+    val lastBuildTS = lastBuildInfo.startTS
+
+    val symbols = HashSet<LookupSymbol>()
+    val fqNames = HashSet<FqName>()
+
+    val historyFilesEither = modulesApiHistory.historyFilesForChangedFiles(modifiedClasspath)
+    val historyFiles = when (historyFilesEither) {
+        is Either.Success<Set<File>> -> historyFilesEither.value
+        is Either.Error -> return ChangesEither.Unknown(historyFilesEither.reason)
+    }
+
+    for (historyFile in historyFiles) {
+        val allBuilds = BuildDiffsStorage.readDiffsFromFile(historyFile, reporter = reporter)
+            ?: return ChangesEither.Unknown("Could not read diffs from $historyFile")
+        val (knownBuilds, newBuilds) = allBuilds.partition { it.ts <= lastBuildTS }
+        if (knownBuilds.isEmpty()) {
+            return ChangesEither.Unknown("No previously known builds for $historyFile")
+        }
+
+        for (buildDiff in newBuilds) {
+            if (!buildDiff.isIncremental) return ChangesEither.Unknown("Non-incremental build from dependency $historyFile")
+
+            val dirtyData = buildDiff.dirtyData
+            symbols.addAll(dirtyData.dirtyLookupSymbols)
+            fqNames.addAll(dirtyData.dirtyClassesFqNames)
+        }
+    }
+
+    return ChangesEither.Known(symbols, fqNames)
+}
@@ -22,7 +22,7 @@ object EmptyModulesApiHistory : ModulesApiHistory {
        Either.Error("Multi-module IC is not configured")
 }

-open class ModulesApiHistoryJvm(protected val modulesInfo: IncrementalModuleInfo) : ModulesApiHistory {
+abstract class ModulesApiHistoryBase(protected val modulesInfo: IncrementalModuleInfo) : ModulesApiHistory {
    protected val projectRootPath: Path = Paths.get(modulesInfo.projectRoot.absolutePath)
    private val dirToHistoryFileCache = HashMap<File, Set<File>>()

@@ -86,7 +86,11 @@ open class ModulesApiHistoryJvm(protected val modulesInfo: IncrementalModuleInfo
        return Either.Success(history)
    }

-    protected open fun getBuildHistoryFilesForJar(jar: File): Either<Set<File>> {
+    protected abstract fun getBuildHistoryFilesForJar(jar: File): Either<Set<File>>
+}
+
+class ModulesApiHistoryJvm(modulesInfo: IncrementalModuleInfo) : ModulesApiHistoryBase(modulesInfo) {
+    override fun getBuildHistoryFilesForJar(jar: File): Either<Set<File>> {
        val classListFile = modulesInfo.jarToClassListFile[jar] ?: return Either.Error("Unknown jar: $jar")
        if (!classListFile.isFile) return Either.Error("Class list file does not exist $classListFile")

@@ -109,7 +113,18 @@ open class ModulesApiHistoryJvm(protected val modulesInfo: IncrementalModuleInfo
    }
 }

-class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiHistoryJvm(modulesInfo) {
+class ModulesApiHistoryJs(modulesInfo: IncrementalModuleInfo) : ModulesApiHistoryBase(modulesInfo) {
+    override fun getBuildHistoryFilesForJar(jar: File): Either<Set<File>> {
+        val moduleEntry = modulesInfo.jarToModule[jar]
+
+        return when {
+            moduleEntry != null -> Either.Success(setOf(moduleEntry.buildHistoryFile))
+            else -> Either.Error("No module is found for jar $jar")
+        }
+    }
+}
+
+class ModulesApiHistoryAndroid(modulesInfo: IncrementalModuleInfo) : ModulesApiHistoryBase(modulesInfo) {
    private val delegate = ModulesApiHistoryJvm(modulesInfo)

    override fun historyFilesForChangedFiles(changedFiles: Set<File>): Either<Set<File>> {
@@ -58,7 +58,8 @@ class ModulesApiHistoryAndroidTest {
            projectRoot = projectRoot,
            dirToModule = mapOf(appKotlinDestination to appEntry, libKotlinDestination to libEntry),
            nameToModules = mapOf("app" to setOf(appEntry), "lib" to setOf(libEntry)),
-            jarToClassListFile = mapOf()
+            jarToClassListFile = mapOf(),
+            jarToModule = mapOf()
        )

        androidHistory = ModulesApiHistoryAndroid(info)