[IC TEST] Support multi module IC tests for klibs

- implement required infrastructure
 - pass changed class path dependencies

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedFiles.kt

@@ -22,6 +22,7 @@ import java.io.Serializable
 sealed class ChangedFiles : Serializable {
     class Known(val modified: List<File>, val removed: List<File>) : ChangedFiles()
     class Unknown : ChangedFiles()
+    class Dependencies(val modified: List<File>, val removed: List<File>) : ChangedFiles()
 
     companion object {
         const val serialVersionUID: Long = 0

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt

@@ -102,7 +102,18 @@ abstract class IncrementalCompilerRunner<
         // If compilation has crashed or we failed to close caches we have to clear them
         var cachesMayBeCorrupted = true
         return try {
-            val changedFiles = providedChangedFiles ?: caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
+            val changedFiles = when (providedChangedFiles) {
+                is ChangedFiles.Dependencies -> {
+                    val changedSources = caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
+                    ChangedFiles.Known(
+                        providedChangedFiles.modified + changedSources.modified,
+                        providedChangedFiles.removed + changedSources.removed
+                    )
+                }
+                null -> caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
+                else -> providedChangedFiles
+            }
+
             val compilationMode = sourcesToCompile(caches, changedFiles, args, messageCollector)
 
             val exitCode = when (compilationMode) {
@@ -167,6 +178,7 @@ abstract class IncrementalCompilerRunner<
         when (changedFiles) {
             is ChangedFiles.Known -> calculateSourcesToCompile(caches, changedFiles, args, messageCollector)
             is ChangedFiles.Unknown -> CompilationMode.Rebuild(BuildAttribute.UNKNOWN_CHANGES_IN_GRADLE_INPUTS)
+            is ChangedFiles.Dependencies -> error("Unexpected ChangedFiles type (ChangedFiles.Dependencies)")
         }
 
     private fun calculateSourcesToCompile(

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt

@@ -41,23 +41,25 @@ fun makeJsIncrementally(
     cachesDir: File,
     sourceRoots: Iterable<File>,
     args: K2JSCompilerArguments,
+    buildHistoryFile: File,
     messageCollector: MessageCollector = MessageCollector.NONE,
     reporter: ICReporter = EmptyICReporter,
-    scopeExpansion: CompileScopeExpansionMode = CompileScopeExpansionMode.NEVER
+    scopeExpansion: CompileScopeExpansionMode = CompileScopeExpansionMode.NEVER,
+    modulesApiHistory: ModulesApiHistory = EmptyModulesApiHistory,
+    providedChangedFiles: ChangedFiles? = null
 ) {
     val allKotlinFiles = sourceRoots.asSequence().flatMap { it.walk() }
         .filter { it.isFile && it.extension.equals("kt", ignoreCase = true) }.toList()
-    val buildHistoryFile = File(cachesDir, "build-history.bin")
 
     val buildReporter = BuildReporter(icReporter = reporter, buildMetricsReporter = DoNothingBuildMetricsReporter)
     withJsIC {
         val compiler = IncrementalJsCompilerRunner(
             cachesDir, buildReporter,
             buildHistoryFile = buildHistoryFile,
-            modulesApiHistory = EmptyModulesApiHistory,
+            modulesApiHistory = modulesApiHistory,
             scopeExpansion = scopeExpansion
         )
-        compiler.compile(allKotlinFiles, args, messageCollector, providedChangedFiles = null)
+        compiler.compile(allKotlinFiles, args, messageCollector, providedChangedFiles)
     }
 }
 

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementaMultiModulelJsKlibCompilerRunnerTest.kt

@@ -0,0 +1,230 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
+import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryJs
+import org.jetbrains.kotlin.incremental.utils.*
+import org.jetbrains.kotlin.library.KLIB_FILE_EXTENSION
+import org.jetbrains.kotlin.utils.DFS
+import java.io.File
+import java.util.regex.Pattern
+
+abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : AbstractIncrementalJsKlibCompilerRunnerTest() {
+
+    private class ModuleBuildConfiguration(val srcDir: File, val dependencies: List<String>)
+
+    private val modulesDir: File by lazy { File(workingDir, "modules") }
+    private val modulesInfo: MutableMap<String, ModuleBuildConfiguration> = mutableMapOf()
+    private val modulesOrder: MutableList<String> = mutableListOf()
+
+    private val dirToModule = mutableMapOf<File, IncrementalModuleEntry>()
+    private val nameToModules = mutableMapOf<String, MutableSet<IncrementalModuleEntry>>()
+    private val jarToModule = mutableMapOf<File, IncrementalModuleEntry>()
+
+    private val modulesApiHistory: ModulesApiHistoryJs by lazy {
+        ModulesApiHistoryJs(IncrementalModuleInfo(workingDir, dirToModule, nameToModules, emptyMap(), jarToModule))
+    }
+
+    override val moduleNames: Collection<String>? get() = modulesOrder
+
+    override fun resetTest(testDir: File, newOutDir: File, newCacheDir: File) {
+        modulesDir.deleteRecursively()
+        modulesDir.mkdirs()
+
+        dirToModule.clear()
+        nameToModules.clear()
+        jarToModule.clear()
+
+        modulesOrder.forEach { setupModuleApiHistory(it, newOutDir, newCacheDir) }
+    }
+
+    override fun setupTest(testDir: File, srcDir: File, cacheDir: File, outDir: File): List<File> {
+        val ktFiles = srcDir.getFiles().filter { it.extension == "kt" }
+
+        val results = mutableMapOf<String, MutableList<Pair<File, String>>>()
+        ktFiles.forEach {
+            modulePattern.matcher(it.name).let { match ->
+                match.find()
+                val moduleName = match.group(1)
+                val fileName = match.group(2)
+                val sources = results.getOrPut(moduleName) { mutableListOf() }
+                sources.add(it to fileName)
+            }
+        }
+
+        val dependencyGraph = parseDependencies(testDir)
+
+        DFS.topologicalOrder(dependencyGraph.keys) { m ->
+            dependencyGraph[m] ?: error("Expected dependencies for module $m")
+        }.reversed().mapTo(modulesOrder) { it }
+
+        for ((moduleName, fileEntries) in results) {
+            val moduleDir = File(workingDir, moduleName).apply { mkdirs() }
+            val moduleSrcDir = File(moduleDir, "src")
+
+            val moduleDependencies = dependencyGraph[moduleName] ?: error("Cannot find dependency for module $moduleName")
+
+            for ((oldFile, newName) in fileEntries) {
+                val newFile = File(moduleSrcDir, newName)
+                oldFile.copyTo(newFile)
+            }
+
+            modulesInfo[moduleName] = ModuleBuildConfiguration(moduleSrcDir, moduleDependencies)
+
+            setupModuleApiHistory(moduleName, outDir, cacheDir)
+        }
+
+        return listOf(srcDir)
+    }
+
+    private fun setupModuleApiHistory(moduleName: String, outDir: File, cacheDir: File) {
+        val depKlibFile = File(modulesDir, moduleName.klib)
+        val moduleBuildDir = File(outDir, moduleName)
+        val moduleCacheDir = File(cacheDir, moduleName)
+        val moduleBuildHistoryFile = buildHistoryFile(moduleCacheDir)
+
+        val moduleEntry = IncrementalModuleEntry(workingDir.absolutePath, moduleName, outDir, moduleBuildHistoryFile)
+
+        dirToModule[moduleBuildDir] = moduleEntry
+        nameToModules.getOrPut(moduleName) { mutableSetOf() }.add(moduleEntry)
+        jarToModule[depKlibFile] = moduleEntry
+    }
+
+    companion object {
+
+        private val modulePattern = Pattern.compile("^(module\\d+)_(\\w+\\.kt)$")
+
+        private fun File.getFiles(): List<File> {
+            return if (isDirectory) listFiles()?.flatMap { it.getFiles() } ?: emptyList()
+            else listOf(this)
+        }
+
+        private val String.klib: String get() = "$this.$KLIB_FILE_EXTENSION"
+
+        private fun parseDependencies(testDir: File): Map<String, List<String>> {
+
+            val actualModulesTxtFile = File(testDir, "dependencies.txt")
+
+            if (!actualModulesTxtFile.exists()) {
+                error("${actualModulesTxtFile.path} is expected")
+            }
+
+            val result = mutableMapOf<String, MutableList<String>>()
+
+            val lines = actualModulesTxtFile.readLines()
+            lines.map { it.split("->") }.map {
+                assert(it.size == 2)
+                val moduleName = it[0]
+                val dependencyPart = it[1]
+
+                val idx = dependencyPart.indexOf('[')
+                val dependencyName = if (idx >= 0) {
+                    // skip annotations
+                    dependencyPart.substring(0, idx)
+                } else dependencyPart
+
+                val dependencies = result.getOrPut(moduleName) { mutableListOf() }
+                if (dependencyName.isNotBlank()) {
+                    dependencies.add(dependencyName)
+                }
+            }
+
+            return result
+        }
+    }
+
+    private fun K2JSCompilerArguments.updateCompilerArguments(
+        moduleDependencies: List<String>,
+        initialDeps: String,
+        destinationFile: File
+    ) {
+        val additionalDeps = moduleDependencies.joinToString(File.pathSeparator) {
+            File(modulesDir, it.klib).absolutePath
+        }
+
+        val sb = StringBuilder(initialDeps)
+        if (additionalDeps.isNotBlank()) {
+            sb.append(File.pathSeparator)
+            sb.append(additionalDeps)
+        }
+
+        libraries = sb.toString()
+        outputFile = destinationFile.path
+    }
+
+    override fun make(
+        cacheDir: File,
+        outDir: File,
+        sourceRoots: Iterable<File>,
+        args: K2JSCompilerArguments
+    ): TestCompilationResult {
+        val reporter = TestICReporter()
+        val messageCollector = TestMessageCollector()
+        val initialDeps = args.libraries ?: ""
+
+        args.repositries = modulesDir.path
+
+        val modifiedLibraries = mutableListOf<Pair<String, File>>()
+        val deletedLibraries = mutableListOf<Pair<String, File>>()
+
+        var compilationIsEnabled = true
+        val isInitial = modulesDir.list()?.isEmpty() ?: true
+
+        for (module in modulesOrder) {
+            val moduleBuildInfo = modulesInfo[module] ?: error("Cannot find config for $module")
+
+            val moduleDependencies = moduleBuildInfo.dependencies
+
+            val moduleModifiedDependencies = modifiedLibraries.filter { it.first in moduleDependencies }.map { it.second }
+            val moduleDeletedDependencies = deletedLibraries.filter { it.first in moduleDependencies }.map { it.second }
+
+            val changedDepsFiles = if (isInitial) null else ChangedFiles.Dependencies(moduleModifiedDependencies, moduleDeletedDependencies)
+
+            val moduleOutDir = File(outDir, module)
+            val moduleCacheDir = File(cacheDir, module)
+            val moduleBuildHistory = buildHistoryFile(moduleCacheDir)
+            val sources = moduleBuildInfo.srcDir.getFiles()
+
+            val outputKlibFile = File(moduleOutDir, module.klib)
+            val dependencyFile = File(modulesDir, module.klib)
+
+            args.updateCompilerArguments(moduleDependencies, initialDeps, outputKlibFile)
+
+            if (compilationIsEnabled) {
+                makeJsIncrementally(
+                    moduleCacheDir,
+                    sources,
+                    args,
+                    moduleBuildHistory,
+                    messageCollector,
+                    reporter,
+                    scopeExpansionMode,
+                    modulesApiHistory,
+                    changedDepsFiles
+                )
+            }
+
+            val oldMD5 = if (dependencyFile.exists()) {
+                val bytes = dependencyFile.readBytes()
+                dependencyFile.delete()
+                bytes.md5()
+            } else 0
+
+            if (!messageCollector.hasErrors()) {
+                val newMD5 = outputKlibFile.readBytes().md5()
+                if (oldMD5 != newMD5) {
+                    modifiedLibraries.add(module to dependencyFile)
+                }
+                outputKlibFile.copyTo(dependencyFile)
+            } else {
+                compilationIsEnabled = false
+            }
+        }
+
+        return TestCompilationResult(reporter, messageCollector)
+    }
+}

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalCompilerRunnerTestBase.kt

@@ -29,6 +29,13 @@ import java.io.File
 abstract class AbstractIncrementalCompilerRunnerTestBase<Args : CommonCompilerArguments> : TestWithWorkingDir() {
     protected abstract fun createCompilerArguments(destinationDir: File, testDir: File): Args
 
+    protected open val moduleNames: Collection<String>? get() = null
+
+    protected open fun setupTest(testDir: File, srcDir: File, cacheDir: File, outDir: File): List<File> =
+        listOf(srcDir)
+
+    protected open fun resetTest(testDir: File, newOutDir: File, newCacheDir: File) {}
+
     private fun createCompilerArgumentsImpl(destinationDir: File, testDir: File): Args = createCompilerArguments(destinationDir, testDir).apply {
         parseCommandLineArguments(parseAdditionalArgs(testDir), this)
     }
@@ -44,9 +51,9 @@ abstract class AbstractIncrementalCompilerRunnerTestBase<Args : CommonCompilerAr
         val outDir = File(workingDir, "out").apply { mkdirs() }
 
         val mapWorkingToOriginalFile = HashMap(copyTestSources(testDir, srcDir, filePrefix = ""))
-        val sourceRoots = listOf(srcDir)
+        val sourceRoots = setupTest(testDir, srcDir, cacheDir, outDir)
         val args = createCompilerArgumentsImpl(outDir, testDir)
-        val (_, _, errors) = initialMake(cacheDir, sourceRoots, args)
+        val (_, _, errors) = initialMake(cacheDir, outDir, sourceRoots, args)
         check(errors.isEmpty()) { "Initial build failed: \n${errors.joinToString("\n")}" }
 
         // modifications
@@ -54,7 +61,7 @@ abstract class AbstractIncrementalCompilerRunnerTestBase<Args : CommonCompilerAr
         val buildLogSteps = parseTestBuildLog(buildLogFile)
         val modifications = getModificationsToPerform(
             testDir,
-            moduleNames = null,
+            moduleNames = moduleNames,
             allowNoFilesWithSuffixInTestData = false,
             touchPolicy = TouchPolicy.CHECKSUM
         )
@@ -74,7 +81,7 @@ abstract class AbstractIncrementalCompilerRunnerTestBase<Args : CommonCompilerAr
         var step = 1
         for ((modificationStep, buildLogStep) in modifications.zip(buildLogSteps)) {
             modificationStep.forEach { it.perform(workingDir, mapWorkingToOriginalFile) }
-            val (_, compiledSources, compileErrors) = incrementalMake(cacheDir, sourceRoots, args)
+            val (_, compiledSources, compileErrors) = incrementalMake(cacheDir, outDir, sourceRoots, createCompilerArguments(outDir, testDir))
 
             expectedSB.appendLine(stepLogAsString(step, buildLogStep.compiledKotlinFiles, buildLogStep.compileErrors))
             expectedSBWithoutErrors.appendLine(
@@ -103,9 +110,9 @@ abstract class AbstractIncrementalCompilerRunnerTestBase<Args : CommonCompilerAr
     }
 
     // these functions are needed only to simplify debugging of IC tests
-    private fun initialMake(cacheDir: File, sourceRoots: List<File>, args: Args) = make(cacheDir, sourceRoots, args)
+    private fun initialMake(cacheDir: File, outDir: File, sourceRoots: List<File>, args: Args) = make(cacheDir, outDir, sourceRoots, args)
 
-    private fun incrementalMake(cacheDir: File, sourceRoots: List<File>, args: Args) = make(cacheDir, sourceRoots, args)
+    private fun incrementalMake(cacheDir: File, outDir: File, sourceRoots: List<File>, args: Args) = make(cacheDir, outDir, sourceRoots, args)
 
     protected open fun rebuildAndCompareOutput(
         sourceRoots: List<File>,
@@ -116,7 +123,9 @@ abstract class AbstractIncrementalCompilerRunnerTestBase<Args : CommonCompilerAr
         // todo: also compare caches
         val rebuildOutDir = File(workingDir, "rebuild-out").apply { mkdirs() }
         val rebuildCacheDir = File(workingDir, "rebuild-cache").apply { mkdirs() }
-        val rebuildResult = make(rebuildCacheDir, sourceRoots, createCompilerArgumentsImpl(rebuildOutDir, testDir))
+        resetTest(testDir, rebuildOutDir, rebuildCacheDir)
+
+        val rebuildResult = make(rebuildCacheDir, rebuildOutDir, sourceRoots, createCompilerArgumentsImpl(rebuildOutDir, testDir))
 
         val rebuildExpectedToSucceed = buildLogSteps.last().compileSucceeded
         val rebuildSucceeded = rebuildResult.exitCode == ExitCode.OK
@@ -130,7 +139,7 @@ abstract class AbstractIncrementalCompilerRunnerTestBase<Args : CommonCompilerAr
     protected open val buildLogFinder: BuildLogFinder
         get() = BuildLogFinder(isGradleEnabled = true)
 
-    protected abstract fun make(cacheDir: File, sourceRoots: Iterable<File>, args: Args): TestCompilationResult
+    protected abstract fun make(cacheDir: File, outDir: File, sourceRoots: Iterable<File>, args: Args): TestCompilationResult
 
     private fun stepLogAsString(step: Int, ktSources: Iterable<String>, errors: Collection<String>, includeErrors: Boolean = true): String {
         val sb = StringBuilder()

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalJsCompilerRunnerTest.kt

@@ -24,10 +24,10 @@ import org.jetbrains.kotlin.incremental.utils.TestMessageCollector
 import java.io.File
 
 abstract class AbstractIncrementalJsCompilerRunnerTest : AbstractIncrementalCompilerRunnerTestBase<K2JSCompilerArguments>() {
-    override fun make(cacheDir: File, sourceRoots: Iterable<File>, args: K2JSCompilerArguments): TestCompilationResult {
+    override fun make(cacheDir: File, outDir: File, sourceRoots: Iterable<File>, args: K2JSCompilerArguments): TestCompilationResult {
         val reporter = TestICReporter()
         val messageCollector = TestMessageCollector()
-        makeJsIncrementally(cacheDir, sourceRoots, args, messageCollector, reporter, scopeExpansionMode)
+        makeJsIncrementally(cacheDir, sourceRoots, args, buildHistoryFile(cacheDir), messageCollector, reporter, scopeExpansionMode)
         return TestCompilationResult(reporter, messageCollector)
     }
 
@@ -44,5 +44,7 @@ abstract class AbstractIncrementalJsCompilerRunnerTest : AbstractIncrementalComp
             metaInfo = true
         }
 
+    protected fun buildHistoryFile(cacheDir: File): File = File(cacheDir, "build-history.bin")
+
     protected open val scopeExpansionMode = CompileScopeExpansionMode.NEVER
 }

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalJvmCompilerRunnerTest.kt

@@ -27,7 +27,7 @@ import java.io.File
 import javax.tools.ToolProvider
 
 abstract class AbstractIncrementalJvmCompilerRunnerTest : AbstractIncrementalCompilerRunnerTestBase<K2JVMCompilerArguments>() {
-    override fun make(cacheDir: File, sourceRoots: Iterable<File>, args: K2JVMCompilerArguments): TestCompilationResult {
+    override fun make(cacheDir: File, outDir: File, sourceRoots: Iterable<File>, args: K2JVMCompilerArguments): TestCompilationResult {
         val reporter = TestICReporter()
         val messageCollector = TestMessageCollector()
         makeIncrementally(cacheDir, sourceRoots, args, reporter = reporter, messageCollector = messageCollector)