diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedFiles.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedFiles.kt index 4cacd1984dd..29e5af2fefe 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedFiles.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedFiles.kt @@ -22,6 +22,7 @@ import java.io.Serializable sealed class ChangedFiles : Serializable { class Known(val modified: List, val removed: List) : ChangedFiles() class Unknown : ChangedFiles() + class Dependencies(val modified: List, val removed: List) : ChangedFiles() companion object { const val serialVersionUID: Long = 0 diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt index 6d5b1c0d6a2..31588994616 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt @@ -102,7 +102,18 @@ abstract class IncrementalCompilerRunner< // If compilation has crashed or we failed to close caches we have to clear them var cachesMayBeCorrupted = true return try { - val changedFiles = providedChangedFiles ?: caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) + val changedFiles = when (providedChangedFiles) { + is ChangedFiles.Dependencies -> { + val changedSources = caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) + ChangedFiles.Known( + providedChangedFiles.modified + changedSources.modified, + providedChangedFiles.removed + changedSources.removed + ) + } + null -> caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles) + else -> providedChangedFiles + } + val compilationMode = sourcesToCompile(caches, changedFiles, args, messageCollector) val exitCode = when (compilationMode) { @@ -167,6 +178,7 @@ abstract class IncrementalCompilerRunner< when (changedFiles) { is ChangedFiles.Known -> calculateSourcesToCompile(caches, changedFiles, args, messageCollector) is ChangedFiles.Unknown -> CompilationMode.Rebuild(BuildAttribute.UNKNOWN_CHANGES_IN_GRADLE_INPUTS) + is ChangedFiles.Dependencies -> error("Unexpected ChangedFiles type (ChangedFiles.Dependencies)") } private fun calculateSourcesToCompile( diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt index 963fa236ddd..4838ee93cda 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt @@ -41,23 +41,25 @@ fun makeJsIncrementally( cachesDir: File, sourceRoots: Iterable, args: K2JSCompilerArguments, + buildHistoryFile: File, messageCollector: MessageCollector = MessageCollector.NONE, reporter: ICReporter = EmptyICReporter, - scopeExpansion: CompileScopeExpansionMode = CompileScopeExpansionMode.NEVER + scopeExpansion: CompileScopeExpansionMode = CompileScopeExpansionMode.NEVER, + modulesApiHistory: ModulesApiHistory = EmptyModulesApiHistory, + providedChangedFiles: ChangedFiles? = null ) { val allKotlinFiles = sourceRoots.asSequence().flatMap { it.walk() } .filter { it.isFile && it.extension.equals("kt", ignoreCase = true) }.toList() - val buildHistoryFile = File(cachesDir, "build-history.bin") val buildReporter = BuildReporter(icReporter = reporter, buildMetricsReporter = DoNothingBuildMetricsReporter) withJsIC { val compiler = IncrementalJsCompilerRunner( cachesDir, buildReporter, buildHistoryFile = buildHistoryFile, - modulesApiHistory = EmptyModulesApiHistory, + modulesApiHistory = modulesApiHistory, scopeExpansion = scopeExpansion ) - compiler.compile(allKotlinFiles, args, messageCollector, providedChangedFiles = null) + compiler.compile(allKotlinFiles, args, messageCollector, providedChangedFiles) } } diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementaMultiModulelJsKlibCompilerRunnerTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementaMultiModulelJsKlibCompilerRunnerTest.kt new file mode 100644 index 00000000000..b4ce8cae0f6 --- /dev/null +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementaMultiModulelJsKlibCompilerRunnerTest.kt @@ -0,0 +1,230 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental + +import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments +import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryJs +import org.jetbrains.kotlin.incremental.utils.* +import org.jetbrains.kotlin.library.KLIB_FILE_EXTENSION +import org.jetbrains.kotlin.utils.DFS +import java.io.File +import java.util.regex.Pattern + +abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : AbstractIncrementalJsKlibCompilerRunnerTest() { + + private class ModuleBuildConfiguration(val srcDir: File, val dependencies: List) + + private val modulesDir: File by lazy { File(workingDir, "modules") } + private val modulesInfo: MutableMap = mutableMapOf() + private val modulesOrder: MutableList = mutableListOf() + + private val dirToModule = mutableMapOf() + private val nameToModules = mutableMapOf>() + private val jarToModule = mutableMapOf() + + private val modulesApiHistory: ModulesApiHistoryJs by lazy { + ModulesApiHistoryJs(IncrementalModuleInfo(workingDir, dirToModule, nameToModules, emptyMap(), jarToModule)) + } + + override val moduleNames: Collection? get() = modulesOrder + + override fun resetTest(testDir: File, newOutDir: File, newCacheDir: File) { + modulesDir.deleteRecursively() + modulesDir.mkdirs() + + dirToModule.clear() + nameToModules.clear() + jarToModule.clear() + + modulesOrder.forEach { setupModuleApiHistory(it, newOutDir, newCacheDir) } + } + + override fun setupTest(testDir: File, srcDir: File, cacheDir: File, outDir: File): List { + val ktFiles = srcDir.getFiles().filter { it.extension == "kt" } + + val results = mutableMapOf>>() + ktFiles.forEach { + modulePattern.matcher(it.name).let { match -> + match.find() + val moduleName = match.group(1) + val fileName = match.group(2) + val sources = results.getOrPut(moduleName) { mutableListOf() } + sources.add(it to fileName) + } + } + + val dependencyGraph = parseDependencies(testDir) + + DFS.topologicalOrder(dependencyGraph.keys) { m -> + dependencyGraph[m] ?: error("Expected dependencies for module $m") + }.reversed().mapTo(modulesOrder) { it } + + for ((moduleName, fileEntries) in results) { + val moduleDir = File(workingDir, moduleName).apply { mkdirs() } + val moduleSrcDir = File(moduleDir, "src") + + val moduleDependencies = dependencyGraph[moduleName] ?: error("Cannot find dependency for module $moduleName") + + for ((oldFile, newName) in fileEntries) { + val newFile = File(moduleSrcDir, newName) + oldFile.copyTo(newFile) + } + + modulesInfo[moduleName] = ModuleBuildConfiguration(moduleSrcDir, moduleDependencies) + + setupModuleApiHistory(moduleName, outDir, cacheDir) + } + + return listOf(srcDir) + } + + private fun setupModuleApiHistory(moduleName: String, outDir: File, cacheDir: File) { + val depKlibFile = File(modulesDir, moduleName.klib) + val moduleBuildDir = File(outDir, moduleName) + val moduleCacheDir = File(cacheDir, moduleName) + val moduleBuildHistoryFile = buildHistoryFile(moduleCacheDir) + + val moduleEntry = IncrementalModuleEntry(workingDir.absolutePath, moduleName, outDir, moduleBuildHistoryFile) + + dirToModule[moduleBuildDir] = moduleEntry + nameToModules.getOrPut(moduleName) { mutableSetOf() }.add(moduleEntry) + jarToModule[depKlibFile] = moduleEntry + } + + companion object { + + private val modulePattern = Pattern.compile("^(module\\d+)_(\\w+\\.kt)$") + + private fun File.getFiles(): List { + return if (isDirectory) listFiles()?.flatMap { it.getFiles() } ?: emptyList() + else listOf(this) + } + + private val String.klib: String get() = "$this.$KLIB_FILE_EXTENSION" + + private fun parseDependencies(testDir: File): Map> { + + val actualModulesTxtFile = File(testDir, "dependencies.txt") + + if (!actualModulesTxtFile.exists()) { + error("${actualModulesTxtFile.path} is expected") + } + + val result = mutableMapOf>() + + val lines = actualModulesTxtFile.readLines() + lines.map { it.split("->") }.map { + assert(it.size == 2) + val moduleName = it[0] + val dependencyPart = it[1] + + val idx = dependencyPart.indexOf('[') + val dependencyName = if (idx >= 0) { + // skip annotations + dependencyPart.substring(0, idx) + } else dependencyPart + + val dependencies = result.getOrPut(moduleName) { mutableListOf() } + if (dependencyName.isNotBlank()) { + dependencies.add(dependencyName) + } + } + + return result + } + } + + private fun K2JSCompilerArguments.updateCompilerArguments( + moduleDependencies: List, + initialDeps: String, + destinationFile: File + ) { + val additionalDeps = moduleDependencies.joinToString(File.pathSeparator) { + File(modulesDir, it.klib).absolutePath + } + + val sb = StringBuilder(initialDeps) + if (additionalDeps.isNotBlank()) { + sb.append(File.pathSeparator) + sb.append(additionalDeps) + } + + libraries = sb.toString() + outputFile = destinationFile.path + } + + override fun make( + cacheDir: File, + outDir: File, + sourceRoots: Iterable, + args: K2JSCompilerArguments + ): TestCompilationResult { + val reporter = TestICReporter() + val messageCollector = TestMessageCollector() + val initialDeps = args.libraries ?: "" + + args.repositries = modulesDir.path + + val modifiedLibraries = mutableListOf>() + val deletedLibraries = mutableListOf>() + + var compilationIsEnabled = true + val isInitial = modulesDir.list()?.isEmpty() ?: true + + for (module in modulesOrder) { + val moduleBuildInfo = modulesInfo[module] ?: error("Cannot find config for $module") + + val moduleDependencies = moduleBuildInfo.dependencies + + val moduleModifiedDependencies = modifiedLibraries.filter { it.first in moduleDependencies }.map { it.second } + val moduleDeletedDependencies = deletedLibraries.filter { it.first in moduleDependencies }.map { it.second } + + val changedDepsFiles = if (isInitial) null else ChangedFiles.Dependencies(moduleModifiedDependencies, moduleDeletedDependencies) + + val moduleOutDir = File(outDir, module) + val moduleCacheDir = File(cacheDir, module) + val moduleBuildHistory = buildHistoryFile(moduleCacheDir) + val sources = moduleBuildInfo.srcDir.getFiles() + + val outputKlibFile = File(moduleOutDir, module.klib) + val dependencyFile = File(modulesDir, module.klib) + + args.updateCompilerArguments(moduleDependencies, initialDeps, outputKlibFile) + + if (compilationIsEnabled) { + makeJsIncrementally( + moduleCacheDir, + sources, + args, + moduleBuildHistory, + messageCollector, + reporter, + scopeExpansionMode, + modulesApiHistory, + changedDepsFiles + ) + } + + val oldMD5 = if (dependencyFile.exists()) { + val bytes = dependencyFile.readBytes() + dependencyFile.delete() + bytes.md5() + } else 0 + + if (!messageCollector.hasErrors()) { + val newMD5 = outputKlibFile.readBytes().md5() + if (oldMD5 != newMD5) { + modifiedLibraries.add(module to dependencyFile) + } + outputKlibFile.copyTo(dependencyFile) + } else { + compilationIsEnabled = false + } + } + + return TestCompilationResult(reporter, messageCollector) + } +} \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalCompilerRunnerTestBase.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalCompilerRunnerTestBase.kt index c80bd2a3851..4a57fee88eb 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalCompilerRunnerTestBase.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalCompilerRunnerTestBase.kt @@ -29,6 +29,13 @@ import java.io.File abstract class AbstractIncrementalCompilerRunnerTestBase : TestWithWorkingDir() { protected abstract fun createCompilerArguments(destinationDir: File, testDir: File): Args + protected open val moduleNames: Collection? get() = null + + protected open fun setupTest(testDir: File, srcDir: File, cacheDir: File, outDir: File): List = + listOf(srcDir) + + protected open fun resetTest(testDir: File, newOutDir: File, newCacheDir: File) {} + private fun createCompilerArgumentsImpl(destinationDir: File, testDir: File): Args = createCompilerArguments(destinationDir, testDir).apply { parseCommandLineArguments(parseAdditionalArgs(testDir), this) } @@ -44,9 +51,9 @@ abstract class AbstractIncrementalCompilerRunnerTestBase, args: Args) = make(cacheDir, sourceRoots, args) + private fun initialMake(cacheDir: File, outDir: File, sourceRoots: List, args: Args) = make(cacheDir, outDir, sourceRoots, args) - private fun incrementalMake(cacheDir: File, sourceRoots: List, args: Args) = make(cacheDir, sourceRoots, args) + private fun incrementalMake(cacheDir: File, outDir: File, sourceRoots: List, args: Args) = make(cacheDir, outDir, sourceRoots, args) protected open fun rebuildAndCompareOutput( sourceRoots: List, @@ -116,7 +123,9 @@ abstract class AbstractIncrementalCompilerRunnerTestBase, args: Args): TestCompilationResult + protected abstract fun make(cacheDir: File, outDir: File, sourceRoots: Iterable, args: Args): TestCompilationResult private fun stepLogAsString(step: Int, ktSources: Iterable, errors: Collection, includeErrors: Boolean = true): String { val sb = StringBuilder() diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalJsCompilerRunnerTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalJsCompilerRunnerTest.kt index d198baf44d6..100b3039f15 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalJsCompilerRunnerTest.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalJsCompilerRunnerTest.kt @@ -24,10 +24,10 @@ import org.jetbrains.kotlin.incremental.utils.TestMessageCollector import java.io.File abstract class AbstractIncrementalJsCompilerRunnerTest : AbstractIncrementalCompilerRunnerTestBase() { - override fun make(cacheDir: File, sourceRoots: Iterable, args: K2JSCompilerArguments): TestCompilationResult { + override fun make(cacheDir: File, outDir: File, sourceRoots: Iterable, args: K2JSCompilerArguments): TestCompilationResult { val reporter = TestICReporter() val messageCollector = TestMessageCollector() - makeJsIncrementally(cacheDir, sourceRoots, args, messageCollector, reporter, scopeExpansionMode) + makeJsIncrementally(cacheDir, sourceRoots, args, buildHistoryFile(cacheDir), messageCollector, reporter, scopeExpansionMode) return TestCompilationResult(reporter, messageCollector) } @@ -44,5 +44,7 @@ abstract class AbstractIncrementalJsCompilerRunnerTest : AbstractIncrementalComp metaInfo = true } + protected fun buildHistoryFile(cacheDir: File): File = File(cacheDir, "build-history.bin") + protected open val scopeExpansionMode = CompileScopeExpansionMode.NEVER } \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalJvmCompilerRunnerTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalJvmCompilerRunnerTest.kt index 327f5d7342f..b93cda46c1f 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalJvmCompilerRunnerTest.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalJvmCompilerRunnerTest.kt @@ -27,7 +27,7 @@ import java.io.File import javax.tools.ToolProvider abstract class AbstractIncrementalJvmCompilerRunnerTest : AbstractIncrementalCompilerRunnerTestBase() { - override fun make(cacheDir: File, sourceRoots: Iterable, args: K2JVMCompilerArguments): TestCompilationResult { + override fun make(cacheDir: File, outDir: File, sourceRoots: Iterable, args: K2JVMCompilerArguments): TestCompilationResult { val reporter = TestICReporter() val messageCollector = TestMessageCollector() makeIncrementally(cacheDir, sourceRoots, args, reporter = reporter, messageCollector = messageCollector)