[IC TEST] Abstract multi module test runner from klib

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementaMultiModulelCompilerRunnerTest.kt

@@ -5,35 +5,45 @@
 
 package org.jetbrains.kotlin.incremental
 
-import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
+import org.jetbrains.kotlin.build.report.ICReporter
-import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryJs
+import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
-import org.jetbrains.kotlin.incremental.utils.*
+import org.jetbrains.kotlin.cli.common.messages.MessageCollector
-import org.jetbrains.kotlin.library.KLIB_FILE_EXTENSION
+import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory
+import org.jetbrains.kotlin.incremental.utils.TestCompilationResult
+import org.jetbrains.kotlin.incremental.utils.TestICReporter
+import org.jetbrains.kotlin.incremental.utils.TestMessageCollector
 import org.jetbrains.kotlin.utils.DFS
 import java.io.File
 import java.util.regex.Pattern
 
-abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : AbstractIncrementalJsKlibCompilerRunnerTest() {
+abstract class AbstractIncrementalMultiModuleCompilerRunnerTest<Args : CommonCompilerArguments, ApiHistory : ModulesApiHistory> :
+    AbstractIncrementalCompilerRunnerTestBase<Args>() {
 
-    private class ModuleBuildConfiguration(val srcDir: File, val dependencies: List<String>)
+    private class ModuleDependency(val moduleName: String, val flags: Set<String>)
+    private class ModuleBuildConfiguration(val srcDir: File, val dependencies: List<ModuleDependency>)
 
-    private val modulesDir: File by lazy { File(workingDir, "modules") }
+    protected val repository: File by lazy { File(workingDir, "repository") }
     private val modulesInfo: MutableMap<String, ModuleBuildConfiguration> = mutableMapOf()
     private val modulesOrder: MutableList<String> = mutableListOf()
 
     private val dirToModule = mutableMapOf<File, IncrementalModuleEntry>()
     private val nameToModules = mutableMapOf<String, MutableSet<IncrementalModuleEntry>>()
+    private val jarToClassListFile = mutableMapOf<File, File>()
     private val jarToModule = mutableMapOf<File, IncrementalModuleEntry>()
 
-    private val modulesApiHistory: ModulesApiHistoryJs by lazy {
+    protected val incrementalModuleInfo: IncrementalModuleInfo by lazy {
-        ModulesApiHistoryJs(IncrementalModuleInfo(workingDir, dirToModule, nameToModules, emptyMap(), jarToModule))
+        IncrementalModuleInfo(workingDir, dirToModule, nameToModules, jarToClassListFile, jarToModule)
     }
 
+    protected abstract val modulesApiHistory: ApiHistory
+
     override val moduleNames: Collection<String>? get() = modulesOrder
 
+    protected abstract val scopeExpansionMode: CompileScopeExpansionMode
+
     override fun resetTest(testDir: File, newOutDir: File, newCacheDir: File) {
-        modulesDir.deleteRecursively()
+        repository.deleteRecursively()
-        modulesDir.mkdirs()
+        repository.mkdirs()
 
         dirToModule.clear()
         nameToModules.clear()
@@ -43,6 +53,7 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
     }
 
     override fun setupTest(testDir: File, srcDir: File, cacheDir: File, outDir: File): List<File> {
+        repository.mkdirs()
         val ktFiles = srcDir.getFiles().filter { it.extension == "kt" }
 
         val results = mutableMapOf<String, MutableList<Pair<File, String>>>()
@@ -59,7 +70,7 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
         val dependencyGraph = parseDependencies(testDir)
 
         DFS.topologicalOrder(dependencyGraph.keys) { m ->
-            dependencyGraph[m] ?: error("Expected dependencies for module $m")
+            (dependencyGraph[m] ?: error("Expected dependencies for module $m")).map { it.moduleName }
         }.reversed().mapTo(modulesOrder) { it }
 
         for ((moduleName, fileEntries) in results) {
@@ -81,8 +92,8 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
         return listOf(srcDir)
     }
 
-    private fun setupModuleApiHistory(moduleName: String, outDir: File, cacheDir: File) {
+    protected open fun setupModuleApiHistory(moduleName: String, outDir: File, cacheDir: File) {
-        val depKlibFile = File(modulesDir, moduleName.klib)
+        val depArtifactFile = File(repository, moduleName.asArtifactFileName())
         val moduleBuildDir = File(outDir, moduleName)
         val moduleCacheDir = File(cacheDir, moduleName)
         val moduleBuildHistoryFile = buildHistoryFile(moduleCacheDir)
@@ -91,7 +102,7 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
 
         dirToModule[moduleBuildDir] = moduleEntry
         nameToModules.getOrPut(moduleName) { mutableSetOf() }.add(moduleEntry)
-        jarToModule[depKlibFile] = moduleEntry
+        jarToModule[depArtifactFile] = moduleEntry
     }
 
     companion object {
@@ -103,9 +114,7 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
             else listOf(this)
         }
 
-        private val String.klib: String get() = "$this.$KLIB_FILE_EXTENSION"
+        private fun parseDependencies(testDir: File): Map<String, List<ModuleDependency>> {
-
-        private fun parseDependencies(testDir: File): Map<String, List<String>> {
 
             val actualModulesTxtFile = File(testDir, "dependencies.txt")
 
@@ -113,71 +122,91 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
                 error("${actualModulesTxtFile.path} is expected")
             }
 
-            val result = mutableMapOf<String, MutableList<String>>()
+            val result = mutableMapOf<String, MutableList<ModuleDependency>>()
 
             val lines = actualModulesTxtFile.readLines()
-            lines.map { it.split("->") }.map {
+            lines.map { it.split("->") }.forEach {
                 assert(it.size == 2)
                 val moduleName = it[0]
                 val dependencyPart = it[1]
 
-                val idx = dependencyPart.indexOf('[')
-                val dependencyName = if (idx >= 0) {
-                    // skip annotations
-                    dependencyPart.substring(0, idx)
-                } else dependencyPart
-
                 val dependencies = result.getOrPut(moduleName) { mutableListOf() }
-                if (dependencyName.isNotBlank()) {
+
-                    dependencies.add(dependencyName)
+                if (dependencyPart.isNotBlank()) {
+                    val idx = dependencyPart.indexOf('[')
+                    val dependency = if (idx >= 0) {
+                        // skip annotations
+                        val depModuleName = dependencyPart.substring(0, idx)
+                        val flagsString = dependencyPart.substring(idx + 1, dependencyPart.length - 1)
+                        val flags = flagsString.split(",").map { s -> s.trim() }.filter { s -> s.isNotEmpty() }.toSet()
+                        ModuleDependency(depModuleName, flags)
+                    } else ModuleDependency(dependencyPart, emptySet())
+                    dependencies.add(dependency)
                 }
             }
 
             return result
         }
+
+        private const val EXPORTED = "exported"
     }
 
-    private fun K2JSCompilerArguments.updateCompilerArguments(
+    protected abstract fun makeForSingleModule(
-        moduleDependencies: List<String>,
+        moduleCacheDir: File,
-        initialDeps: String,
+        sourceRoots: Iterable<File>,
-        destinationFile: File
+        args: Args,
-    ) {
+        moduleBuildHistoryFile: File,
-        val additionalDeps = moduleDependencies.joinToString(File.pathSeparator) {
+        messageCollector: MessageCollector,
-            File(modulesDir, it.klib).absolutePath
+        reporter: ICReporter,
+        scopeExpansion: CompileScopeExpansionMode,
+        modulesApiHistory: ApiHistory,
+        providedChangedFiles: ChangedFiles?
+    )
+
+    private fun collectEffectiveDependencies(moduleName: String): List<String> {
+        val result = mutableSetOf<String>()
+
+        val moduleInfo = modulesInfo[moduleName] ?: error("Cannot find module info for $moduleName")
+
+        for (dep in moduleInfo.dependencies) {
+            val depName = dep.moduleName
+            result.add(depName)
+
+            val depInfo = modulesInfo[depName] ?: error("Cannot find module info for dependency $moduleName -> $depName")
+            for (depdep in depInfo.dependencies) {
+                if (EXPORTED in depdep.flags) {
+                    result.add(depdep.moduleName)
+                }
+            }
         }
 
-        val sb = StringBuilder(initialDeps)
+        return result.toList()
-        if (additionalDeps.isNotBlank()) {
-            sb.append(File.pathSeparator)
-            sb.append(additionalDeps)
-        }
-
-        libraries = sb.toString()
-        outputFile = destinationFile.path
     }
 
+    protected abstract fun Args.updateForSingleModule(moduleDependencies: List<String>, outFile: File)
+
+    protected abstract fun String.asOutputFileName(): String
+    protected abstract fun String.asArtifactFileName(): String
+
+    protected abstract fun transformToDependency(moduleName: String, rawArtifact: File): File
+
     override fun make(
         cacheDir: File,
         outDir: File,
         sourceRoots: Iterable<File>,
-        args: K2JSCompilerArguments
+        args: Args
     ): TestCompilationResult {
         val reporter = TestICReporter()
         val messageCollector = TestMessageCollector()
-        val initialDeps = args.libraries ?: ""
-
-        args.repositries = modulesDir.path
 
         val modifiedLibraries = mutableListOf<Pair<String, File>>()
         val deletedLibraries = mutableListOf<Pair<String, File>>()
 
         var compilationIsEnabled = true
-        val isInitial = modulesDir.list()?.isEmpty() ?: true
+        val isInitial = repository.list()?.isEmpty() ?: true
 
         for (module in modulesOrder) {
-            val moduleBuildInfo = modulesInfo[module] ?: error("Cannot find config for $module")
+            val moduleDependencies = collectEffectiveDependencies(module)
-
-            val moduleDependencies = moduleBuildInfo.dependencies
 
             val moduleModifiedDependencies = modifiedLibraries.filter { it.first in moduleDependencies }.map { it.second }
             val moduleDeletedDependencies = deletedLibraries.filter { it.first in moduleDependencies }.map { it.second }
@@ -187,15 +216,15 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
             val moduleOutDir = File(outDir, module)
             val moduleCacheDir = File(cacheDir, module)
             val moduleBuildHistory = buildHistoryFile(moduleCacheDir)
+
+            val moduleBuildInfo = modulesInfo[module] ?: error("Cannot find config for $module")
             val sources = moduleBuildInfo.srcDir.getFiles()
 
-            val outputKlibFile = File(moduleOutDir, module.klib)
+            val outputFile = File(moduleOutDir, module.asOutputFileName())
-            val dependencyFile = File(modulesDir, module.klib)
-
-            args.updateCompilerArguments(moduleDependencies, initialDeps, outputKlibFile)
 
             if (compilationIsEnabled) {
-                makeJsIncrementally(
+                args.updateForSingleModule(moduleDependencies, outputFile)
+                makeForSingleModule(
                     moduleCacheDir,
                     sources,
                     args,
@@ -208,6 +237,7 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
                 )
             }
 
+            val dependencyFile = File(repository, module.asArtifactFileName())
             val oldMD5 = if (dependencyFile.exists()) {
                 val bytes = dependencyFile.readBytes()
                 dependencyFile.delete()
@@ -215,11 +245,11 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
             } else 0
 
             if (!messageCollector.hasErrors()) {
-                val newMD5 = outputKlibFile.readBytes().md5()
+                transformToDependency(module, outputFile)
+                val newMD5 = dependencyFile.readBytes().md5()
                 if (oldMD5 != newMD5) {
                     modifiedLibraries.add(module to dependencyFile)
                 }
-                outputKlibFile.copyTo(dependencyFile)
             } else {
                 compilationIsEnabled = false
             }

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalCompilerRunnerTestBase.kt

@@ -81,7 +81,7 @@ abstract class AbstractIncrementalCompilerRunnerTestBase<Args : CommonCompilerAr
         var step = 1
         for ((modificationStep, buildLogStep) in modifications.zip(buildLogSteps)) {
             modificationStep.forEach { it.perform(workingDir, mapWorkingToOriginalFile) }
-            val (_, compiledSources, compileErrors) = incrementalMake(cacheDir, outDir, sourceRoots, createCompilerArguments(outDir, testDir))
+            val (_, compiledSources, compileErrors) = incrementalMake(cacheDir, outDir, sourceRoots, createCompilerArgumentsImpl(outDir, testDir))
 
             expectedSB.appendLine(stepLogAsString(step, buildLogStep.compiledKotlinFiles, buildLogStep.compileErrors))
             expectedSBWithoutErrors.appendLine(
@@ -174,6 +174,9 @@ abstract class AbstractIncrementalCompilerRunnerTestBase<Args : CommonCompilerAr
             KtUsefulTestCase.assertExists(it)
         }
 
+        @JvmStatic
+        protected fun buildHistoryFile(cacheDir: File): File = File(cacheDir, "build-history.bin")
+
         private const val ARGUMENTS_FILE_NAME = "args.txt"
 
         private fun parseAdditionalArgs(testDir: File): List<String> {

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalJsCompilerRunnerTest.kt

@@ -44,7 +44,5 @@ abstract class AbstractIncrementalJsCompilerRunnerTest : AbstractIncrementalComp
             metaInfo = true
         }
 
-    protected fun buildHistoryFile(cacheDir: File): File = File(cacheDir, "build-history.bin")
-
     protected open val scopeExpansionMode = CompileScopeExpansionMode.NEVER
 }

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest.kt

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import org.jetbrains.kotlin.build.report.ICReporter
+import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
+import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryJs
+import org.jetbrains.kotlin.incremental.testingUtils.BuildLogFinder
+import org.jetbrains.kotlin.library.KLIB_FILE_EXTENSION
+import java.io.File
+
+abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest :
+    AbstractIncrementalMultiModuleCompilerRunnerTest<K2JSCompilerArguments, ModulesApiHistoryJs>() {
+
+    override fun createCompilerArguments(destinationDir: File, testDir: File): K2JSCompilerArguments =
+        K2JSCompilerArguments().apply {
+            libraries = STDLIB_DEPENDENCY
+            outputFile = File(destinationDir, "${testDir.name}.$KLIB_FILE_EXTENSION").path
+            sourceMap = false
+            irProduceKlibDir = false
+            irProduceKlibFile = true
+            irOnly = true
+            repositries = repository.absolutePath
+        }
+
+    override val buildLogFinder: BuildLogFinder
+        get() = super.buildLogFinder.copy(isJsIrEnabled = true, isKlibEnabled = true)
+
+    override fun makeForSingleModule(
+        moduleCacheDir: File,
+        sourceRoots: Iterable<File>,
+        args: K2JSCompilerArguments,
+        moduleBuildHistoryFile: File,
+        messageCollector: MessageCollector,
+        reporter: ICReporter,
+        scopeExpansion: CompileScopeExpansionMode,
+        modulesApiHistory: ModulesApiHistoryJs,
+        providedChangedFiles: ChangedFiles?
+    ) {
+        makeJsIncrementally(
+            moduleCacheDir,
+            sourceRoots,
+            args,
+            moduleBuildHistoryFile,
+            messageCollector,
+            reporter,
+            scopeExpansionMode,
+            modulesApiHistory,
+            providedChangedFiles
+        )
+    }
+
+    override val modulesApiHistory: ModulesApiHistoryJs by lazy {
+        ModulesApiHistoryJs(incrementalModuleInfo)
+    }
+
+    override val scopeExpansionMode: CompileScopeExpansionMode get() = CompileScopeExpansionMode.NEVER
+
+    override fun String.asOutputFileName(): String = klib
+    override fun String.asArtifactFileName(): String = klib
+
+    override fun transformToDependency(moduleName: String, rawArtifact: File): File {
+        val dependencyFile = File(repository, moduleName.klib)
+        rawArtifact.copyTo(dependencyFile)
+        return dependencyFile
+    }
+
+    override fun K2JSCompilerArguments.updateForSingleModule(moduleDependencies: List<String>, outFile: File) {
+        val additionalDeps = moduleDependencies.joinToString(File.pathSeparator) {
+            File(repository, it.klib).absolutePath
+        }
+
+        val sb = StringBuilder(STDLIB_DEPENDENCY)
+        if (additionalDeps.isNotBlank()) {
+            sb.append(File.pathSeparator)
+            sb.append(additionalDeps)
+        }
+
+        libraries = sb.toString()
+        outputFile = outFile.path
+    }
+
+    companion object {
+        private val String.klib: String get() = "$this.$KLIB_FILE_EXTENSION"
+
+        private const val STDLIB_DEPENDENCY = "build/js-ir-runtime/full-runtime.klib"
+    }
+}