[IC TEST] Abstract multi module test runner from klib

2020-12-03 09:38:37 +03:00
parent 5b1c70a430
commit 43022a292d
4 changed files with 184 additions and 61 deletions
@@ -5,35 +5,45 @@

 package org.jetbrains.kotlin.incremental

-import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
-import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryJs
-import org.jetbrains.kotlin.incremental.utils.*
-import org.jetbrains.kotlin.library.KLIB_FILE_EXTENSION
+import org.jetbrains.kotlin.build.report.ICReporter
+import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
+import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory
+import org.jetbrains.kotlin.incremental.utils.TestCompilationResult
+import org.jetbrains.kotlin.incremental.utils.TestICReporter
+import org.jetbrains.kotlin.incremental.utils.TestMessageCollector
 import org.jetbrains.kotlin.utils.DFS
 import java.io.File
 import java.util.regex.Pattern

-abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : AbstractIncrementalJsKlibCompilerRunnerTest() {
+abstract class AbstractIncrementalMultiModuleCompilerRunnerTest<Args : CommonCompilerArguments, ApiHistory : ModulesApiHistory> :
+    AbstractIncrementalCompilerRunnerTestBase<Args>() {

-    private class ModuleBuildConfiguration(val srcDir: File, val dependencies: List<String>)
+    private class ModuleDependency(val moduleName: String, val flags: Set<String>)
+    private class ModuleBuildConfiguration(val srcDir: File, val dependencies: List<ModuleDependency>)

-    private val modulesDir: File by lazy { File(workingDir, "modules") }
+    protected val repository: File by lazy { File(workingDir, "repository") }
    private val modulesInfo: MutableMap<String, ModuleBuildConfiguration> = mutableMapOf()
    private val modulesOrder: MutableList<String> = mutableListOf()

    private val dirToModule = mutableMapOf<File, IncrementalModuleEntry>()
    private val nameToModules = mutableMapOf<String, MutableSet<IncrementalModuleEntry>>()
+    private val jarToClassListFile = mutableMapOf<File, File>()
    private val jarToModule = mutableMapOf<File, IncrementalModuleEntry>()

-    private val modulesApiHistory: ModulesApiHistoryJs by lazy {
-        ModulesApiHistoryJs(IncrementalModuleInfo(workingDir, dirToModule, nameToModules, emptyMap(), jarToModule))
+    protected val incrementalModuleInfo: IncrementalModuleInfo by lazy {
+        IncrementalModuleInfo(workingDir, dirToModule, nameToModules, jarToClassListFile, jarToModule)
    }

+    protected abstract val modulesApiHistory: ApiHistory
+
    override val moduleNames: Collection<String>? get() = modulesOrder

+    protected abstract val scopeExpansionMode: CompileScopeExpansionMode
+
    override fun resetTest(testDir: File, newOutDir: File, newCacheDir: File) {
-        modulesDir.deleteRecursively()
-        modulesDir.mkdirs()
+        repository.deleteRecursively()
+        repository.mkdirs()

        dirToModule.clear()
        nameToModules.clear()
@@ -43,6 +53,7 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
    }

    override fun setupTest(testDir: File, srcDir: File, cacheDir: File, outDir: File): List<File> {
+        repository.mkdirs()
        val ktFiles = srcDir.getFiles().filter { it.extension == "kt" }

        val results = mutableMapOf<String, MutableList<Pair<File, String>>>()
@@ -59,7 +70,7 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
        val dependencyGraph = parseDependencies(testDir)

        DFS.topologicalOrder(dependencyGraph.keys) { m ->
-            dependencyGraph[m] ?: error("Expected dependencies for module $m")
+            (dependencyGraph[m] ?: error("Expected dependencies for module $m")).map { it.moduleName }
        }.reversed().mapTo(modulesOrder) { it }

        for ((moduleName, fileEntries) in results) {
@@ -81,8 +92,8 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
        return listOf(srcDir)
    }

-    private fun setupModuleApiHistory(moduleName: String, outDir: File, cacheDir: File) {
-        val depKlibFile = File(modulesDir, moduleName.klib)
+    protected open fun setupModuleApiHistory(moduleName: String, outDir: File, cacheDir: File) {
+        val depArtifactFile = File(repository, moduleName.asArtifactFileName())
        val moduleBuildDir = File(outDir, moduleName)
        val moduleCacheDir = File(cacheDir, moduleName)
        val moduleBuildHistoryFile = buildHistoryFile(moduleCacheDir)
@@ -91,7 +102,7 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract

        dirToModule[moduleBuildDir] = moduleEntry
        nameToModules.getOrPut(moduleName) { mutableSetOf() }.add(moduleEntry)
-        jarToModule[depKlibFile] = moduleEntry
+        jarToModule[depArtifactFile] = moduleEntry
    }

    companion object {
@@ -103,9 +114,7 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
            else listOf(this)
        }

-        private val String.klib: String get() = "$this.$KLIB_FILE_EXTENSION"
-
-        private fun parseDependencies(testDir: File): Map<String, List<String>> {
+        private fun parseDependencies(testDir: File): Map<String, List<ModuleDependency>> {

            val actualModulesTxtFile = File(testDir, "dependencies.txt")

@@ -113,71 +122,91 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
                error("${actualModulesTxtFile.path} is expected")
            }

-            val result = mutableMapOf<String, MutableList<String>>()
+            val result = mutableMapOf<String, MutableList<ModuleDependency>>()

            val lines = actualModulesTxtFile.readLines()
-            lines.map { it.split("->") }.map {
+            lines.map { it.split("->") }.forEach {
                assert(it.size == 2)
                val moduleName = it[0]
                val dependencyPart = it[1]

-                val idx = dependencyPart.indexOf('[')
-                val dependencyName = if (idx >= 0) {
-                    // skip annotations
-                    dependencyPart.substring(0, idx)
-                } else dependencyPart
-
                val dependencies = result.getOrPut(moduleName) { mutableListOf() }
-                if (dependencyName.isNotBlank()) {
-                    dependencies.add(dependencyName)
+
+                if (dependencyPart.isNotBlank()) {
+                    val idx = dependencyPart.indexOf('[')
+                    val dependency = if (idx >= 0) {
+                        // skip annotations
+                        val depModuleName = dependencyPart.substring(0, idx)
+                        val flagsString = dependencyPart.substring(idx + 1, dependencyPart.length - 1)
+                        val flags = flagsString.split(",").map { s -> s.trim() }.filter { s -> s.isNotEmpty() }.toSet()
+                        ModuleDependency(depModuleName, flags)
+                    } else ModuleDependency(dependencyPart, emptySet())
+                    dependencies.add(dependency)
                }
            }

            return result
        }
+
+        private const val EXPORTED = "exported"
    }

-    private fun K2JSCompilerArguments.updateCompilerArguments(
-        moduleDependencies: List<String>,
-        initialDeps: String,
-        destinationFile: File
-    ) {
-        val additionalDeps = moduleDependencies.joinToString(File.pathSeparator) {
-            File(modulesDir, it.klib).absolutePath
+    protected abstract fun makeForSingleModule(
+        moduleCacheDir: File,
+        sourceRoots: Iterable<File>,
+        args: Args,
+        moduleBuildHistoryFile: File,
+        messageCollector: MessageCollector,
+        reporter: ICReporter,
+        scopeExpansion: CompileScopeExpansionMode,
+        modulesApiHistory: ApiHistory,
+        providedChangedFiles: ChangedFiles?
+    )
+
+    private fun collectEffectiveDependencies(moduleName: String): List<String> {
+        val result = mutableSetOf<String>()
+
+        val moduleInfo = modulesInfo[moduleName] ?: error("Cannot find module info for $moduleName")
+
+        for (dep in moduleInfo.dependencies) {
+            val depName = dep.moduleName
+            result.add(depName)
+
+            val depInfo = modulesInfo[depName] ?: error("Cannot find module info for dependency $moduleName -> $depName")
+            for (depdep in depInfo.dependencies) {
+                if (EXPORTED in depdep.flags) {
+                    result.add(depdep.moduleName)
+                }
+            }
        }

-        val sb = StringBuilder(initialDeps)
-        if (additionalDeps.isNotBlank()) {
-            sb.append(File.pathSeparator)
-            sb.append(additionalDeps)
-        }
-
-        libraries = sb.toString()
-        outputFile = destinationFile.path
+        return result.toList()
    }

+    protected abstract fun Args.updateForSingleModule(moduleDependencies: List<String>, outFile: File)
+
+    protected abstract fun String.asOutputFileName(): String
+    protected abstract fun String.asArtifactFileName(): String
+
+    protected abstract fun transformToDependency(moduleName: String, rawArtifact: File): File
+
    override fun make(
        cacheDir: File,
        outDir: File,
        sourceRoots: Iterable<File>,
-        args: K2JSCompilerArguments
+        args: Args
    ): TestCompilationResult {
        val reporter = TestICReporter()
        val messageCollector = TestMessageCollector()
-        val initialDeps = args.libraries ?: ""
-
-        args.repositries = modulesDir.path

        val modifiedLibraries = mutableListOf<Pair<String, File>>()
        val deletedLibraries = mutableListOf<Pair<String, File>>()

        var compilationIsEnabled = true
-        val isInitial = modulesDir.list()?.isEmpty() ?: true
+        val isInitial = repository.list()?.isEmpty() ?: true

        for (module in modulesOrder) {
-            val moduleBuildInfo = modulesInfo[module] ?: error("Cannot find config for $module")
-
-            val moduleDependencies = moduleBuildInfo.dependencies
+            val moduleDependencies = collectEffectiveDependencies(module)

            val moduleModifiedDependencies = modifiedLibraries.filter { it.first in moduleDependencies }.map { it.second }
            val moduleDeletedDependencies = deletedLibraries.filter { it.first in moduleDependencies }.map { it.second }
@@ -187,15 +216,15 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
            val moduleOutDir = File(outDir, module)
            val moduleCacheDir = File(cacheDir, module)
            val moduleBuildHistory = buildHistoryFile(moduleCacheDir)
+
+            val moduleBuildInfo = modulesInfo[module] ?: error("Cannot find config for $module")
            val sources = moduleBuildInfo.srcDir.getFiles()

-            val outputKlibFile = File(moduleOutDir, module.klib)
-            val dependencyFile = File(modulesDir, module.klib)
-
-            args.updateCompilerArguments(moduleDependencies, initialDeps, outputKlibFile)
+            val outputFile = File(moduleOutDir, module.asOutputFileName())

            if (compilationIsEnabled) {
-                makeJsIncrementally(
+                args.updateForSingleModule(moduleDependencies, outputFile)
+                makeForSingleModule(
                    moduleCacheDir,
                    sources,
                    args,
@@ -208,6 +237,7 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
                )
            }

+            val dependencyFile = File(repository, module.asArtifactFileName())
            val oldMD5 = if (dependencyFile.exists()) {
                val bytes = dependencyFile.readBytes()
                dependencyFile.delete()
@@ -215,11 +245,11 @@ abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest : Abstract
            } else 0

            if (!messageCollector.hasErrors()) {
-                val newMD5 = outputKlibFile.readBytes().md5()
+                transformToDependency(module, outputFile)
+                val newMD5 = dependencyFile.readBytes().md5()
                if (oldMD5 != newMD5) {
                    modifiedLibraries.add(module to dependencyFile)
                }
-                outputKlibFile.copyTo(dependencyFile)
            } else {
                compilationIsEnabled = false
            }
@@ -81,7 +81,7 @@ abstract class AbstractIncrementalCompilerRunnerTestBase<Args : CommonCompilerAr
        var step = 1
        for ((modificationStep, buildLogStep) in modifications.zip(buildLogSteps)) {
            modificationStep.forEach { it.perform(workingDir, mapWorkingToOriginalFile) }
-            val (_, compiledSources, compileErrors) = incrementalMake(cacheDir, outDir, sourceRoots, createCompilerArguments(outDir, testDir))
+            val (_, compiledSources, compileErrors) = incrementalMake(cacheDir, outDir, sourceRoots, createCompilerArgumentsImpl(outDir, testDir))

            expectedSB.appendLine(stepLogAsString(step, buildLogStep.compiledKotlinFiles, buildLogStep.compileErrors))
            expectedSBWithoutErrors.appendLine(
@@ -174,6 +174,9 @@ abstract class AbstractIncrementalCompilerRunnerTestBase<Args : CommonCompilerAr
            KtUsefulTestCase.assertExists(it)
        }

+        @JvmStatic
+        protected fun buildHistoryFile(cacheDir: File): File = File(cacheDir, "build-history.bin")
+
        private const val ARGUMENTS_FILE_NAME = "args.txt"

        private fun parseAdditionalArgs(testDir: File): List<String> {
@@ -44,7 +44,5 @@ abstract class AbstractIncrementalJsCompilerRunnerTest : AbstractIncrementalComp
            metaInfo = true
        }

-    protected fun buildHistoryFile(cacheDir: File): File = File(cacheDir, "build-history.bin")
-
    protected open val scopeExpansionMode = CompileScopeExpansionMode.NEVER
 }
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import org.jetbrains.kotlin.build.report.ICReporter
+import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
+import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistoryJs
+import org.jetbrains.kotlin.incremental.testingUtils.BuildLogFinder
+import org.jetbrains.kotlin.library.KLIB_FILE_EXTENSION
+import java.io.File
+
+abstract class AbstractIncrementalMultiModuleJsKlibCompilerRunnerTest :
+    AbstractIncrementalMultiModuleCompilerRunnerTest<K2JSCompilerArguments, ModulesApiHistoryJs>() {
+
+    override fun createCompilerArguments(destinationDir: File, testDir: File): K2JSCompilerArguments =
+        K2JSCompilerArguments().apply {
+            libraries = STDLIB_DEPENDENCY
+            outputFile = File(destinationDir, "${testDir.name}.$KLIB_FILE_EXTENSION").path
+            sourceMap = false
+            irProduceKlibDir = false
+            irProduceKlibFile = true
+            irOnly = true
+            repositries = repository.absolutePath
+        }
+
+    override val buildLogFinder: BuildLogFinder
+        get() = super.buildLogFinder.copy(isJsIrEnabled = true, isKlibEnabled = true)
+
+    override fun makeForSingleModule(
+        moduleCacheDir: File,
+        sourceRoots: Iterable<File>,
+        args: K2JSCompilerArguments,
+        moduleBuildHistoryFile: File,
+        messageCollector: MessageCollector,
+        reporter: ICReporter,
+        scopeExpansion: CompileScopeExpansionMode,
+        modulesApiHistory: ModulesApiHistoryJs,
+        providedChangedFiles: ChangedFiles?
+    ) {
+        makeJsIncrementally(
+            moduleCacheDir,
+            sourceRoots,
+            args,
+            moduleBuildHistoryFile,
+            messageCollector,
+            reporter,
+            scopeExpansionMode,
+            modulesApiHistory,
+            providedChangedFiles
+        )
+    }
+
+    override val modulesApiHistory: ModulesApiHistoryJs by lazy {
+        ModulesApiHistoryJs(incrementalModuleInfo)
+    }
+
+    override val scopeExpansionMode: CompileScopeExpansionMode get() = CompileScopeExpansionMode.NEVER
+
+    override fun String.asOutputFileName(): String = klib
+    override fun String.asArtifactFileName(): String = klib
+
+    override fun transformToDependency(moduleName: String, rawArtifact: File): File {
+        val dependencyFile = File(repository, moduleName.klib)
+        rawArtifact.copyTo(dependencyFile)
+        return dependencyFile
+    }
+
+    override fun K2JSCompilerArguments.updateForSingleModule(moduleDependencies: List<String>, outFile: File) {
+        val additionalDeps = moduleDependencies.joinToString(File.pathSeparator) {
+            File(repository, it.klib).absolutePath
+        }
+
+        val sb = StringBuilder(STDLIB_DEPENDENCY)
+        if (additionalDeps.isNotBlank()) {
+            sb.append(File.pathSeparator)
+            sb.append(additionalDeps)
+        }
+
+        libraries = sb.toString()
+        outputFile = outFile.path
+    }
+
+    companion object {
+        private val String.klib: String get() = "$this.$KLIB_FILE_EXTENSION"
+
+        private const val STDLIB_DEPENDENCY = "build/js-ir-runtime/full-runtime.klib"
+    }
+}