From 85025b85b8502230b60fdac2a9a82c18a1d92631 Mon Sep 17 00:00:00 2001 From: Roman Artemev Date: Tue, 23 Nov 2021 20:44:48 +0300 Subject: [PATCH] [JS IR] Implement runner to test IC invalidation - add corresponding gradle task --- js/js.tests/build.gradle.kts | 13 + .../incremental/AbstractInvalidationTest.kt | 274 ++++++++++++++++++ 2 files changed, 287 insertions(+) create mode 100644 js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt diff --git a/js/js.tests/build.gradle.kts b/js/js.tests/build.gradle.kts index 70417895069..35cb536377d 100644 --- a/js/js.tests/build.gradle.kts +++ b/js/js.tests/build.gradle.kts @@ -274,6 +274,8 @@ projectTest(parallel = true, jUnitMode = JUnitMode.JUnit5) { outputs.dir("$buildDir/out") outputs.dir("$buildDir/out-min") + systemProperty("kotlin.js.stdlib.klib.path", "libraries/stdlib/js-ir/build/libs/kotlin-stdlib-js-ir-js-$version.klib") + configureTestDistribution() } @@ -348,3 +350,14 @@ projectTest("wasmTest", true) { setUpBoxTests() } + +projectTest("invalidationTest", jUnitMode = JUnitMode.JUnit4) { + workingDir = rootDir + + include("org/jetbrains/kotlin/incremental/*") + + dependsOn(":dist") + dependsOn(":kotlin-stdlib-js-ir:compileKotlinJs") + + systemProperty("kotlin.js.stdlib.klib.path", "libraries/stdlib/js-ir/build/libs/kotlin-stdlib-js-ir-js-$version.klib") +} \ No newline at end of file diff --git a/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt b/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt new file mode 100644 index 00000000000..e662e59cde3 --- /dev/null +++ b/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt @@ -0,0 +1,274 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental + +import com.intellij.openapi.vfs.StandardFileSystems +import com.intellij.openapi.vfs.VirtualFileManager +import com.intellij.openapi.vfs.local.CoreLocalFileSystem +import com.intellij.psi.PsiManager +import com.intellij.psi.SingleRootFileViewProvider +import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport +import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles +import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment +import org.jetbrains.kotlin.config.CommonConfigurationKeys +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.ir.backend.js.generateKLib +import org.jetbrains.kotlin.ir.backend.js.ic.PersistentCacheConsumer +import org.jetbrains.kotlin.ir.backend.js.ic.actualizeCacheForModule +import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker +import org.jetbrains.kotlin.ir.backend.js.prepareAnalyzedSourceModule +import org.jetbrains.kotlin.ir.declarations.IrModuleFragment +import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.psi.KtFile +import org.jetbrains.kotlin.test.KotlinTestWithEnvironment +import org.jetbrains.kotlin.test.util.JUnit4Assertions +import java.io.File +import kotlin.io.path.createTempDirectory + +private var stdlibCacheDir: File? = null + +abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() { + override fun createEnvironment(): KotlinCoreEnvironment { + return KotlinCoreEnvironment.createForTests(TestDisposable(), CompilerConfiguration(), EnvironmentConfigFiles.JS_CONFIG_FILES) + } + + private fun parseProjectInfo(testName: String, infoFile: File): ProjectInfo { + return ProjectInfoParser(infoFile).parse(testName) + } + + private fun parseModuleInfo(moduleName: String, infoFile: File): ModuleInfo { + return ModuleInfoParser(infoFile).parse(moduleName) + } + + @Suppress("UNUSED_PARAMETER") + private fun emptyChecker( + currentModule: IrModuleFragment, + dependencies: Collection, + deserializer: JsIrLinker, + configuration: CompilerConfiguration, + dirtyFiles: Collection?, // if null consider the whole module dirty + cacheConsumer: PersistentCacheConsumer, + exportedDeclarations: Set, + mainArguments: List?, + ) { } + + private fun initializeStdlibCache() { + if (stdlibCacheDir != null) return + val cacheDir = createTempDirectory().toFile() + cacheDir.deleteOnExit() + + val configuration = createConfiguration("stdlib") + + actualizeCacheForModule(stdlibKlibPath, cacheDir.canonicalPath, configuration, listOf(stdlibKlibPath), emptyList(), IrFactoryImpl, ::emptyChecker) + + stdlibCacheDir = cacheDir + } + + protected fun doTest(testPath: String) { + val testDirectory = File(testPath) + val testName = testDirectory.name + val projectInfoFile = File(testDirectory, PROJECT_INFO_FILE) + val projectInfo = parseProjectInfo(testName, projectInfoFile) + + if (projectInfo.muted) return + + val modulesInfos = mutableMapOf() + for (module in projectInfo.modules) { + val moduleDirectory = File(testDirectory, module) + val moduleInfo = File(moduleDirectory, MODULE_INFO_FILE) + modulesInfos[module] = parseModuleInfo(module, moduleInfo) + } + + initializeStdlibCache() + + val workingDir = testWorkingDir(projectInfo.name) + val sourceDir = File(workingDir, "sources").also { it.invalidateDir() } + val buildDir = File(workingDir, "build").also { it.invalidateDir() } + + initializeWorkingDir(projectInfo, testDirectory, sourceDir, buildDir) + + executeProjectSteps(projectInfo, modulesInfos, testDirectory, sourceDir, buildDir) + } + + private val stdlibKlibPath = System.getProperty("kotlin.js.stdlib.klib.path") ?: error("Please set stdlib path") + + private val String.artifactName: String get() = "$this.klib" + + private fun resolveModuleArtifact(moduleName: String, buildDir: File): File { + if (moduleName == "stdlib") return File(stdlibKlibPath) + return File(File(buildDir, moduleName), moduleName.artifactName) + } + + private fun resolveModuleCache(moduleName: String, buildDir: File): File { + if (moduleName == "stdlib") return stdlibCacheDir ?: error("stdlib cache corruption") + return File(File(buildDir, moduleName), "cache") + } + + private fun createConfiguration(moduleName: String): CompilerConfiguration { + val copy = environment.configuration.copy() + copy.put(CommonConfigurationKeys.MODULE_NAME, moduleName) + return copy + } + + private fun executeProjectSteps( + projectInfo: ProjectInfo, + moduleInfos: Map, + testDir: File, + sourceDir: File, + buildDir: File + ) { + for (projStep in projectInfo.steps) { + for (module in projStep.order) { + val moduleTestDir = File(testDir, module) + val moduleSourceDir = File(sourceDir, module) + val moduleInfo = moduleInfos[module] ?: error("No module info found for $module") + val moduleStep = moduleInfo.steps[projStep.id] + for (modification in moduleStep.modifications) { + modification.execute(moduleTestDir, moduleSourceDir) + } + + val dependencies = moduleStep.dependencies.map { resolveModuleArtifact(it, buildDir) } + + val outputKlibFile = resolveModuleArtifact(module, buildDir) + + val configuration = createConfiguration(module) + + buildArtifact(configuration, module, moduleSourceDir, dependencies, outputKlibFile) + + val dirtyFiles = moduleStep.dirtyFiles.map { File(moduleSourceDir, it) } + val icCaches = moduleStep.dependencies.map { resolveModuleCache(it, buildDir) } + + val moduleCacheDir = resolveModuleCache(module, buildDir) + + buildCachesAndCheck(moduleStep.id, configuration, moduleSourceDir, outputKlibFile, moduleCacheDir, dependencies, icCaches, dirtyFiles) + } + } + } + + private fun File.filteredKtFiles(): Collection { + assert(isDirectory && exists()) + return listFiles { _, name -> name.endsWith(".kt") }!!.toList() + } + + private fun buildCachesAndCheck( + stepId: Int, + configuration: CompilerConfiguration, + sourceDir: File, + moduleKlibFile: File, + moduleCacheDir: File, + dependencies: List, + icCaches: List, + expectedDirtyFiles: List + ) { + @Suppress("UNUSED_PARAMETER") + fun dirtyFilesChecker( + currentModule: IrModuleFragment, + dependencies: Collection, + deserializer: JsIrLinker, + configuration: CompilerConfiguration, + invalidatedDirtyFiles: Collection?, // if null consider the whole module dirty + cacheConsumer: PersistentCacheConsumer, + exportedDeclarations: Set, + mainArguments: List?, + ) { + val actualDirtyFiles = invalidatedDirtyFiles?.map { File(it).canonicalPath } ?: sourceDir.filteredKtFiles().map { it.canonicalPath } + val expectedDirtyFilesCanonical = expectedDirtyFiles.map { it.canonicalPath } + + JUnit4Assertions.assertSameElements(expectedDirtyFilesCanonical, actualDirtyFiles) { "For module $moduleKlibFile at step $stepId" } + } + + val dependenciesPaths = mutableListOf() + dependencies.mapTo(dependenciesPaths) { it.canonicalPath } + dependenciesPaths.add(moduleKlibFile.canonicalPath) + + val upToDate = actualizeCacheForModule( + moduleKlibFile.canonicalPath, + moduleCacheDir.canonicalPath, + configuration, + dependenciesPaths, + icCaches.map { it.canonicalPath },// + moduleCacheDir.canonicalPath, + IrFactoryImpl, + ::dirtyFilesChecker + ) + + JUnit4Assertions.assertEquals(expectedDirtyFiles.isEmpty(), upToDate) { + val filePaths = expectedDirtyFiles.joinToString(",", "[", "]") + "Up to date is not expected for module $moduleKlibFile at step $stepId. Expected dirtyFiles are $filePaths" + } + } + + private fun KotlinCoreEnvironment.createPsiFile(file: File): KtFile { + val psiManager = PsiManager.getInstance(project) + val fileSystem = VirtualFileManager.getInstance().getFileSystem(StandardFileSystems.FILE_PROTOCOL) as CoreLocalFileSystem + + val vFile = fileSystem.findFileByIoFile(file) ?: error("File not found: $file") + + return SingleRootFileViewProvider(psiManager, vFile).allFiles.find { + it is KtFile && it.virtualFile.canonicalPath == vFile.canonicalPath + } as KtFile + } + + private fun buildArtifact( + configuration: CompilerConfiguration, + moduleName: String, + sourceDir: File, + dependencies: Collection, + outputKlibFile: File + ) { + if (outputKlibFile.exists()) outputKlibFile.delete() + + val projectJs = environment.project + + val sourceFiles = sourceDir.filteredKtFiles().map { environment.createPsiFile(it) } + + val sourceModule = prepareAnalyzedSourceModule( + projectJs, + sourceFiles, + configuration, + dependencies.map { it.canonicalPath }, + emptyList(), // TODO + AnalyzerWithCompilerReport(configuration) + ) + + generateKLib(sourceModule, IrFactoryImpl, outputKlibFile.canonicalPath, nopack = false, jsOutputName = moduleName) + } + + private fun initializeWorkingDir(projectInfo: ProjectInfo, testDir: File, sourceDir: File, buildDir: File) { + for (module in projectInfo.modules) { + val moduleSourceDir = File(sourceDir, module).also { it.invalidateDir() } + File(buildDir, module).invalidateDir() + val testModuleDir = File(testDir, module) + + testModuleDir.listFiles { _, fileName -> fileName.endsWith(".kt") }!!.forEach { file -> + assert(!file.isDirectory) + val fileName = file.name + val workingFile = File(moduleSourceDir, fileName) + file.copyTo(workingFile) + } + } + } + + private fun File.invalidateDir() { + if (exists()) deleteRecursively() + mkdirs() + } + + private fun testWorkingDir(testName: String): File { + val dir = File(File(File(TEST_DATA_DIR_PATH), "incrementalOut/invalidation"), testName) + + dir.invalidateDir() + + return dir + } + + companion object { + const val PROJECT_INFO_FILE = "project.info" + const val MODULE_INFO_FILE = "module.info" + const val TEST_DATA_DIR_PATH = "js/js.translator/testData/" + } + +} \ No newline at end of file