azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[JS IR] IC invalidation refactoring

Browse Source 
- Huge refactoring for IC
  - Update hash combination logic
  - Introduce value class for IC hashes
  - Calc md5 directly by function IR
  - Split IC logic by classes
  - Move JsIrLinkerLoader into separate file
  - CacheUpdateStatus is a sealed class
  - Render TYPE_PARAMETER reified flag

^KT-51081 Fixed
^KT-51084 Fixed
This commit is contained in:
Alexander Korepanov
2022-03-15 05:34:19 +00:00
committed by Space
parent c38dd1c004
commit 69295f2cf0
246 changed files with 1724 additions and 2048 deletions
Download Patch File Download Diff File Expand all files Collapse all files
js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt
+28 -49
View File
@@ -16,6 +16,7 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
import org.jetbrains.kotlin.codegen.*
import org.jetbrains.kotlin.config.CommonConfigurationKeys
import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.ir.backend.js.WholeWorldStageController
import org.jetbrains.kotlin.ir.backend.js.generateKLib
import org.jetbrains.kotlin.ir.backend.js.ic.*
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
@@ -23,6 +24,7 @@ import org.jetbrains.kotlin.ir.backend.js.prepareAnalyzedSourceModule
import org.jetbrains.kotlin.ir.declarations.IrFactory
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl
import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.test.KotlinTestWithEnvironment
@@ -52,8 +54,7 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
deserializer: JsIrLinker,
configuration: CompilerConfiguration,
dirtyFiles: Collection<String>?, // if null consider the whole module dirty
deletedFiles: Collection<String>,
cacheConsumer: PersistentCacheConsumer,
artifactCache: ArtifactCache,
exportedDeclarations: Set<FqName>,
mainArguments: List<String>?,
) {
@@ -160,7 +161,6 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
buildCachesAndCheck(
moduleStep,
configuration,
moduleSourceDir,
outputKlibFile,
moduleCacheDir,
dependencies,
@@ -180,7 +180,6 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
private fun buildCachesAndCheck(
moduleStep: ModuleInfo.ModuleStep,
configuration: CompilerConfiguration,
sourceDir: File,
moduleKlibFile: File,
moduleCacheDir: File,
dependencies: List<File>,
@@ -188,35 +187,6 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
expectedDirtyFiles: List<File>,
expectedDeletedFiles: List<File>
) {
@Suppress("UNUSED_PARAMETER")
fun dirtyFilesChecker(
currentModule: IrModuleFragment,
dependencies: Collection<IrModuleFragment>,
deserializer: JsIrLinker,
configuration: CompilerConfiguration,
invalidatedDirtyFiles: Collection<String>?, // if null consider the whole module dirty
deletedFiles: Collection<String>,
cacheConsumer: PersistentCacheConsumer,
exportedDeclarations: Set<FqName>,
mainArguments: List<String>?,
) {
val actualDirtyFiles =
invalidatedDirtyFiles?.map { File(it).canonicalPath } ?: sourceDir.filteredKtFiles().map { it.canonicalPath }
val expectedDirtyFilesCanonical = expectedDirtyFiles.map { it.canonicalPath }
JUnit4Assertions.assertSameElements(expectedDirtyFilesCanonical, actualDirtyFiles) {
"Mismatched DIRTY files for module $moduleKlibFile at step ${moduleStep.id}"
}
val actualDeletedFiles =
deletedFiles.map { File(it).canonicalPath }
val expectedDeletedFilesCanonical = expectedDeletedFiles.map { it.canonicalPath }
JUnit4Assertions.assertSameElements(expectedDeletedFilesCanonical, actualDeletedFiles) {
"Mismatched DELETED files for module $moduleKlibFile at step ${moduleStep.id}"
}
}
val dependenciesPaths = mutableListOf<String>()
dependencies.mapTo(dependenciesPaths) { it.canonicalPath }
dependenciesPaths.add(moduleKlibFile.canonicalPath)
@@ -229,18 +199,31 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
icCaches.map { it.canonicalPath },// + moduleCacheDir.canonicalPath,
IrFactoryImpl,
null, // TODO: mainArguments
::dirtyFilesChecker
::emptyChecker
)
if (StepDirectives.FAST_PATH_UPDATE in moduleStep.directives) {
JUnit4Assertions.assertEquals(CacheUpdateStatus.FAST_PATH, updateStatus) {
JUnit4Assertions.assertEquals(CacheUpdateStatus.FastPath, updateStatus) {
"Cache has to be checked by fast path, instead it $updateStatus"
}
}
JUnit4Assertions.assertEquals(expectedDirtyFiles.isEmpty(), updateStatus.upToDate) {
val filePaths = expectedDirtyFiles.joinToString(",", "[", "]")
"Up to date is not expected for module $moduleKlibFile at step ${moduleStep.id}. Expected dirtyFiles are $filePaths"
val (updated, removed) = when (updateStatus) {
is CacheUpdateStatus.FastPath -> emptySet<String>() to emptySet<String>()
is CacheUpdateStatus.NoDirtyFiles -> emptySet<String>() to updateStatus.removed
is CacheUpdateStatus.Dirty -> updateStatus.updated to updateStatus.removed
}
val actualDirtyFiles = updated.map { File(it).canonicalPath }
val expectedDirtyFilesCanonical = expectedDirtyFiles.map { it.canonicalPath }
JUnit4Assertions.assertSameElements(expectedDirtyFilesCanonical, actualDirtyFiles) {
"Mismatched DIRTY files for module $moduleKlibFile at step ${moduleStep.id}"
}
val actualDeletedFiles = removed.map { File(it).canonicalPath }
val expectedDeletedFilesCanonical = expectedDeletedFiles.map { it.canonicalPath }
JUnit4Assertions.assertSameElements(expectedDeletedFilesCanonical, actualDeletedFiles) {
"Mismatched DELETED files for module $moduleKlibFile at step ${moduleStep.id}"
}
}
@@ -248,28 +231,24 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
moduleName: String,
cachePath: String,
compilerConfiguration: CompilerConfiguration,
dependencies: Collection<ModulePath>,
dependencies: Collection<String>,
icCachePaths: Collection<String>,
irFactory: IrFactory,
mainArguments: List<String>?,
executor: CacheExecutor
): CacheUpdateStatus {
val modulePath = moduleName.toCanonicalPath()
val icCacheMap: Map<ModulePath, CacheInfo> = loadCacheInfo(icCachePaths).also {
it[modulePath] = CacheInfo.loadOrCreate(cachePath, modulePath)
}
val modulePath = File(moduleName).canonicalPath
val statuses = mutableMapOf<String, CacheUpdateStatus>()
actualizeCaches(
val cacheUpdater = CacheUpdater(
modulePath,
compilerConfiguration,
dependencies,
icCacheMap.map { it.value.path },
compilerConfiguration,
icCachePaths + cachePath,
{ irFactory },
mainArguments,
executor
) { updateStatus, updatedModule ->
)
cacheUpdater.actualizeCaches { updateStatus, updatedModule ->
statuses[updatedModule] = updateStatus
}
return statuses[modulePath] ?: error("Status is missed for $modulePath")
js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt
+17 -109
View File
@@ -10,7 +10,6 @@ import org.jetbrains.kotlin.ir.backend.js.WholeWorldStageController
import org.jetbrains.kotlin.ir.backend.js.ic.*
import org.jetbrains.kotlin.ir.backend.js.moduleName
import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC
import org.jetbrains.kotlin.ir.util.IdSignature
import org.jetbrains.kotlin.js.test.handlers.JsBoxRunner
import org.jetbrains.kotlin.konan.properties.propertyList
import org.jetbrains.kotlin.library.KLIB_PROPERTY_DEPENDS
@@ -25,103 +24,16 @@ import org.jetbrains.kotlin.test.services.configuration.JsEnvironmentConfigurato
import org.jetbrains.kotlin.test.services.jsLibraryProvider
import java.io.File
class TestModuleCache(val files: MutableMap<String, FileCache>) {
constructor() : this(mutableMapOf())
private lateinit var storedModuleName: String
fun cacheProvider(): PersistentCacheProvider {
return object : PersistentCacheProvider {
override fun fileFingerPrint(path: String): Hash {
return 0L
}
override fun inlineGraphForFile(path: String, sigResolver: (Int) -> IdSignature): Collection<Pair<IdSignature, TransHash>> {
error("Is not supported")
}
override fun inlineHashes(path: String, sigResolver: (Int) -> IdSignature): Map<IdSignature, TransHash> {
error("Is not supported")
}
override fun allInlineHashes(sigResolver: (String, Int) -> IdSignature): Map<IdSignature, TransHash> {
error("Is not supported")
}
override fun binaryAst(path: String): ByteArray? {
return files[path]?.ast ?: ByteArray(0)
}
override fun dts(path: String): ByteArray? {
return files[path]?.dts
}
override fun sourceMap(path: String): ByteArray? {
return files[path]?.sourceMap
}
override fun filePaths(): Iterable<String> {
return files.keys
}
override fun moduleName(): String {
return storedModuleName
}
private class TestArtifactCache(var moduleName: String? = null) : ArtifactCache() {
override fun fetchArtifacts() = KLibArtifact(
moduleName = moduleName ?: error("Module name is not set"),
fileArtifacts = binaryAsts.entries.map {
SrcFileArtifact(it.key, "", it.value)
}
}
)
fun cacheConsumer(): PersistentCacheConsumer {
return object : PersistentCacheConsumer {
override fun commitInlineFunctions(
path: String,
hashes: Collection<Pair<IdSignature, TransHash>>,
sigResolver: (IdSignature) -> Int
) {
}
override fun commitFileFingerPrint(path: String, fingerprint: Hash) {
}
override fun commitInlineGraph(
path: String,
hashes: Collection<Pair<IdSignature, TransHash>>,
sigResolver: (IdSignature) -> Int
) {
}
override fun commitBinaryAst(path: String, astData: ByteArray) {
val storage = files.getOrPut(path) { FileCache(path, null, null, null) }
storage.ast = astData
}
override fun commitBinaryDts(path: String, dstData: ByteArray) {
val storage = files.getOrPut(path) { FileCache(path, null, null, null) }
storage.dts = dstData
}
override fun commitSourceMap(path: String, mapData: ByteArray) {
val storage = files.getOrPut(path) { FileCache(path, null, null, null) }
storage.sourceMap = mapData
}
override fun invalidateForFile(path: String) {
files.remove(path)
}
override fun invalidate() {
}
override fun commitLibraryInfo(libraryPath: String, moduleName: String, flatHash: ULong, transHash: ULong, configHash: ULong) {
storedModuleName = moduleName
}
}
}
fun createModuleCache(): ModuleCache = ModuleCache(storedModuleName, files)
fun invalidateForFile(srcPath: String) = binaryAsts.remove(srcPath)
fun getAst(srcPath: String) = binaryAsts[srcPath]
}
class JsIrIncrementalDataProvider(private val testServices: TestServices) : TestService {
@@ -129,17 +41,15 @@ class JsIrIncrementalDataProvider(private val testServices: TestServices) : Test
private val defaultRuntimeKlib = System.getProperty("kotlin.js.reduced.stdlib.path")
private val kotlinTestKLib = System.getProperty("kotlin.js.kotlin.test.path")
private val predefinedKlibHasIcCache = mutableMapOf<String, TestModuleCache?>(
private val predefinedKlibHasIcCache = mutableMapOf<String, TestArtifactCache?>(
File(fullRuntimeKlib).absolutePath to null,
File(kotlinTestKLib).absolutePath to null,
File(defaultRuntimeKlib).absolutePath to null
)
private val icCache: MutableMap<String, TestModuleCache> = mutableMapOf()
private val icCache: MutableMap<String, TestArtifactCache> = mutableMapOf()
fun getCaches(): Map<String, ModuleCache> {
return icCache.map { it.key to it.value.createModuleCache() }.toMap()
}
fun getCaches() = icCache.map { it.value.fetchArtifacts() }
fun getCacheForModule(module: TestModule): Map<String, ByteArray> {
val path = JsEnvironmentConfigurator.getJsKlibArtifactPath(testServices, module.name)
@@ -147,14 +57,12 @@ class JsIrIncrementalDataProvider(private val testServices: TestServices) : Test
val moduleCache = icCache[canonicalPath] ?: error("No cache found for $path")
val oldBinaryAsts = mutableMapOf<String, ByteArray>()
val dataProvider = moduleCache.cacheProvider()
val dataConsumer = moduleCache.cacheConsumer()
for (testFile in module.files) {
if (JsEnvironmentConfigurationDirectives.RECOMPILE in testFile.directives) {
val fileName = "/${testFile.name}"
oldBinaryAsts[fileName] = dataProvider.binaryAst(fileName) ?: error("No AST found for $fileName")
dataConsumer.invalidateForFile(fileName)
oldBinaryAsts[fileName] = moduleCache.getAst(fileName) ?: error("No AST found for $fileName")
moduleCache.invalidateForFile(fileName)
}
}
@@ -202,11 +110,11 @@ class JsIrIncrementalDataProvider(private val testServices: TestServices) : Test
var moduleCache = predefinedKlibHasIcCache[canonicalPath]
if (moduleCache == null) {
moduleCache = icCache[canonicalPath] ?: TestModuleCache()
moduleCache = icCache[canonicalPath] ?: TestArtifactCache()
val libs = allDependencies.associateBy { File(it.libraryFile.path).canonicalPath }
val nameToKotlinLibrary: Map<ModuleName, KotlinLibrary> = libs.values.associateBy { it.moduleName }
val nameToKotlinLibrary: Map<String, KotlinLibrary> = libs.values.associateBy { it.moduleName }
val dependencyGraph = libs.values.associateWith {
it.manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true).map { depName ->
@@ -218,12 +126,12 @@ class JsIrIncrementalDataProvider(private val testServices: TestServices) : Test
val testPackage = extractTestPackage(testServices)
rebuildCacheForDirtyFiles(
moduleCache.moduleName = rebuildCacheForDirtyFiles(
currentLib,
configuration,
dependencyGraph,
dirtyFiles,
moduleCache.cacheConsumer(),
moduleCache,
IrFactoryImplForJsIC(WholeWorldStageController()),
setOf(FqName.fromSegments(listOfNotNull(testPackage, JsBoxRunner.TEST_FUNCTION))),
mainArguments,