azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[JS IR IC] Implement fast path invalidation check

Browse Source 
Compute library md5 hash and check it first with cached one. If hashes
are equal it means that cache is up-to-date and no additional checks are
needed

 - store library hashes (flat + trans) into cache info file
 - change invalidator return value
This commit is contained in:
Roman Artemev
2021-12-13 17:18:09 +03:00
committed by teamcity
parent 5ba6ca4c16
commit b719865c25
6 changed files with 132 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt
+4 -7
View File
@@ -208,7 +208,7 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
val start = System.currentTimeMillis()
val updated = actualizeCacheForModule(
val updateStatus = actualizeCacheForModule(
includes,
outputFilePath,
configurationJs,
@@ -218,13 +218,10 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
::buildCacheForModuleFiles
)
if (updated) {
messageCollector.report(INFO, "IC per-file cache building duration: ${System.currentTimeMillis() - start}ms")
if (updateStatus.upToDate) {
messageCollector.report(INFO, "IC per-file cache up-to-date check duration: ${System.currentTimeMillis() - start}ms")
} else {
messageCollector.report(
INFO,
"IC per-file cache up-to-date check duration: ${System.currentTimeMillis() - start}ms"
)
messageCollector.report(INFO, "IC per-file cache building duration: ${System.currentTimeMillis() - start}ms")
}
return OK
}
compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/fileUtil.kt
+8 -7
View File
@@ -45,7 +45,7 @@ fun buildCache(
if (!forceClean) {
val oldCacheInfo = CacheInfo.load(cachePath)
if (oldCacheInfo != null && md5 == oldCacheInfo.md5) return false
if (oldCacheInfo != null && md5 == oldCacheInfo.flatHash) return false
}
val icDir = File(cachePath)
@@ -57,7 +57,7 @@ fun buildCache(
icData.serializedIcData.writeTo(File(cachePath))
CacheInfo(cachePath, mainModule.libPath, md5).save()
CacheInfo(cachePath, mainModule.libPath, md5, 0UL).save()
return true
}
@@ -117,18 +117,19 @@ fun checkCaches(
if (c.libPath !in allLibs) error("Missing library: ${c.libPath}")
result[c.libPath] = File(c.path).readIcData()
md5[c.libPath] = c.md5
md5[c.libPath] = c.flatHash
}
return IcCacheInfo(result, md5)
}
// TODO md5 hash
data class CacheInfo(val path: String, val libPath: String, val md5: ULong) {
data class CacheInfo(val path: String, val libPath: String, var flatHash: ULong, var transHash: ULong) {
fun save() {
PrintWriter(File(File(path), "info")).use {
it.println(libPath)
it.println(md5.toString(16))
it.println(flatHash.toString(16))
it.println(transHash.toString(16))
}
}
@@ -138,9 +139,9 @@ data class CacheInfo(val path: String, val libPath: String, val md5: ULong) {
if (!info.exists()) return null
val (libPath, md5) = info.readLines()
val (libPath, flatHash, transHash) = info.readLines()
return CacheInfo(path, libPath, md5.toULong(16))
return CacheInfo(path, libPath, flatHash.toULong(16), transHash.toULong(16))
}
}
}
compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/invalidation.kt
+97 -21
View File
@@ -36,6 +36,7 @@ import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
import org.jetbrains.kotlin.psi2ir.generators.TypeTranslatorImpl
import org.jetbrains.kotlin.storage.LockBasedStorageManager
import java.io.File
import java.security.MessageDigest
import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile as ProtoFile
private fun KotlinLibrary.fingerprint(fileIndex: Int): Hash {
@@ -142,7 +143,7 @@ private fun KotlinLibrary.filesAndSigReaders(): List<Pair<String, IdSignatureDes
}
private fun buildCacheForModule(
libraryPath: String,
libraryInfo: CacheInfo,
configuration: CompilerConfiguration,
irModule: IrModuleFragment,
deserializer: JsIrLinker,
@@ -195,7 +196,7 @@ private fun buildCacheForModule(
cacheExecutor.execute(irModule, dependencies, deserializer, configuration, dirtyFiles, cacheConsumer, emptySet(), null) // TODO: main arguments?
cacheConsumer.commitLibraryPath(libraryPath)
cacheConsumer.commitLibraryPath(libraryInfo.libPath.toCanonicalPath(), libraryInfo.flatHash, libraryInfo.transHash)
}
private fun loadModules(
@@ -268,10 +269,10 @@ private fun createCacheConsumer(path: String): PersistentCacheConsumer {
return PersistentCacheConsumerImpl(path)
}
private fun loadCacheInfo(cachePaths: Collection<String>): MutableMap<ModulePath, String> {
private fun loadCacheInfo(cachePaths: Collection<String>): MutableMap<ModulePath, CacheInfo> {
val caches = cachePaths.map { CacheInfo.load(it) ?: error("Cannot load IC cache from $it") }
val result = mutableMapOf<ModulePath, String>()
return caches.associateTo(result) { it.libPath.toCanonicalPath() to it.path }
val result = mutableMapOf<ModulePath, CacheInfo>()
return caches.associateByTo(result) { it.libPath.toCanonicalPath() }
}
private fun loadLibraries(configuration: CompilerConfiguration, dependencies: Collection<String>): Map<ModulePath, KotlinLibrary> {
@@ -304,6 +305,76 @@ fun interface CacheExecutor {
)
}
private fun File.md5(additional: Iterable<ULong> = emptyList()): ULong {
val md5 = MessageDigest.getInstance("MD5")
for (ul in additional) {
md5.update(ul.toLong().toByteArray())
}
fun File.process(prefix: String = "") {
if (isDirectory) {
this.listFiles()!!.sortedBy { it.name }.forEach {
md5.update((prefix + it.name).toByteArray())
it.process(prefix + it.name + "/")
}
} else {
md5.update(readBytes())
}
}
this.process()
val d = md5.digest()
return ((d[0].toULong() and 0xFFUL)
or ((d[1].toULong() and 0xFFUL) shl 8)
or ((d[2].toULong() and 0xFFUL) shl 16)
or ((d[3].toULong() and 0xFFUL) shl 24)
or ((d[4].toULong() and 0xFFUL) shl 32)
or ((d[5].toULong() and 0xFFUL) shl 40)
or ((d[6].toULong() and 0xFFUL) shl 48)
or ((d[7].toULong() and 0xFFUL) shl 56)
)
}
private fun checkLibrariesHash(
libraries: Map<ModulePath, KotlinLibrary>,
dependencyGraph: Map<KotlinLibrary, List<KotlinLibrary>>,
icCacheMap: Map<ModulePath, CacheInfo>,
modulePath: ModulePath
): Boolean {
val currentLib = libraries[modulePath] ?: error("1")
val currentCache = icCacheMap[modulePath] ?: error("2")
val flatHash = File(modulePath).md5()
val dependencies = dependencyGraph[currentLib] ?: error("3")
var transHash = flatHash
for (dep in dependencies) {
val depCache = icCacheMap[dep.libraryFile.canonicalPath] ?: error("4")
transHash += depCache.transHash
}
if (currentCache.transHash != transHash) {
currentCache.flatHash = flatHash
currentCache.transHash = transHash
return false
}
return true
}
enum class CacheUpdateStatus(val upToDate: Boolean) {
DIRTY(upToDate = false),
NO_DIRTY_FILES(upToDate = true),
FAST_PATH(upToDate = true)
}
// Returns true if caches up-to-date
fun actualizeCacheForModule(
moduleName: String,
cachePath: String,
@@ -312,41 +383,46 @@ fun actualizeCacheForModule(
icCachePaths: Collection<String>,
irFactory: IrFactory,
executor: CacheExecutor
): Boolean {
val icCacheMap: Map<ModulePath, String> = loadCacheInfo(icCachePaths).also {
it[moduleName.toCanonicalPath()] = cachePath
): CacheUpdateStatus {
val modulePath = moduleName.toCanonicalPath()
val cacheInfo = CacheInfo.load(cachePath) ?: CacheInfo(cachePath, modulePath, 0UL, 0UL)
val icCacheMap: Map<ModulePath, CacheInfo> = loadCacheInfo(icCachePaths).also {
it[modulePath] = cacheInfo
}
val libraries: Map<ModulePath, KotlinLibrary> = loadLibraries(compilerConfiguration, dependencies)
val persistentCacheProviders = icCacheMap.map { (lib, cache) ->
libraries[lib.toCanonicalPath()]!! to createCacheProvider(cache)
}.toMap()
val nameToKotlinLibrary: Map<ModuleName, KotlinLibrary> = libraries.values.associateBy { it.moduleName }
val dependencyGraph = libraries.values.associateWith {
it.manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true).map { depName ->
nameToKotlinLibrary[depName] ?: error("No Library found for $depName")
}
}
if (checkLibrariesHash(libraries, dependencyGraph, icCacheMap, modulePath)) {
return CacheUpdateStatus.FAST_PATH // up-to-date
}
val persistentCacheProviders = icCacheMap.map { (lib, cache) ->
libraries[lib.toCanonicalPath()]!! to createCacheProvider(cache.path)
}.toMap()
val currentModule = libraries[moduleName.toCanonicalPath()] ?: error("No loaded library found for path $moduleName")
val persistentCacheConsumer = createCacheConsumer(cachePath)
return actualizeCacheForModule(currentModule, compilerConfiguration, dependencyGraph, persistentCacheProviders, persistentCacheConsumer, irFactory, executor)
return actualizeCacheForModule(currentModule, cacheInfo, compilerConfiguration, dependencyGraph, persistentCacheProviders, persistentCacheConsumer, irFactory, executor)
}
private fun actualizeCacheForModule(
library: KotlinLibrary,
libraryInfo: CacheInfo,
configuration: CompilerConfiguration,
dependencyGraph: Map<KotlinLibrary, Collection<KotlinLibrary>>,
persistentCacheProviders: Map<KotlinLibrary, PersistentCacheProvider>,
persistentCacheConsumer: PersistentCacheConsumer,
irFactory: IrFactory,
cacheExecutor: CacheExecutor
): Boolean {
): CacheUpdateStatus {
// 1. Invalidate
val dependencies = dependencyGraph[library]!!
@@ -393,7 +469,7 @@ private fun actualizeCacheForModule(
fileFingerPrints
)
if (dirtySet.isEmpty()) return true // up-to-date
if (dirtySet.isEmpty()) return CacheUpdateStatus.NO_DIRTY_FILES // up-to-date
// 2. Build
@@ -431,7 +507,7 @@ private fun actualizeCacheForModule(
val deserializers = dirtySet.associateWith { currentModuleDeserializer.signatureDeserializerForFile(it).signatureToIndexMapping() }
buildCacheForModule(
library.libraryFile.canonicalPath,
libraryInfo,
configuration,
currentIrModule,
jsIrLinker,
@@ -443,7 +519,7 @@ private fun actualizeCacheForModule(
fileFingerPrints,
cacheExecutor
)
return false // invalidated and re-built
return CacheUpdateStatus.DIRTY // invalidated and re-built
}
fun rebuildCacheForDirtyFiles(
@@ -525,10 +601,10 @@ fun buildCacheForModuleFiles(
fun loadModuleCaches(icCachePaths: Collection<String>): Map<String, ModuleCache> {
val icCacheMap: Map<ModulePath, String> = loadCacheInfo(icCachePaths)
val icCacheMap: Map<ModulePath, CacheInfo> = loadCacheInfo(icCachePaths)
return icCacheMap.entries.associate { (lib, cache) ->
val provider = createCacheProvider(cache)
val provider = createCacheProvider(cache.path)
val files = provider.filePaths()
lib to ModuleCache(lib, files.associate { f ->
f to FileCache(f, provider.binaryAst(f), provider.dts(f), provider.sourceMap(f))
compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheAccessors.kt
+5 -4
View File
@@ -203,7 +203,7 @@ interface PersistentCacheConsumer {
fun commitSourceMap(path: String, mapData: ByteArray)
fun invalidateForFile(path: String)
fun commitLibraryPath(libraryPath: String)
fun commitLibraryPath(libraryPath: String, flatHash: ULong, transHash: ULong)
companion object {
val EMPTY = object : PersistentCacheConsumer {
@@ -234,7 +234,7 @@ interface PersistentCacheConsumer {
}
override fun commitLibraryPath(libraryPath: String) {
override fun commitLibraryPath(libraryPath: String, flatHash: ULong, transHash: ULong) {
}
@@ -333,7 +333,7 @@ class PersistentCacheConsumerImpl(private val cachePath: String) : PersistentCac
commitByteArrayToCacheFile(path, fileSourceMap, mapData)
}
override fun commitLibraryPath(libraryPath: String) {
override fun commitLibraryPath(libraryPath: String, flatHash: ULong, transHash: ULong) {
val infoFile = File(File(cachePath), "info")
if (infoFile.exists()) {
infoFile.delete()
@@ -342,7 +342,8 @@ class PersistentCacheConsumerImpl(private val cachePath: String) : PersistentCac
PrintWriter(infoFile).use {
it.println(libraryPath)
it.println("0")
it.println(flatHash.toString(16))
it.println(transHash.toString(16))
}
}
}
js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt
+17 -7
View File
@@ -17,6 +17,7 @@ import org.jetbrains.kotlin.codegen.*
import org.jetbrains.kotlin.config.CommonConfigurationKeys
import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.ir.backend.js.generateKLib
import org.jetbrains.kotlin.ir.backend.js.ic.CacheUpdateStatus
import org.jetbrains.kotlin.ir.backend.js.ic.PersistentCacheConsumer
import org.jetbrains.kotlin.ir.backend.js.ic.actualizeCacheForModule
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
@@ -145,7 +146,7 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
val moduleCacheDir = resolveModuleCache(module, buildDir)
buildCachesAndCheck(moduleStep.id, configuration, moduleSourceDir, outputKlibFile, moduleCacheDir, dependencies, icCaches, dirtyFiles)
buildCachesAndCheck(moduleStep, configuration, moduleSourceDir, outputKlibFile, moduleCacheDir, dependencies, icCaches, dirtyFiles)
}
}
}
@@ -156,7 +157,7 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
}
private fun buildCachesAndCheck(
stepId: Int,
moduleStep: ModuleInfo.ModuleStep,
configuration: CompilerConfiguration,
sourceDir: File,
moduleKlibFile: File,
@@ -176,17 +177,20 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
exportedDeclarations: Set<FqName>,
mainArguments: List<String>?,
) {
val actualDirtyFiles = invalidatedDirtyFiles?.map { File(it).canonicalPath } ?: sourceDir.filteredKtFiles().map { it.canonicalPath }
val actualDirtyFiles =
invalidatedDirtyFiles?.map { File(it).canonicalPath } ?: sourceDir.filteredKtFiles().map { it.canonicalPath }
val expectedDirtyFilesCanonical = expectedDirtyFiles.map { it.canonicalPath }
JUnit4Assertions.assertSameElements(expectedDirtyFilesCanonical, actualDirtyFiles) { "For module $moduleKlibFile at step $stepId" }
JUnit4Assertions.assertSameElements(expectedDirtyFilesCanonical, actualDirtyFiles) {
"For module $moduleKlibFile at step ${moduleStep.id}"
}
}
val dependenciesPaths = mutableListOf<String>()
dependencies.mapTo(dependenciesPaths) { it.canonicalPath }
dependenciesPaths.add(moduleKlibFile.canonicalPath)
val upToDate = actualizeCacheForModule(
val updateStatus = actualizeCacheForModule(
moduleKlibFile.canonicalPath,
moduleCacheDir.canonicalPath,
configuration,
@@ -196,9 +200,15 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
::dirtyFilesChecker
)
JUnit4Assertions.assertEquals(expectedDirtyFiles.isEmpty(), upToDate) {
if (StepDirectives.FAST_PATH_UPDATE in moduleStep.directives) {
JUnit4Assertions.assertEquals(CacheUpdateStatus.FAST_PATH, updateStatus) {
"Cache has to be checked by fast path, instead it $updateStatus"
}
}
JUnit4Assertions.assertEquals(expectedDirtyFiles.isEmpty(), updateStatus.upToDate) {
val filePaths = expectedDirtyFiles.joinToString(",", "[", "]")
"Up to date is not expected for module $moduleKlibFile at step $stepId. Expected dirtyFiles are $filePaths"
"Up to date is not expected for module $moduleKlibFile at step ${moduleStep.id}. Expected dirtyFiles are $filePaths"
}
}
js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt
+1 -2
View File
@@ -9,7 +9,6 @@ import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.ir.backend.js.WholeWorldStageController
import org.jetbrains.kotlin.ir.backend.js.ic.*
import org.jetbrains.kotlin.ir.backend.js.moduleName
import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl
import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC
import org.jetbrains.kotlin.ir.util.IdSignature
import org.jetbrains.kotlin.js.test.handlers.JsBoxRunner
@@ -115,7 +114,7 @@ class TestModuleCache(val moduleName: String, val files: MutableMap<String, File
files.remove(path)
}
override fun commitLibraryPath(libraryPath: String) {
override fun commitLibraryPath(libraryPath: String, flatHash: ULong, transHash: ULong) {
}
}