azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Reduce file reads in cache libraries

Browse Source 
#KT-64249 Fixed
This commit is contained in:
Mark Mann
2023-12-11 11:08:00 -08:00
committed by Space Team
parent d2b810c373
commit bef0946ab7
1 changed files with 21 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt
+21 -22
View File
@@ -145,34 +145,27 @@ class CachedLibraries(
} }
} }
private val cacheDirsContents = mutableMapOf<String, Set<String>>() private fun File.trySelectCacheFor(library: KotlinLibrary): Cache? {
private val librariesFileDirs = mutableMapOf<KotlinLibrary, List<File>>()
private fun selectCache(library: KotlinLibrary, cacheDir: File): Cache? {
// See Linker.renameOutput why is it ok to have an empty cache directory. // See Linker.renameOutput why is it ok to have an empty cache directory.
val cacheDirContents = cacheDirsContents.getOrPut(cacheDir.absolutePath) { val cacheDirContents = listFilesOrEmpty.map { it.absolutePath }.toSet()
cacheDir.listFilesOrEmpty.map { it.absolutePath }.toSet()
}
if (cacheDirContents.isEmpty()) return null if (cacheDirContents.isEmpty()) return null
val cacheBinaryPartDir = cacheDir.child(PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME) val cacheBinaryPartDir = child(PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME)
val cacheBinaryPartDirContents = cacheDirsContents.getOrPut(cacheBinaryPartDir.absolutePath) { val cacheBinaryPartDirContents = cacheBinaryPartDir.listFilesOrEmpty.map { it.absolutePath }.toSet()
cacheBinaryPartDir.listFilesOrEmpty.map { it.absolutePath }.toSet()
}
val baseName = getCachedLibraryName(library) val baseName = getCachedLibraryName(library)
val dynamicFile = cacheBinaryPartDir.child(getArtifactName(target, baseName, CompilerOutputKind.DYNAMIC_CACHE)) val dynamicFile = cacheBinaryPartDir.child(getArtifactName(target, baseName, CompilerOutputKind.DYNAMIC_CACHE))
val staticFile = cacheBinaryPartDir.child(getArtifactName(target, baseName, CompilerOutputKind.STATIC_CACHE)) val staticFile = cacheBinaryPartDir.child(getArtifactName(target, baseName, CompilerOutputKind.STATIC_CACHE))
if (dynamicFile.absolutePath in cacheBinaryPartDirContents && staticFile.absolutePath in cacheBinaryPartDirContents) if (dynamicFile.absolutePath in cacheBinaryPartDirContents && staticFile.absolutePath in cacheBinaryPartDirContents)
error("Both dynamic and static caches files cannot be in the same directory." + error("Both dynamic and static caches files cannot be in the same directory." +
" Library: ${library.libraryName}, path to cache: ${cacheDir.absolutePath}") " Library: ${library.libraryName}, path to cache: $absolutePath")
return when { return when {
dynamicFile.absolutePath in cacheBinaryPartDirContents -> Cache.Monolithic(target, Kind.DYNAMIC, dynamicFile.absolutePath) dynamicFile.absolutePath in cacheBinaryPartDirContents -> Cache.Monolithic(target, Kind.DYNAMIC, dynamicFile.absolutePath)
staticFile.absolutePath in cacheBinaryPartDirContents -> Cache.Monolithic(target, Kind.STATIC, staticFile.absolutePath) staticFile.absolutePath in cacheBinaryPartDirContents -> Cache.Monolithic(target, Kind.STATIC, staticFile.absolutePath)
else -> { else -> {
val libraryFileDirs = librariesFileDirs.getOrPut(library) { val libraryFileDirs = library.getFilesWithFqNames().map {
library.getFilesWithFqNames().map { cacheDir.child(CacheSupport.cacheFileId(it.fqName, it.filePath)) } child(CacheSupport.cacheFileId(it.fqName, it.filePath))
} }
Cache.PerFile(target, Kind.STATIC, cacheDir.absolutePath, libraryFileDirs, Cache.PerFile(target, Kind.STATIC, absolutePath, libraryFileDirs,
complete = cacheDirContents.containsAll(libraryFileDirs.map { it.absolutePath })) complete = cacheDirContents.containsAll(libraryFileDirs.map { it.absolutePath }))
} }
} }
@@ -181,23 +174,29 @@ class CachedLibraries(
private val uniqueNameToLibrary = allLibraries.associateBy { it.uniqueName } private val uniqueNameToLibrary = allLibraries.associateBy { it.uniqueName }
private val uniqueNameToHash = mutableMapOf<String, FingerprintHash>() private val uniqueNameToHash = mutableMapOf<String, FingerprintHash>()
private val cacheNameToImplicitDirMapping: Map<String, File> =
implicitCacheDirectories.flatMap { dir -> dir.listFilesOrEmpty.map { it.name to it } }
.toMap()
private fun KotlinLibrary.trySelectCacheAt(dirBuilder: (String) -> File?) =
sequenceOf(getPerFileCachedLibraryName(this), getCachedLibraryName(this))
.map(dirBuilder)
.mapNotNull { it?.trySelectCacheFor(this) }
.firstOrNull()
private val allCaches: Map<KotlinLibrary, Cache> = allLibraries.mapNotNull { library -> private val allCaches: Map<KotlinLibrary, Cache> = allLibraries.mapNotNull { library ->
val explicitPath = explicitCaches[library] val explicitPath = explicitCaches[library]
val cache = if (explicitPath != null) { val cache = if (explicitPath != null) {
selectCache(library, File(explicitPath)) File(explicitPath).trySelectCacheFor(library)
?: error("No cache found for library ${library.libraryName} at $explicitPath") ?: error("No cache found for library ${library.libraryName} at $explicitPath")
} else { } else {
val libraryPath = library.libraryFile.absolutePath val libraryPath = library.libraryFile.absolutePath
implicitCacheDirectories.firstNotNullOfOrNull { dir -> library.trySelectCacheAt { cacheNameToImplicitDirMapping[it] }
selectCache(library, dir.child(getPerFileCachedLibraryName(library)))
?: selectCache(library, dir.child(getCachedLibraryName(library)))
}
?: autoCacheDirectory.takeIf { autoCacheableFrom.any { libraryPath.startsWith(it.absolutePath) } } ?: autoCacheDirectory.takeIf { autoCacheableFrom.any { libraryPath.startsWith(it.absolutePath) } }
?.let { ?.let {
val dir = computeVersionedCacheDirectory(it, library, uniqueNameToLibrary, uniqueNameToHash) val dir = computeVersionedCacheDirectory(it, library, uniqueNameToLibrary, uniqueNameToHash)
selectCache(library, dir.child(getPerFileCachedLibraryName(library))) library.trySelectCacheAt { cacheName -> dir.child(cacheName) }
?: selectCache(library, dir.child(getCachedLibraryName(library)))
} }
} }