diff --git a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt index 80e9a8a99fc..3213961a2d5 100644 --- a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt +++ b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2NativeCompilerArguments.kt @@ -167,6 +167,14 @@ class K2NativeCompilerArguments : CommonCompilerArguments() { ) var autoCacheDir: String? = null + @Argument( + value = INCREMENTAL_CACHE_DIR, + valueDescription = "", + description = "Path to the directory where to put incremental build caches", + delimiter = "" + ) + var incrementalCacheDir: String? = null + @Argument(value="-Xcheck-dependencies", deprecatedName = "--check_dependencies", description = "Check dependencies and download the missing ones") var checkDependencies: Boolean = false @@ -485,6 +493,7 @@ class K2NativeCompilerArguments : CommonCompilerArguments() { const val INCLUDE_ARG = "-Xinclude" const val CACHED_LIBRARY = "-Xcached-library" const val ADD_CACHE = "-Xadd-cache" + const val INCREMENTAL_CACHE_DIR = "-Xic-cache-dir" const val SHORT_MODULE_NAME_ARG = "-Xshort-module-name" } } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheBuilder.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheBuilder.kt index ed16d31a0da..88e05828f35 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheBuilder.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheBuilder.kt @@ -5,6 +5,8 @@ package org.jetbrains.kotlin.backend.konan +import org.jetbrains.kotlin.backend.common.serialization.FingerprintHash +import org.jetbrains.kotlin.backend.common.serialization.SerializedIrFileFingerprint import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.konan.file.File @@ -13,6 +15,7 @@ import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.library.metadata.resolver.TopologicalLibraryOrder import org.jetbrains.kotlin.library.uniqueName import org.jetbrains.kotlin.backend.konan.descriptors.isInteropLibrary +import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.library.unresolvedDependencies internal fun KotlinLibrary.getAllTransitiveDependencies(allLibraries: Map): List { @@ -32,85 +35,256 @@ internal fun KotlinLibrary.getAllTransitiveDependencies(allLibraries: Map, CompilerConfiguration.() -> Unit) -> Unit ) { private val configuration = konanConfig.configuration private val autoCacheableFrom = configuration.get(KonanConfigKeys.AUTO_CACHEABLE_FROM)!!.map { File(it) } + private val icEnabled = configuration.get(CommonConfigurationKeys.INCREMENTAL_COMPILATION)!! + private val includedLibraries = configuration.get(KonanConfigKeys.INCLUDED_LIBRARIES).orEmpty().toSet() + private val generateTestRunner = configuration.getNotNull(KonanConfigKeys.GENERATE_TEST_RUNNER) - fun needToBuild() = konanConfig.isFinalBinary && konanConfig.ignoreCacheReason == null && autoCacheableFrom.isNotEmpty() + fun needToBuild() = konanConfig.isFinalBinary && konanConfig.ignoreCacheReason == null && (autoCacheableFrom.isNotEmpty() || icEnabled) + + private val allLibraries by lazy { konanConfig.resolvedLibraries.getFullList(TopologicalLibraryOrder) } + private val uniqueNameToLibrary by lazy { allLibraries.associateBy { it.uniqueName } } + + private val caches = mutableMapOf() + private val cacheRootDirectories = mutableMapOf() + + // If libA depends on libB, then dependableLibraries[libB] contains libA. + private val dependableLibraries = mutableMapOf>() + + private fun findAllDependable(libraries: List): Set { + val visited = mutableSetOf() + + fun dfs(library: KotlinLibrary) { + visited.add(library) + dependableLibraries[library]?.forEach { + if (it !in visited) dfs(it) + } + } + + libraries.forEach { if (it !in visited) dfs(it) } + return visited + } + + private data class LibraryFile(val library: KotlinLibrary, val file: String) { + override fun toString() = "${library.uniqueName}|$file" + } + + private val KotlinLibrary.isExternal + get() = autoCacheableFrom.any { libraryFile.absolutePath.startsWith(it.absolutePath) } fun build() { - val allLibraries = konanConfig.resolvedLibraries.getFullList(TopologicalLibraryOrder) - val uniqueNameToLibrary = allLibraries.associateBy { it.uniqueName } - val caches = mutableMapOf() + val externalLibrariesToCache = mutableListOf() + val icedLibraries = mutableListOf() - allLibraries.forEach librariesLoop@{ library -> - konanConfig.cachedLibraries.getLibraryCache(library)?.let { - caches[library] = it.rootDirectory - return@librariesLoop + allLibraries.forEach { library -> + val isDefaultOrExternal = library.isDefault || library.isExternal + val cache = konanConfig.cachedLibraries.getLibraryCache(library, !isDefaultOrExternal) + cache?.let { + caches[library] = it + cacheRootDirectories[library] = it.rootDirectory } - val libraryPath = library.libraryFile.absolutePath - val isLibraryAutoCacheable = library.isDefault || autoCacheableFrom.any { libraryPath.startsWith(it.absolutePath) } - if (!isLibraryAutoCacheable) - return@librariesLoop + if (isDefaultOrExternal) { + if (cache == null) externalLibrariesToCache += library + } else { + icedLibraries += library + } + library.unresolvedDependencies.forEach { + val dependency = uniqueNameToLibrary[it.path]!! + dependableLibraries.getOrPut(dependency) { mutableListOf() }.add(library) + } + } - val dependencies = library.getAllTransitiveDependencies(uniqueNameToLibrary) - val dependencyCaches = dependencies.map { - caches[it] ?: run { - configuration.report(CompilerMessageSeverity.LOGGING, - "SKIPPING ${library.libraryName} as some of the dependencies aren't cached") - return@librariesLoop - } + externalLibrariesToCache.forEach { buildLibraryCache(it, true, emptyList()) } + + if (!icEnabled) return + + // Every library dependable on one of the changed external libraries needs its cache to be fully rebuilt. + val needFullRebuild = findAllDependable(externalLibrariesToCache) + + val libraryFilesWithFqNames = mutableMapOf>() + + val changedFiles = mutableListOf() + val removedFiles = mutableListOf() + val addedFiles = mutableListOf() + val reversedPerFileDependencies = mutableMapOf>() + val reversedWholeLibraryDependencies = mutableMapOf>() + for (library in icedLibraries) { + if (library in needFullRebuild) continue + val cache = caches[library] ?: continue + if (cache !is CachedLibraries.Cache.PerFile) { + require(library.isInteropLibrary()) + continue } - // TODO: Uncomment after making per-file caches default. - val makePerFileCache = false//!library.isInteropLibrary() - configuration.report(CompilerMessageSeverity.LOGGING, "CACHING ${library.libraryName}") - val libraryCacheDirectory = if (library.isDefault) - konanConfig.systemCacheDirectory - else - CachedLibraries.computeVersionedCacheDirectory(konanConfig.autoCacheDirectory, library, uniqueNameToLibrary) - val libraryCache = libraryCacheDirectory.child( - if (makePerFileCache) - CachedLibraries.getPerFileCachedLibraryName(library) - else - CachedLibraries.getCachedLibraryName(library) - ) - try { - // TODO: Run monolithic cache builds in parallel. - libraryCacheDirectory.mkdirs() - spawnCompilation(konanConfig.additionalCacheFlags /* TODO: Some way to put them directly to CompilerConfiguration? */) { - val libraries = dependencies.filter { !it.isDefault }.map { it.libraryFile.absolutePath } - val cachedLibraries = dependencies.zip(dependencyCaches).associate { it.first.libraryFile.absolutePath to it.second } - configuration.report(CompilerMessageSeverity.LOGGING, " dependencies:\n " + - libraries.joinToString("\n ")) - configuration.report(CompilerMessageSeverity.LOGGING, " caches used:\n " + - cachedLibraries.entries.joinToString("\n ") { "${it.key}: ${it.value}" }) - configuration.report(CompilerMessageSeverity.LOGGING, " cache dir: " + - libraryCacheDirectory.absolutePath) + val libraryCacheRootDir = File(cache.path) + val cachedFiles = libraryCacheRootDir.listFiles.map { it.name } - setupCommonOptionsForCaches(konanConfig) - put(KonanConfigKeys.PRODUCE, CompilerOutputKind.STATIC_CACHE) - put(KonanConfigKeys.LIBRARY_TO_ADD_TO_CACHE, libraryPath) - put(KonanConfigKeys.NODEFAULTLIBS, true) - put(KonanConfigKeys.NOENDORSEDLIBS, true) - put(KonanConfigKeys.NOSTDLIB, true) - put(KonanConfigKeys.LIBRARY_FILES, libraries) - put(KonanConfigKeys.CACHED_LIBRARIES, cachedLibraries) - put(KonanConfigKeys.CACHE_DIRECTORIES, listOf(libraryCacheDirectory.absolutePath)) - put(KonanConfigKeys.MAKE_PER_FILE_CACHE, makePerFileCache) + val actualFilesWithFqNames = library.getFilesWithFqNames() + libraryFilesWithFqNames[library] = actualFilesWithFqNames + val actualFiles = actualFilesWithFqNames.withIndex() + .associate { CacheSupport.cacheFileId(it.value.fqName, it.value.filePath) to it.index } + .toMutableMap() + + for (cachedFile in cachedFiles) { + val libraryFile = LibraryFile(library, cachedFile) + val fileIndex = actualFiles[cachedFile] + if (fileIndex == null) { + removedFiles.add(libraryFile) + } else { + actualFiles.remove(cachedFile) + val actualContentHash = SerializedIrFileFingerprint(library, fileIndex).fileFingerprint + val previousContentHash = FingerprintHash.fromByteArray(cache.getFileHash(cachedFile)) + if (previousContentHash != actualContentHash) + changedFiles.add(libraryFile) + + val dependencies = cache.getFileDependencies(cachedFile) + for (dependency in dependencies) { + val dependentLibrary = uniqueNameToLibrary[dependency.libName] + ?: error("Unknown dependent library ${dependency.libName}") + when (val kind = dependency.kind) { + is DependenciesTracker.DependencyKind.WholeModule -> + reversedWholeLibraryDependencies.getOrPut(dependentLibrary) { mutableListOf() }.add(libraryFile) + is DependenciesTracker.DependencyKind.CertainFiles -> + kind.files.forEach { + reversedPerFileDependencies.getOrPut(LibraryFile(dependentLibrary, it)) { mutableListOf() }.add(libraryFile) + } + } + } } - caches[library] = libraryCache.absolutePath - } catch (t: Throwable) { - configuration.report(CompilerMessageSeverity.LOGGING, "${t.message}\n${t.stackTraceToString()}") - configuration.report(CompilerMessageSeverity.WARNING, - "Failed to build cache: ${t.message}\n${t.stackTraceToString()}\n" + - "Falling back to not use cache for ${library.libraryName}") + } + for (newFile in actualFiles.keys) + addedFiles.add(LibraryFile(library, newFile)) + } - libraryCache.deleteRecursively() + configuration.report(CompilerMessageSeverity.LOGGING, "IC analysis results") + configuration.report(CompilerMessageSeverity.LOGGING, " CACHED:") + icedLibraries.filter { caches[it] != null }.forEach { configuration.report(CompilerMessageSeverity.LOGGING, " ${it.libraryName}") } + configuration.report(CompilerMessageSeverity.LOGGING, " CLEAN BUILD:") + icedLibraries.filter { caches[it] == null }.forEach { configuration.report(CompilerMessageSeverity.LOGGING, " ${it.libraryName}") } + configuration.report(CompilerMessageSeverity.LOGGING, " FULL REBUILD:") + icedLibraries.filter { it in needFullRebuild }.forEach { configuration.report(CompilerMessageSeverity.LOGGING, " ${it.libraryName}") } + configuration.report(CompilerMessageSeverity.LOGGING, " ADDED FILES:") + addedFiles.forEach { configuration.report(CompilerMessageSeverity.LOGGING, " $it") } + configuration.report(CompilerMessageSeverity.LOGGING, " REMOVED FILES:") + removedFiles.forEach { configuration.report(CompilerMessageSeverity.LOGGING, " $it") } + configuration.report(CompilerMessageSeverity.LOGGING, " CHANGED FILES:") + changedFiles.forEach { configuration.report(CompilerMessageSeverity.LOGGING, " $it") } + + val dirtyFiles = mutableSetOf() + + fun dfs(libraryFile: LibraryFile) { + dirtyFiles += libraryFile + reversedPerFileDependencies[libraryFile]?.forEach { + if (it !in dirtyFiles) dfs(it) + } + } + + removedFiles.forEach { + if (it !in dirtyFiles) dfs(it) + } + changedFiles.forEach { + if (it !in dirtyFiles) dfs(it) + } + dirtyFiles.addAll(addedFiles) + + removedFiles.forEach { + dirtyFiles.remove(it) + File(caches[it.library]!!.rootDirectory).child(it.file).deleteRecursively() + } + + val groupedDirtyFiles = dirtyFiles.groupBy { it.library } + configuration.report(CompilerMessageSeverity.LOGGING, " DIRTY FILES:") + groupedDirtyFiles.values.flatten().forEach { + configuration.report(CompilerMessageSeverity.LOGGING, " $it") + } + + for (library in icedLibraries) { + val filesToCache = groupedDirtyFiles[library]?.let { libraryFiles -> + val filesWithFqNames = libraryFilesWithFqNames[library]!!.associateBy { + CacheSupport.cacheFileId(it.fqName, it.filePath) + } + libraryFiles.map { filesWithFqNames[it.file]!!.filePath } + }.orEmpty() + + when { + library in needFullRebuild -> buildLibraryCache(library, false, emptyList()) + caches[library] == null || filesToCache.isNotEmpty() -> buildLibraryCache(library, false, filesToCache) } } } + + private fun buildLibraryCache(library: KotlinLibrary, isExternal: Boolean, filesToCache: List) { + val dependencies = library.getAllTransitiveDependencies(uniqueNameToLibrary) + val dependencyCaches = dependencies.map { + cacheRootDirectories[it] ?: run { + configuration.report(CompilerMessageSeverity.LOGGING, + "SKIPPING ${library.libraryName} as some of the dependencies aren't cached") + return + } + } + + configuration.report(CompilerMessageSeverity.LOGGING, "CACHING ${library.libraryName}") + filesToCache.forEach { configuration.report(CompilerMessageSeverity.LOGGING, " $it") } + + // Produce monolithic caches for external libraries for now. + val makePerFileCache = !isExternal && !library.isInteropLibrary() + + val libraryCacheDirectory = when { + library.isDefault -> konanConfig.systemCacheDirectory + isExternal -> CachedLibraries.computeVersionedCacheDirectory(konanConfig.autoCacheDirectory, library, uniqueNameToLibrary) + else -> konanConfig.incrementalCacheDirectory!! + } + val libraryCache = libraryCacheDirectory.child( + if (makePerFileCache) + CachedLibraries.getPerFileCachedLibraryName(library) + else + CachedLibraries.getCachedLibraryName(library) + ) + try { + // TODO: Run monolithic cache builds in parallel. + libraryCacheDirectory.mkdirs() + spawnCompilation(konanConfig.additionalCacheFlags /* TODO: Some way to put them directly to CompilerConfiguration? */) { + val libraryPath = library.libraryFile.absolutePath + val libraries = dependencies.filter { !it.isDefault }.map { it.libraryFile.absolutePath } + val cachedLibraries = dependencies.zip(dependencyCaches).associate { it.first.libraryFile.absolutePath to it.second } + configuration.report(CompilerMessageSeverity.LOGGING, " dependencies:\n " + + libraries.joinToString("\n ")) + configuration.report(CompilerMessageSeverity.LOGGING, " caches used:\n " + + cachedLibraries.entries.joinToString("\n ") { "${it.key}: ${it.value}" }) + configuration.report(CompilerMessageSeverity.LOGGING, " cache dir: " + + libraryCacheDirectory.absolutePath) + + setupCommonOptionsForCaches(konanConfig) + put(KonanConfigKeys.PRODUCE, CompilerOutputKind.STATIC_CACHE) + put(KonanConfigKeys.LIBRARY_TO_ADD_TO_CACHE, libraryPath) + put(KonanConfigKeys.NODEFAULTLIBS, true) + put(KonanConfigKeys.NOENDORSEDLIBS, true) + put(KonanConfigKeys.NOSTDLIB, true) + put(KonanConfigKeys.LIBRARY_FILES, libraries) + if (generateTestRunner != TestRunnerKind.NONE && libraryPath in includedLibraries) { + put(KonanConfigKeys.GENERATE_TEST_RUNNER, generateTestRunner) + put(KonanConfigKeys.INCLUDED_LIBRARIES, listOf(libraryPath)) + } + put(KonanConfigKeys.CACHED_LIBRARIES, cachedLibraries) + put(KonanConfigKeys.CACHE_DIRECTORIES, listOf(libraryCacheDirectory.absolutePath)) + put(KonanConfigKeys.MAKE_PER_FILE_CACHE, makePerFileCache) + if (filesToCache.isNotEmpty()) + put(KonanConfigKeys.FILES_TO_CACHE, filesToCache) + } + cacheRootDirectories[library] = libraryCache.absolutePath + } catch (t: Throwable) { + configuration.report(CompilerMessageSeverity.LOGGING, "${t.message}\n${t.stackTraceToString()}") + configuration.report(CompilerMessageSeverity.WARNING, + "Failed to build cache: ${t.message}\n${t.stackTraceToString()}\n" + + "Falling back to not use cache for ${library.libraryName}") + + libraryCache.deleteRecursively() + } + } } \ No newline at end of file diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheStorage.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheStorage.kt index 750674a9ca1..7c110b44a97 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheStorage.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheStorage.kt @@ -8,6 +8,7 @@ package org.jetbrains.kotlin.backend.konan import org.jetbrains.kotlin.backend.konan.serialization.ClassFieldsSerializer import org.jetbrains.kotlin.backend.konan.serialization.EagerInitializedPropertySerializer import org.jetbrains.kotlin.backend.konan.serialization.InlineFunctionBodyReferenceSerializer +import org.jetbrains.kotlin.konan.file.File internal class CacheStorage(private val generationState: NativeGenerationState) { private val outputFiles = generationState.outputFiles @@ -16,8 +17,10 @@ internal class CacheStorage(private val generationState: NativeGenerationState) fun renameOutput(outputFiles: OutputFiles) { // For caches the output file is a directory. It might be created by someone else, // we have to delete it in order for the next renaming operation to succeed. - // TODO: what if the directory is not empty? - java.io.File(outputFiles.mainFileName).delete() + val tempDirectoryForRemoval = File(outputFiles.mainFileName + "-to-remove") + if (outputFiles.mainFile.exists && !outputFiles.mainFile.renameTo(tempDirectoryForRemoval)) + return + tempDirectoryForRemoval.deleteRecursively() if (!outputFiles.tempCacheDirectory!!.renameTo(outputFiles.mainFile)) outputFiles.tempCacheDirectory.deleteRecursively() } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt index 7f058038db5..2fa0886515a 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheSupport.kt @@ -80,10 +80,11 @@ class PartialCacheInfo(val klib: KotlinLibrary, val strategy: CacheDeserializati class CacheSupport( private val configuration: CompilerConfiguration, - resolvedLibraries: KotlinLibraryResolveResult, + private val resolvedLibraries: KotlinLibraryResolveResult, ignoreCacheReason: String?, systemCacheDirectory: File, autoCacheDirectory: File, + incrementalCacheDirectory: File?, target: KonanTarget, val produce: CompilerOutputKind ) { @@ -103,7 +104,9 @@ class CacheSupport( add(File(it).takeIf { it.isDirectory } ?: configuration.reportCompilationError("cache directory $it is not found or is not a directory")) } - systemCacheDirectory.takeIf { autoCacheableFrom.isNotEmpty() }?.let { add(it) } + systemCacheDirectory.takeIf { autoCacheableFrom.isNotEmpty() || incrementalCacheDirectory != null }?.let { add(it) } + autoCacheDirectory.takeIf { autoCacheableFrom.isNotEmpty() }?.let { add(it) } + incrementalCacheDirectory?.let { add(it) } } internal fun tryGetImplicitOutput(cacheDeserializationStrategy: CacheDeserializationStrategy?): String? { @@ -178,10 +181,11 @@ class CacheSupport( configuration.get(KonanConfigKeys.PRE_LINK_CACHES, false) companion object { - fun cacheFileId(fqName: String, filePath: String) = "${if (fqName == "") "ROOT" else fqName}.${filePath.hashCode().toString(Character.MAX_RADIX)}" + fun cacheFileId(fqName: String, filePath: String) = + "${if (fqName == "") "ROOT" else fqName}.${filePath.hashCode().toString(Character.MAX_RADIX)}" } - init { + fun checkConsistency() { // Ensure dependencies of every cached library are cached too: resolvedLibraries.getFullList { libraries -> libraries.map { library -> diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt index 67df442efe8..0e3d0899357 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt @@ -84,11 +84,13 @@ class CachedLibraries( } } - class PerFile(target: KonanTarget, kind: Kind, path: String, private val fileDirs: List) + class PerFile(target: KonanTarget, kind: Kind, path: String, fileDirs: List, val complete: Boolean) : Cache(target, kind, path, File(path).absolutePath) { + private val existingFileDirs = if (complete) fileDirs else fileDirs.filter { it.exists } + private val perFileBitcodeDependencies by lazy { - fileDirs.associate { + existingFileDirs.associate { val data = it.child(PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME).child(BITCODE_DEPENDENCIES_FILE_NAME).readStrings() it.name to DependenciesSerializer.deserialize(it.absolutePath, data) } @@ -108,26 +110,26 @@ class CachedLibraries( override fun computeBitcodeDependencies() = perFileBitcodeDependencies.values.flatten() - override fun computeBinariesPaths() = fileDirs.map { + override fun computeBinariesPaths() = existingFileDirs.map { it.child(PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME).child(getArtifactName(target, it.name, kind.toCompilerOutputKind())).absolutePath } override fun computeSerializedInlineFunctionBodies() = mutableListOf().also { - fileDirs.forEach { fileDir -> + existingFileDirs.forEach { fileDir -> val data = fileDir.child(PER_FILE_CACHE_IR_LEVEL_DIR_NAME).child(INLINE_FUNCTION_BODIES_FILE_NAME).readBytes() InlineFunctionBodyReferenceSerializer.deserializeTo(data, it) } } override fun computeSerializedClassFields() = mutableListOf().also { - fileDirs.forEach { fileDir -> + existingFileDirs.forEach { fileDir -> val data = fileDir.child(PER_FILE_CACHE_IR_LEVEL_DIR_NAME).child(CLASS_FIELDS_FILE_NAME).readBytes() ClassFieldsSerializer.deserializeTo(data, it) } } override fun computeSerializedEagerInitializedFiles() = mutableListOf().also { - fileDirs.forEach { fileDir -> + existingFileDirs.forEach { fileDir -> val data = fileDir.child(PER_FILE_CACHE_IR_LEVEL_DIR_NAME).child(EAGER_INITIALIZED_PROPERTIES_FILE_NAME).readBytes() EagerInitializedPropertySerializer.deserializeTo(data, it) } @@ -162,8 +164,8 @@ class CachedLibraries( val libraryFileDirs = librariesFileDirs.getOrPut(library) { library.getFilesWithFqNames().map { cacheDir.child(CacheSupport.cacheFileId(it.fqName, it.filePath)) } } - Cache.PerFile(target, Kind.STATIC, cacheDir.absolutePath, libraryFileDirs) - .takeIf { cacheDirContents.containsAll(libraryFileDirs.map { it.absolutePath }) } + Cache.PerFile(target, Kind.STATIC, cacheDir.absolutePath, libraryFileDirs, + complete = cacheDirContents.containsAll(libraryFileDirs.map { it.absolutePath })) } } } @@ -196,8 +198,8 @@ class CachedLibraries( fun isLibraryCached(library: KotlinLibrary): Boolean = getLibraryCache(library) != null - fun getLibraryCache(library: KotlinLibrary): Cache? = - allCaches[library] + fun getLibraryCache(library: KotlinLibrary, allowIncomplete: Boolean = false): Cache? = + allCaches[library]?.takeIf { allowIncomplete || (it as? Cache.PerFile)?.complete != false } val hasStaticCaches = allCaches.values.any { when (it.kind) { diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index 3a572aafb36..edd29e1d919 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -406,6 +406,8 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration internal val additionalCacheFlags by lazy { platformManager.loader(target).additionalCacheFlags } + internal val threadsCount = configuration.get(CommonConfigurationKeys.PARALLEL_BACKEND_THREADS) ?: 1 + private fun StringBuilder.appendCommonCacheFlavor() { append(target.toString()) if (debug) append("-g") @@ -440,9 +442,19 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration } private val systemCacheRootDirectory = File(distribution.konanHome).child("klib").child("cache") - internal val systemCacheDirectory = systemCacheRootDirectory.child(systemCacheFlavorString) - private val autoCacheRootDirectory = configuration.get(KonanConfigKeys.AUTO_CACHE_DIR)?.let { File(it) } ?: systemCacheRootDirectory - internal val autoCacheDirectory = autoCacheRootDirectory.child(userCacheFlavorString) + internal val systemCacheDirectory = systemCacheRootDirectory.child(systemCacheFlavorString).also { it.mkdirs() } + private val autoCacheRootDirectory = configuration.get(KonanConfigKeys.AUTO_CACHE_DIR)?.let { + File(it).apply { + if (!isDirectory) configuration.reportCompilationError("auto cache directory $this is not found or is not a directory") + } + } ?: systemCacheRootDirectory + internal val autoCacheDirectory = autoCacheRootDirectory.child(userCacheFlavorString).also { it.mkdirs() } + private val incrementalCacheRootDirectory = configuration.get(KonanConfigKeys.INCREMENTAL_CACHE_DIR)?.let { + File(it).apply { + if (!isDirectory) configuration.reportCompilationError("incremental cache directory $this is not found or is not a directory") + } + } + internal val incrementalCacheDirectory = incrementalCacheRootDirectory?.child(userCacheFlavorString)?.also { it.mkdirs() } internal val ignoreCacheReason = when { optimizationsEnabled -> "for optimized compilation" @@ -457,6 +469,7 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration ignoreCacheReason = ignoreCacheReason, systemCacheDirectory = systemCacheDirectory, autoCacheDirectory = autoCacheDirectory, + incrementalCacheDirectory = incrementalCacheDirectory, target = target, produce = produce ) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt index eccbfed3b6c..621361ab735 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt @@ -39,6 +39,8 @@ class KonanConfigKeys { = CompilerConfigurationKey.create>("paths to the root directories from which dependencies are to be cached automatically") val AUTO_CACHE_DIR: CompilerConfigurationKey = CompilerConfigurationKey.create("path to the directory where to put caches for auto-cacheable dependencies") + val INCREMENTAL_CACHE_DIR: CompilerConfigurationKey + = CompilerConfigurationKey.create("path to the directory where to put incremental build caches") val CACHED_LIBRARIES: CompilerConfigurationKey> = CompilerConfigurationKey.create>("mapping from library paths to cache paths") val FILES_TO_CACHE: CompilerConfigurationKey> diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt index 8c807e474a0..72d134205b6 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt @@ -40,14 +40,17 @@ class KonanDriver( ) { fun run() { val fileNames = configuration.get(KonanConfigKeys.LIBRARY_TO_ADD_TO_CACHE)?.let { libPath -> - if (configuration.get(KonanConfigKeys.MAKE_PER_FILE_CACHE) != true) - configuration.get(KonanConfigKeys.FILES_TO_CACHE) - else { - val lib = createKonanLibrary(File(libPath), "default", null, true) - (0 until lib.fileCount()).map { fileIndex -> - val proto = IrFile.parseFrom(lib.file(fileIndex).codedInputStream, ExtensionRegistryLite.newInstance()) - proto.fileEntry.name + val filesToCache = configuration.get(KonanConfigKeys.FILES_TO_CACHE) + when { + !filesToCache.isNullOrEmpty() -> filesToCache + configuration.get(KonanConfigKeys.MAKE_PER_FILE_CACHE) == true -> { + val lib = createKonanLibrary(File(libPath), "default", null, true) + (0 until lib.fileCount()).map { fileIndex -> + val proto = IrFile.parseFrom(lib.file(fileIndex).codedInputStream, ExtensionRegistryLite.newInstance()) + proto.fileEntry.name + } } + else -> null } } if (fileNames != null) { @@ -75,6 +78,8 @@ class KonanDriver( konanConfig = KonanConfig(project, configuration) // TODO: Just set freshly built caches. } + konanConfig.cacheSupport.checkConsistency() + DynamicCompilerDriver().run(konanConfig, environment) } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/SetupConfiguration.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/SetupConfiguration.kt index ebddfb102d4..3694bf5718b 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/SetupConfiguration.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/SetupConfiguration.kt @@ -177,6 +177,11 @@ fun CompilerConfiguration.setupFromArguments(arguments: K2NativeCompilerArgument put(CACHE_DIRECTORIES, arguments.cacheDirectories.toNonNullList()) put(AUTO_CACHEABLE_FROM, arguments.autoCacheableFrom.toNonNullList()) arguments.autoCacheDir?.let { put(AUTO_CACHE_DIR, it) } + val incrementalCacheDir = arguments.incrementalCacheDir + if ((incrementalCacheDir != null) xor (arguments.incrementalCompilation == true)) + report(ERROR, "For incremental compilation both flags should be supplied: " + + "-Xenable-incremental-compilation and ${K2NativeCompilerArguments.INCREMENTAL_CACHE_DIR}") + incrementalCacheDir?.let { put(INCREMENTAL_CACHE_DIR, it) } arguments.filesToCache?.let { put(FILES_TO_CACHE, it.toList()) } put(MAKE_PER_FILE_CACHE, arguments.makePerFileCache) val nThreadsRaw = parseBackendThreads(arguments.backendThreads) @@ -302,6 +307,7 @@ internal fun CompilerConfiguration.setupCommonOptionsForCaches(konanConfig: Kona put(BinaryOptions.freezing, konanConfig.freezing) put(BinaryOptions.runtimeAssertionsMode, konanConfig.runtimeAssertsMode) put(LAZY_IR_FOR_CACHES, konanConfig.lazyIrForCaches) + put(CommonConfigurationKeys.PARALLEL_BACKEND_THREADS, konanConfig.threadsCount) } private fun Array?.toNonNullList() = this?.asList().orEmpty() diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/TopLevelPhases.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/TopLevelPhases.kt index e6965000cd0..d77f4cd7013 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/TopLevelPhases.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/TopLevelPhases.kt @@ -12,7 +12,6 @@ import org.jetbrains.kotlin.backend.konan.driver.utilities.CExportFiles import org.jetbrains.kotlin.backend.konan.driver.utilities.createTempFiles import org.jetbrains.kotlin.backend.konan.ir.konanLibrary import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment -import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.ir.declarations.IrClass import org.jetbrains.kotlin.ir.declarations.IrDeclaration import org.jetbrains.kotlin.ir.declarations.IrModuleFragment @@ -119,8 +118,8 @@ internal fun PhaseEngine.runBackend(backendContext: Contex } val fragments = backendEngine.splitIntoFragments(irModule) - val nThreads = context.config.configuration.get(CommonConfigurationKeys.PARALLEL_BACKEND_THREADS) ?: 1 - if (nThreads == 1) { + val threadsCount = context.config.threadsCount + if (threadsCount == 1) { fragments.forEach { fragment -> runAfterLowerings(fragment, createGenerationStateAndRunLowerings(fragment)) } @@ -133,9 +132,9 @@ internal fun PhaseEngine.runBackend(backendContext: Contex // We'd love to run entire pipeline in parallel, but it's difficult (mainly because of the lowerings, // which need cross-file access all the time and it's not easy to overcome this). So, for now, // we split the pipeline into two parts - everything before lowerings (including them) - // which is run sequentially, and everything else which in run in parallel. + // which is run sequentially, and everything else which is run in parallel. val generationStates = fragmentsList.map { fragment -> createGenerationStateAndRunLowerings(fragment) } - val executor = Executors.newFixedThreadPool(nThreads) + val executor = Executors.newFixedThreadPool(threadsCount) val thrownFromThread = AtomicReference(null) val tasks = fragmentsList.zip(generationStates).map { (fragment, generationState) -> Callable {