From 90044108e05a44f6bd8ad8d4e7a48ec15562c3f7 Mon Sep 17 00:00:00 2001 From: Igor Chevdar Date: Thu, 11 Jun 2020 23:15:37 +0500 Subject: [PATCH] [caches] Make output for cache a directory (cherry picked from commit 9850bb322bee2bc98fc453a440e41af0dac0b0f2) --- .../kotlin/backend/konan/CachedLibraries.kt | 37 +++++++------- .../jetbrains/kotlin/backend/konan/Linker.kt | 21 ++++---- .../kotlin/backend/konan/OutputFiles.kt | 48 +++++++++++-------- .../org/jetbrains/kotlin/CacheTesting.kt | 2 +- .../plugin/konan/tasks/KonanCacheTask.kt | 8 +--- .../utilities/GeneratePlatformLibraries.kt | 10 ++-- 6 files changed, 67 insertions(+), 59 deletions(-) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt index 8b33b775a5f..59f5630541c 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt @@ -23,34 +23,39 @@ class CachedLibraries( enum class Kind { DYNAMIC, STATIC } } + private fun selectCache(library: KotlinLibrary, cacheDir: File): Cache? { + // See Linker.renameOutput why is it ok to have an empty cache directory. + if (cacheDir.listFilesOrEmpty.isEmpty()) return null + val baseName = getCachedLibraryName(library) + val dynamicFile = cacheDir.child(getArtifactName(baseName, CompilerOutputKind.DYNAMIC_CACHE)) + val staticFile = cacheDir.child(getArtifactName(baseName, CompilerOutputKind.STATIC_CACHE)) + + if (dynamicFile.exists && staticFile.exists) + error("Both dynamic and static caches files cannot be in the same directory." + + " Library: ${library.libraryName}, path to cache: ${cacheDir.absolutePath}") + return when { + dynamicFile.exists -> Cache(Cache.Kind.DYNAMIC, dynamicFile.absolutePath) + staticFile.exists -> Cache(Cache.Kind.STATIC, staticFile.absolutePath) + else -> error("No cache found for library ${library.libraryName} at ${cacheDir.absolutePath}") + } + } + private val allCaches: Map = allLibraries.mapNotNull { library -> val explicitPath = explicitCaches[library] val cache = if (explicitPath != null) { - val kind = when { - explicitPath.endsWith(target.family.dynamicSuffix) -> Cache.Kind.DYNAMIC - explicitPath.endsWith(target.family.staticSuffix) -> Cache.Kind.STATIC - else -> error("unexpected cache: $explicitPath") - } - Cache(kind, explicitPath) + selectCache(library, File(explicitPath)) + ?: error("No cache found for library ${library.libraryName} at $explicitPath") } else { implicitCacheDirectories.firstNotNullResult { dir -> - val baseName = getCachedLibraryName(library) - val dynamicFile = dir.child(getArtifactName(baseName, CompilerOutputKind.DYNAMIC_CACHE)) - val staticFile = dir.child(getArtifactName(baseName, CompilerOutputKind.STATIC_CACHE)) - - when { - dynamicFile.exists -> Cache(Cache.Kind.DYNAMIC, dynamicFile.absolutePath) - staticFile.exists -> Cache(Cache.Kind.STATIC, staticFile.absolutePath) - else -> null - } + selectCache(library, dir.child(getCachedLibraryName(library))) } } cache?.let { library to it } }.toMap() - fun getArtifactName(baseName: String, kind: CompilerOutputKind) = + private fun getArtifactName(baseName: String, kind: CompilerOutputKind) = "${kind.prefix(target)}$baseName${kind.suffix(target)}" fun isLibraryCached(library: KotlinLibrary): Boolean = diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt index 3766b115e6e..25d787bfc98 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Linker.kt @@ -5,7 +5,6 @@ import org.jetbrains.kotlin.konan.file.File import org.jetbrains.kotlin.konan.target.CompilerOutputKind import org.jetbrains.kotlin.konan.target.Family import org.jetbrains.kotlin.konan.target.LinkerOutputKind -import org.jetbrains.kotlin.backend.konan.files.renameAtomic import org.jetbrains.kotlin.konan.library.KonanLibrary internal fun determineLinkerOutput(context: Context): LinkerOutputKind = @@ -45,21 +44,21 @@ internal class Linker(val context: Context) { val libraryProvidedLinkerFlags = context.llvm.allNativeDependencies.map { it.linkerOpts }.flatten() + if (context.config.produce.isCache) + context.config.outputFiles.tempCacheDirectory!!.mkdirs() runLinker(objectFiles, includedBinaries, libraryProvidedLinkerFlags) + renameOutput() } private fun renameOutput() { if (context.config.produce.isCache) { val outputFiles = context.config.outputFiles - val outputFile = java.io.File(outputFiles.mainFileMangled) - val outputDsymBundle = java.io.File(outputFiles.mainFileMangled + ".dSYM") - if (renameAtomic(outputFile.absolutePath, outputFiles.mainFile, /* replaceExisting = */ false)) - outputDsymBundle.renameTo(java.io.File(outputFiles.mainFile + ".dSYM")) - else { - outputFile.delete() - outputDsymBundle.deleteRecursively() - } + // For caches the output file is a directory. It might be created by someone else, + // We have to delete it in order to the next renaming operation to succeed. + java.io.File(outputFiles.mainFile).delete() + if (!java.io.File(outputFiles.tempCacheDirectory!!.absolutePath).renameTo(java.io.File(outputFiles.mainFile))) + outputFiles.tempCacheDirectory.deleteRecursively() } } @@ -96,7 +95,7 @@ internal class Linker(val context: Context) { } else { emptyList() } - executable = context.config.outputFiles.mainFileMangled + executable = context.config.outputFiles.nativeBinaryFile } else { val framework = File(context.config.outputFile) val dylibName = framework.name.removeSuffix(".framework") @@ -127,7 +126,7 @@ internal class Linker(val context: Context) { caches.dynamic + libraryProvidedLinkerFlags + additionalLinkerArgs, optimize = optimize, debug = debug, kind = linkerOutput, - outputDsymBundle = context.config.outputFiles.mainFileMangled + ".dSYM", + outputDsymBundle = context.config.outputFiles.symbolicInfoFile, needsProfileLibrary = needsProfileLibrary).forEach { it.logWith(context::log) it.execute() diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt index 77eeabe9181..753042dcc7c 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt @@ -27,33 +27,41 @@ class OutputFiles(outputPath: String?, target: KonanTarget, val produce: Compile /** * Header file for dynamic library. */ - val cAdapterHeader by lazy { File("${outputName}_api.h") } - val cAdapterDef by lazy { File("${outputName}.def") } + val cAdapterHeader by lazy { File("${outputName}_api.h") } + val cAdapterDef by lazy { File("${outputName}.def") } /** * Main compiler's output file. */ - val mainFile = outputName - .prefixBaseNameIfNeeded(prefix) - .suffixIfNeeded(suffix) + val mainFile = + if (produce.isCache) + outputName + else + outputName.fullOutputName() - val mainFileMangled = if (!produce.isCache) mainFile else { - (outputName + Random.nextLong().toString()) - .prefixBaseNameIfNeeded(prefix) - .suffixIfNeeded(suffix) - } + val tempCacheDirectory = + if (produce.isCache) + File(outputName + Random.nextLong().toString()) + else null - private fun String.prefixBaseNameIfNeeded(prefix: String): String { - return if (produce.isCache) - prefixBaseNameAlways(prefix) - else prefixBaseNameIfNot(prefix) - } + val nativeBinaryFile = + if (produce.isCache) + tempCacheDirectory!!.child(File(outputName.fullOutputName()).absoluteFile.name).absolutePath + else mainFile - private fun String.suffixIfNeeded(prefix: String): String { - return if (produce.isCache) - suffixAlways(prefix) - else suffixIfNot(prefix) - } + val symbolicInfoFile = "$nativeBinaryFile.dSYM" + + private fun String.fullOutputName() = prefixBaseNameIfNeeded(prefix).suffixIfNeeded(suffix) + + private fun String.prefixBaseNameIfNeeded(prefix: String) = + if (produce.isCache) + prefixBaseNameAlways(prefix) + else prefixBaseNameIfNot(prefix) + + private fun String.suffixIfNeeded(prefix: String) = + if (produce.isCache) + suffixAlways(prefix) + else suffixIfNot(prefix) private fun String.prefixBaseNameAlways(prefix: String): String { val file = File(this).absoluteFile diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/CacheTesting.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/CacheTesting.kt index ecf62e3fe6c..4e07ea9d2b5 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/CacheTesting.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/CacheTesting.kt @@ -25,7 +25,7 @@ fun configureCacheTesting(project: Project): CacheTesting? { val target = project.testTarget val cacheDir = project.file("${project.buildDir}/cache") - val cacheFile = "$cacheDir/${cacheKind.prefix(target)}stdlib-cache${cacheKind.suffix(target)}" + val cacheFile = "$cacheDir/stdlib-cache" val dist = project.kotlinNativeDist val stdlib = "$dist/klib/common/stdlib" val compilerArgs = listOf("-Xcached-library=$stdlib,$cacheFile") diff --git a/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCacheTask.kt b/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCacheTask.kt index 3ff6c8ce028..9093f9810e6 100644 --- a/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCacheTask.kt +++ b/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCacheTask.kt @@ -31,15 +31,11 @@ open class KonanCacheTask: DefaultTask() { val cacheDirectory: File get() = cacheRoot.resolve("$target-g$cacheKind") - @get:OutputFile + @get:OutputDirectory protected val cacheFile: File get() { - val konanTarget = HostManager().targetByName(target) val klibName = originalKlib.nameWithoutExtension - val cachePrefix = cacheKind.outputKind.prefix(konanTarget) - val cacheSuffix = cacheKind.outputKind.suffix(konanTarget) - val cacheName = "${cachePrefix}${klibName}-cache${cacheSuffix}" - return cacheDirectory.resolve(cacheName) + return cacheDirectory.resolve("${klibName}-cache") } @Input diff --git a/utilities/cli-runner/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/GeneratePlatformLibraries.kt b/utilities/cli-runner/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/GeneratePlatformLibraries.kt index 492e84acdfb..47220bbabe0 100644 --- a/utilities/cli-runner/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/GeneratePlatformLibraries.kt +++ b/utilities/cli-runner/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/GeneratePlatformLibraries.kt @@ -260,7 +260,7 @@ private fun generateLibrary( } } -private fun getCacheFile( +private fun getLibraryCacheDir( libraryName: String, target: KonanTarget, cacheDirectory: File, @@ -279,14 +279,14 @@ private fun buildCache( rebuild: Boolean, logger: Logger ) = with(cacheInfo) { - val cacheFile = getCacheFile(def.name, target, cacheDirectory, cacheKind) - if (cacheFile.exists && !rebuild) { + val libraryCacheDir = getLibraryCacheDir(def.name, target, cacheDirectory, cacheKind) + if (libraryCacheDir.listFilesOrEmpty.isNotEmpty() && !rebuild) { logger.verbose("Skip precompiling ${def.name} as it's already precompiled") return } if (rebuild) { - cacheFile.delete() + libraryCacheDir.deleteRecursively() } val compilerArgs = arrayOf( @@ -307,7 +307,7 @@ private fun buildStdlibCache( cacheInfo: CacheInfo, logger: Logger ) = with(cacheInfo) { - val stdlibCacheFile = getCacheFile("stdlib", target, cacheDirectory, cacheKind) + val stdlibCacheFile = getLibraryCacheDir("stdlib", target, cacheDirectory, cacheKind) if (stdlibCacheFile.exists) { logger.verbose("Skip precompiling standard library as it's already precompiled") return