[caches] Make output for cache a directory

(cherry picked from commit 9850bb322bee2bc98fc453a440e41af0dac0b0f2)
2020-06-11 23:15:37 +05:00
parent 57621d12c5
commit 90044108e0
6 changed files with 67 additions and 59 deletions
@@ -23,34 +23,39 @@ class CachedLibraries(
        enum class Kind { DYNAMIC, STATIC }
    }

+    private fun selectCache(library: KotlinLibrary, cacheDir: File): Cache? {
+        // See Linker.renameOutput why is it ok to have an empty cache directory.
+        if (cacheDir.listFilesOrEmpty.isEmpty()) return null
+        val baseName = getCachedLibraryName(library)
+        val dynamicFile = cacheDir.child(getArtifactName(baseName, CompilerOutputKind.DYNAMIC_CACHE))
+        val staticFile = cacheDir.child(getArtifactName(baseName, CompilerOutputKind.STATIC_CACHE))
+
+        if (dynamicFile.exists && staticFile.exists)
+            error("Both dynamic and static caches files cannot be in the same directory." +
+                    " Library: ${library.libraryName}, path to cache: ${cacheDir.absolutePath}")
+        return when {
+            dynamicFile.exists -> Cache(Cache.Kind.DYNAMIC, dynamicFile.absolutePath)
+            staticFile.exists -> Cache(Cache.Kind.STATIC, staticFile.absolutePath)
+            else -> error("No cache found for library ${library.libraryName} at ${cacheDir.absolutePath}")
+        }
+    }
+
    private val allCaches: Map<KotlinLibrary, Cache> = allLibraries.mapNotNull { library ->
        val explicitPath = explicitCaches[library]

        val cache = if (explicitPath != null) {
-            val kind = when {
-                explicitPath.endsWith(target.family.dynamicSuffix) -> Cache.Kind.DYNAMIC
-                explicitPath.endsWith(target.family.staticSuffix) -> Cache.Kind.STATIC
-                else -> error("unexpected cache: $explicitPath")
-            }
-            Cache(kind, explicitPath)
+            selectCache(library, File(explicitPath))
+                    ?: error("No cache found for library ${library.libraryName} at $explicitPath")
        } else {
            implicitCacheDirectories.firstNotNullResult { dir ->
-                val baseName = getCachedLibraryName(library)
-                val dynamicFile = dir.child(getArtifactName(baseName, CompilerOutputKind.DYNAMIC_CACHE))
-                val staticFile = dir.child(getArtifactName(baseName, CompilerOutputKind.STATIC_CACHE))
-
-                when {
-                    dynamicFile.exists -> Cache(Cache.Kind.DYNAMIC, dynamicFile.absolutePath)
-                    staticFile.exists -> Cache(Cache.Kind.STATIC, staticFile.absolutePath)
-                    else -> null
-                }
+                selectCache(library, dir.child(getCachedLibraryName(library)))
            }
        }

        cache?.let { library to it }
    }.toMap()

-    fun getArtifactName(baseName: String, kind: CompilerOutputKind) =
+    private fun getArtifactName(baseName: String, kind: CompilerOutputKind) =
            "${kind.prefix(target)}$baseName${kind.suffix(target)}"

    fun isLibraryCached(library: KotlinLibrary): Boolean =
@@ -5,7 +5,6 @@ import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.target.CompilerOutputKind
 import org.jetbrains.kotlin.konan.target.Family
 import org.jetbrains.kotlin.konan.target.LinkerOutputKind
-import org.jetbrains.kotlin.backend.konan.files.renameAtomic
 import org.jetbrains.kotlin.konan.library.KonanLibrary

 internal fun determineLinkerOutput(context: Context): LinkerOutputKind =
@@ -45,21 +44,21 @@ internal class Linker(val context: Context) {

        val libraryProvidedLinkerFlags = context.llvm.allNativeDependencies.map { it.linkerOpts }.flatten()

+        if (context.config.produce.isCache)
+            context.config.outputFiles.tempCacheDirectory!!.mkdirs()
        runLinker(objectFiles, includedBinaries, libraryProvidedLinkerFlags)
+
        renameOutput()
    }

    private fun renameOutput() {
        if (context.config.produce.isCache) {
            val outputFiles = context.config.outputFiles
-            val outputFile = java.io.File(outputFiles.mainFileMangled)
-            val outputDsymBundle = java.io.File(outputFiles.mainFileMangled + ".dSYM")
-            if (renameAtomic(outputFile.absolutePath, outputFiles.mainFile, /* replaceExisting = */ false))
-                outputDsymBundle.renameTo(java.io.File(outputFiles.mainFile + ".dSYM"))
-            else {
-                outputFile.delete()
-                outputDsymBundle.deleteRecursively()
-            }
+            // For caches the output file is a directory. It might be created by someone else,
+            // We have to delete it in order to the next renaming operation to succeed.
+            java.io.File(outputFiles.mainFile).delete()
+            if (!java.io.File(outputFiles.tempCacheDirectory!!.absolutePath).renameTo(java.io.File(outputFiles.mainFile)))
+                outputFiles.tempCacheDirectory.deleteRecursively()
        }
    }

@@ -96,7 +95,7 @@ internal class Linker(val context: Context) {
            } else {
                emptyList()
            }
-            executable = context.config.outputFiles.mainFileMangled
+            executable = context.config.outputFiles.nativeBinaryFile
        } else {
            val framework = File(context.config.outputFile)
            val dylibName = framework.name.removeSuffix(".framework")
@@ -127,7 +126,7 @@ internal class Linker(val context: Context) {
                            caches.dynamic +
                            libraryProvidedLinkerFlags + additionalLinkerArgs,
                    optimize = optimize, debug = debug, kind = linkerOutput,
-                    outputDsymBundle = context.config.outputFiles.mainFileMangled + ".dSYM",
+                    outputDsymBundle = context.config.outputFiles.symbolicInfoFile,
                    needsProfileLibrary = needsProfileLibrary).forEach {
                it.logWith(context::log)
                it.execute()
@@ -27,33 +27,41 @@ class OutputFiles(outputPath: String?, target: KonanTarget, val produce: Compile
    /**
     * Header file for dynamic library.
     */
-    val cAdapterHeader      by lazy { File("${outputName}_api.h") }
-    val cAdapterDef         by lazy { File("${outputName}.def") }
+    val cAdapterHeader by lazy { File("${outputName}_api.h") }
+    val cAdapterDef    by lazy { File("${outputName}.def") }

    /**
     * Main compiler's output file.
     */
-    val mainFile = outputName
-            .prefixBaseNameIfNeeded(prefix)
-            .suffixIfNeeded(suffix)
+    val mainFile =
+            if (produce.isCache)
+                outputName
+            else
+                outputName.fullOutputName()

-    val mainFileMangled = if (!produce.isCache) mainFile else {
-        (outputName + Random.nextLong().toString())
-                .prefixBaseNameIfNeeded(prefix)
-                .suffixIfNeeded(suffix)
-    }
+    val tempCacheDirectory =
+            if (produce.isCache)
+                File(outputName + Random.nextLong().toString())
+            else null

-    private fun String.prefixBaseNameIfNeeded(prefix: String): String {
-        return if (produce.isCache)
-            prefixBaseNameAlways(prefix)
-        else prefixBaseNameIfNot(prefix)
-    }
+    val nativeBinaryFile =
+            if (produce.isCache)
+                tempCacheDirectory!!.child(File(outputName.fullOutputName()).absoluteFile.name).absolutePath
+            else mainFile

-    private fun String.suffixIfNeeded(prefix: String): String {
-        return if (produce.isCache)
-            suffixAlways(prefix)
-        else suffixIfNot(prefix)
-    }
+    val symbolicInfoFile = "$nativeBinaryFile.dSYM"
+
+    private fun String.fullOutputName() = prefixBaseNameIfNeeded(prefix).suffixIfNeeded(suffix)
+
+    private fun String.prefixBaseNameIfNeeded(prefix: String) =
+            if (produce.isCache)
+                prefixBaseNameAlways(prefix)
+            else prefixBaseNameIfNot(prefix)
+
+    private fun String.suffixIfNeeded(prefix: String) =
+            if (produce.isCache)
+                suffixAlways(prefix)
+            else suffixIfNot(prefix)

    private fun String.prefixBaseNameAlways(prefix: String): String {
        val file = File(this).absoluteFile
@@ -25,7 +25,7 @@ fun configureCacheTesting(project: Project): CacheTesting? {
    val target = project.testTarget

    val cacheDir = project.file("${project.buildDir}/cache")
-    val cacheFile = "$cacheDir/${cacheKind.prefix(target)}stdlib-cache${cacheKind.suffix(target)}"
+    val cacheFile = "$cacheDir/stdlib-cache"
    val dist = project.kotlinNativeDist
    val stdlib = "$dist/klib/common/stdlib"
    val compilerArgs = listOf("-Xcached-library=$stdlib,$cacheFile")
@@ -31,15 +31,11 @@ open class KonanCacheTask: DefaultTask() {
    val cacheDirectory: File
        get() = cacheRoot.resolve("$target-g$cacheKind")

-    @get:OutputFile
+    @get:OutputDirectory
    protected val cacheFile: File
        get() {
-            val konanTarget = HostManager().targetByName(target)
            val klibName = originalKlib.nameWithoutExtension
-            val cachePrefix = cacheKind.outputKind.prefix(konanTarget)
-            val cacheSuffix = cacheKind.outputKind.suffix(konanTarget)
-            val cacheName = "${cachePrefix}${klibName}-cache${cacheSuffix}"
-            return cacheDirectory.resolve(cacheName)
+            return cacheDirectory.resolve("${klibName}-cache")
        }

    @Input
@@ -260,7 +260,7 @@ private fun generateLibrary(
    }
 }

-private fun getCacheFile(
+private fun getLibraryCacheDir(
        libraryName: String,
        target: KonanTarget,
        cacheDirectory: File,
@@ -279,14 +279,14 @@ private fun buildCache(
        rebuild: Boolean,
        logger: Logger
 ) = with(cacheInfo) {
-    val cacheFile = getCacheFile(def.name, target, cacheDirectory, cacheKind)
-    if (cacheFile.exists && !rebuild) {
+    val libraryCacheDir = getLibraryCacheDir(def.name, target, cacheDirectory, cacheKind)
+    if (libraryCacheDir.listFilesOrEmpty.isNotEmpty() && !rebuild) {
        logger.verbose("Skip precompiling ${def.name} as it's already precompiled")
        return
    }

    if (rebuild) {
-        cacheFile.delete()
+        libraryCacheDir.deleteRecursively()
    }

    val compilerArgs = arrayOf(
@@ -307,7 +307,7 @@ private fun buildStdlibCache(
        cacheInfo: CacheInfo,
        logger: Logger
 ) = with(cacheInfo) {
-    val stdlibCacheFile = getCacheFile("stdlib", target, cacheDirectory, cacheKind)
+    val stdlibCacheFile = getLibraryCacheDir("stdlib", target, cacheDirectory, cacheKind)
    if (stdlibCacheFile.exists) {
        logger.verbose("Skip precompiling standard library as it's already precompiled")
        return