From bc96e42585800d615255d53349da000bf995b2e9 Mon Sep 17 00:00:00 2001 From: Martin Petrov Date: Wed, 25 Mar 2020 13:22:09 -0400 Subject: [PATCH] Produce deterministic klib files. Build systems like bazel rely on the output of a compilation being deterministic. File checksums determine when an artifact is safe to cache across builds. One can produce a klib deterministically by using a fixed timestamp during klib compression. Before this change: ``` $ kotlinc-native -produce library -output /tmp/first/color.klib Color.kt $ kotlinc-native -produce library -output /tmp/second/color.klib Color.kt $ shasum /tmp/first/color.klib /tmp/second/color.klib bc3f73678ff025cfbec9009f9a851a8ca74e1037 /tmp/first/color.klib 65aa37886fbd53285f2e449a4dab6a2ad02732e6 /tmp/second/color.klib ``` After this change: ``` $ kotlinc-native -produce library -output /tmp/first/color.klib Color.kt $ kotlinc-native -produce library -output /tmp/second/color.klib Color.kt $ shasum /tmp/first/color.klib /tmp/second/color.klib fcba304493916ae34d372188991f87b60a113cf3 /tmp/first/color.klib fcba304493916ae34d372188991f87b60a113cf3 /tmp/second/color.klib ``` --- .../src/org/jetbrains/kotlin/konan/file/File.kt | 12 +++++++++--- .../src/org/jetbrains/kotlin/konan/file/ZipUtil.kt | 4 +++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/compiler/util-io/src/org/jetbrains/kotlin/konan/file/File.kt b/compiler/util-io/src/org/jetbrains/kotlin/konan/file/File.kt index 2bb6fdc12b2..af139394705 100644 --- a/compiler/util-io/src/org/jetbrains/kotlin/konan/file/File.kt +++ b/compiler/util-io/src/org/jetbrains/kotlin/konan/file/File.kt @@ -14,7 +14,9 @@ import java.lang.Exception import java.nio.MappedByteBuffer import java.nio.channels.FileChannel import java.nio.file.* +import java.nio.file.attribute.BasicFileAttributeView import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.attribute.FileTime data class File constructor(internal val javaPath: Path) { constructor(parent: Path, child: String): this(parent.resolve(child)) @@ -62,10 +64,10 @@ data class File constructor(internal val javaPath: Path) { Files.copy(javaPath, destination.javaPath, StandardCopyOption.REPLACE_EXISTING) } - fun recursiveCopyTo(destination: File) { + fun recursiveCopyTo(destination: File, resetTimeAttributes: Boolean = false) { val sourcePath = javaPath val destPath = destination.javaPath - sourcePath.recursiveCopyTo(destPath) + sourcePath.recursiveCopyTo(destPath, resetTimeAttributes = resetTimeAttributes) } fun mkdirs() = Files.createDirectories(javaPath) @@ -185,7 +187,7 @@ fun createTempFile(name: String, suffix: String? = null) fun createTempDir(name: String): File = Files.createTempDirectory(name).File() -fun Path.recursiveCopyTo(destPath: Path) { +fun Path.recursiveCopyTo(destPath: Path, resetTimeAttributes: Boolean = false) { val sourcePath = this Files.walk(sourcePath).forEach next@ { oldPath -> @@ -202,6 +204,10 @@ fun Path.recursiveCopyTo(destPath: Path) { } else { Files.copy(oldPath, newPath, StandardCopyOption.REPLACE_EXISTING) } + if (resetTimeAttributes) { + val zero = FileTime.fromMillis(0) + Files.getFileAttributeView(newPath, BasicFileAttributeView::class.java).setTimes(zero, zero, zero); + } } } diff --git a/compiler/util-io/src/org/jetbrains/kotlin/konan/file/ZipUtil.kt b/compiler/util-io/src/org/jetbrains/kotlin/konan/file/ZipUtil.kt index 65fea7bbdb6..a94ce04f054 100644 --- a/compiler/util-io/src/org/jetbrains/kotlin/konan/file/ZipUtil.kt +++ b/compiler/util-io/src/org/jetbrains/kotlin/konan/file/ZipUtil.kt @@ -41,7 +41,9 @@ private fun File.toPath() = Paths.get(this.path) fun File.zipDirAs(unixFile: File) { unixFile.withMutableZipFileSystem { - this.recursiveCopyTo(it.file("/")) + // Time attributes are not preserved to ensure that the output zip file bytes deterministic for a fixed set of + // input files. + this.recursiveCopyTo(it.file("/"), resetTimeAttributes = true) } }