Produce deterministic jar files.

This resets all timestamps present in jars produced by kotlinc. This is important for build systems like bazel that rely on deterministic outputs. Before: ``` $ kotlinc ~/test.kt -d /tmp/a.jar $ kotlinc ~/test.kt -d /tmp/b.jar 9ab80cd40c9293a7a66adf1154d4e6e9aa92d5b0 /tmp/a.jar 1ba022697317f796bd123fb4dc95418a18bcb51a /tmp/a.jar 6d2a2683470c24928f3fbd6768a4c57f55b0d196 /tmp/b.jar $ unzip -l /tmp/a.jar Archive: /tmp/a.jar Length Date Time Name --------- ---------- ----- ---- 75 09-25-2020 16:48 META-INF/MANIFEST.MF 683 09-25-2020 16:48 TestKt.class 28 09-25-2020 16:48 META-INF/main.kotlin_module --------- ------- 786 3 files ``` After: ``` $ kotlinc ~/test.kt -d /tmp/a.jar $ kotlinc ~/test.kt -d /tmp/b.jar $ shasum /tmp/a.jar /tmp/b.jar 9ab80cd40c9293a7a66adf1154d4e6e9aa92d5b0 /tmp/a.jar 9ab80cd40c9293a7a66adf1154d4e6e9aa92d5b0 /tmp/b.jar $ unzip -l /tmp/a.jar Archive: /tmp/a.jar Length Date Time Name --------- ---------- ----- ---- 75 12-31-1969 19:00 META-INF/MANIFEST.MF 590 12-31-1969 19:00 TestKt.class 36 12-31-1969 19:00 META-INF/main.kotlin_module --------- ------- 701 3 files ``` See https://github.com/JetBrains/kotlin/pull/3226 for a similar change.
2020-09-25 16:45:12 -04:00
parent a3830b2611
commit 68fdeaf865
7 changed files with 130 additions and 10 deletions
@@ -377,6 +377,12 @@ class K2JVMCompilerArguments : CommonCompilerArguments() {
    )
    var noKotlinNothingValueException: Boolean by FreezableVar(false)

+    @Argument(
+        value = "-Xno-reset-jar-timestamps",
+        description = "Do not reset jar entry timestamps to a fixed date"
+    )
+    var noResetJarTimestamps: Boolean by FreezableVar(false)
+
    @Argument(
        value = "-Xno-unified-null-checks",
        description = "Use pre-1.4 exception types in null checks instead of java.lang.NPE. See KT-22275 for more details"
@@ -31,11 +31,15 @@ import org.jetbrains.kotlin.utils.ExceptionUtilsKt;
 import org.jetbrains.kotlin.utils.PathUtil;

 import java.io.*;
+import java.util.*;
 import java.util.jar.*;

 import static org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR;

 public class CompileEnvironmentUtil {
+
+    public static long DOS_EPOCH = new GregorianCalendar(1980, Calendar.JANUARY, 1, 0, 0, 0).getTimeInMillis();
+
    @NotNull
    public static ModuleChunk loadModuleChunk(File buildFile, MessageCollector messageCollector) {
        if (!buildFile.exists()) {
@@ -51,7 +55,11 @@ public class CompileEnvironmentUtil {

    // TODO: includeRuntime should be not a flag but a path to runtime
    private static void doWriteToJar(
-            OutputFileCollection outputFiles, OutputStream fos, @Nullable FqName mainClass, boolean includeRuntime
+            OutputFileCollection outputFiles,
+            OutputStream fos,
+            @Nullable FqName mainClass,
+            boolean includeRuntime,
+            boolean resetJarTimestamps
    ) {
        try {
            Manifest manifest = new Manifest();
@@ -61,13 +69,25 @@ public class CompileEnvironmentUtil {
            if (mainClass != null) {
                mainAttributes.putValue("Main-Class", mainClass.asString());
            }
-            JarOutputStream stream = new JarOutputStream(fos, manifest);
+
+            JarOutputStream stream = new JarOutputStream(fos);
+            JarEntry manifestEntry = new JarEntry(JarFile.MANIFEST_NAME);
+            if (resetJarTimestamps) {
+                manifestEntry.setTime(DOS_EPOCH);
+            }
+            stream.putNextEntry(manifestEntry);
+            manifest.write(new BufferedOutputStream(stream));
+
            for (OutputFile outputFile : outputFiles.asList()) {
-                stream.putNextEntry(new JarEntry(outputFile.getRelativePath()));
+                JarEntry entry = new JarEntry(outputFile.getRelativePath());
+                if (resetJarTimestamps) {
+                    entry.setTime(DOS_EPOCH);
+                }
+                stream.putNextEntry(entry);
                stream.write(outputFile.asByteArray());
            }
            if (includeRuntime) {
-                writeRuntimeToJar(stream);
+                writeRuntimeToJar(stream, resetJarTimestamps);
            }
            stream.finish();
        }
@@ -76,11 +96,13 @@ public class CompileEnvironmentUtil {
        }
    }

-    public static void writeToJar(File jarPath, boolean jarRuntime, FqName mainClass, OutputFileCollection outputFiles) {
+    public static void writeToJar(
+            File jarPath, boolean jarRuntime, boolean resetJarTimestamps, FqName mainClass, OutputFileCollection outputFiles
+    ) {
        FileOutputStream outputStream = null;
        try {
            outputStream = new FileOutputStream(jarPath);
-            doWriteToJar(outputFiles, outputStream, mainClass, jarRuntime);
+            doWriteToJar(outputFiles, outputStream, mainClass, jarRuntime, resetJarTimestamps);
            outputStream.close();
        }
        catch (FileNotFoundException e) {
@@ -94,21 +116,24 @@ public class CompileEnvironmentUtil {
        }
    }

-    private static void writeRuntimeToJar(JarOutputStream stream) throws IOException {
+    private static void writeRuntimeToJar(JarOutputStream stream, boolean resetJarTimestamps) throws IOException {
        File stdlibPath = PathUtil.getKotlinPathsForCompiler().getStdlibPath();
        if (!stdlibPath.exists()) {
            throw new CompileEnvironmentException("Couldn't find kotlin-stdlib at " + stdlibPath);
        }
-        copyJarImpl(stream, stdlibPath);
+        copyJarImpl(stream, stdlibPath, resetJarTimestamps);
    }

-    private static void copyJarImpl(JarOutputStream stream, File jarPath) throws IOException {
+    private static void copyJarImpl(JarOutputStream stream, File jarPath, boolean resetJarTimestamps) throws IOException {
        try (JarInputStream jis = new JarInputStream(new FileInputStream(jarPath))) {
            while (true) {
                JarEntry e = jis.getNextJarEntry();
                if (e == null) {
                    break;
                }
+                if (resetJarTimestamps) {
+                    e.setTime(DOS_EPOCH);
+                }
                if (FileUtilRt.extensionEquals(e.getName(), "class")) {
                    stream.putNextEntry(e);
                    FileUtil.copy(jis, stream);
@@ -88,7 +88,8 @@ object KotlinToJVMBytecodeCompiler {
        val messageCollector = configuration.get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, MessageCollector.NONE)
        if (jarPath != null) {
            val includeRuntime = configuration.get(JVMConfigurationKeys.INCLUDE_RUNTIME, false)
-            CompileEnvironmentUtil.writeToJar(jarPath, includeRuntime, mainClassFqName, outputFiles)
+            val resetJarTimestamps = !configuration.get(JVMConfigurationKeys.NO_RESET_JAR_TIMESTAMPS, false)
+            CompileEnvironmentUtil.writeToJar(jarPath, includeRuntime, resetJarTimestamps, mainClassFqName, outputFiles)
            if (reportOutputFiles) {
                val message = OutputMessageUtil.formatOutputMessage(outputFiles.asList().flatMap { it.sourceFiles }.distinct(), jarPath)
                messageCollector.report(OUTPUT, message)
@@ -185,6 +185,7 @@ fun CompilerConfiguration.configureAdvancedJvmOptions(arguments: K2JVMCompilerAr
    put(JVMConfigurationKeys.EMIT_JVM_TYPE_ANNOTATIONS, arguments.emitJvmTypeAnnotations)
    put(JVMConfigurationKeys.NO_OPTIMIZED_CALLABLE_REFERENCES, arguments.noOptimizedCallableReferences)
    put(JVMConfigurationKeys.NO_KOTLIN_NOTHING_VALUE_EXCEPTION, arguments.noKotlinNothingValueException)
+    put(JVMConfigurationKeys.NO_RESET_JAR_TIMESTAMPS, arguments.noResetJarTimestamps)
    put(JVMConfigurationKeys.NO_UNIFIED_NULL_CHECKS, arguments.noUnifiedNullChecks)

    if (!JVMConstructorCallNormalizationMode.isSupportedValue(arguments.constructorCallNormalizationMode)) {
@@ -129,6 +129,9 @@ public class JVMConfigurationKeys {
    public static final CompilerConfigurationKey<Boolean> NO_KOTLIN_NOTHING_VALUE_EXCEPTION =
            CompilerConfigurationKey.create("Do not use KotlinNothingValueException available since 1.4");

+    public static final CompilerConfigurationKey<Boolean> NO_RESET_JAR_TIMESTAMPS =
+            CompilerConfigurationKey.create("Do not reset timestamps in jar entries");
+
    public static final CompilerConfigurationKey<Boolean> NO_UNIFIED_NULL_CHECKS =
            CompilerConfigurationKey.create("Use pre-1.4 exception types in null checks instead of java.lang.NPE");

@@ -78,6 +78,7 @@ where advanced options include:
                             Do not use optimized callable reference superclasses available from 1.4
  -Xno-param-assertions      Don't generate not-null assertions on parameters of methods accessible from Java
  -Xno-receiver-assertions   Don't generate not-null assertion for extension receiver arguments of platform types
+  -Xno-reset-jar-timestamps  Do not reset jar entry timestamps to a fixed date
  -Xno-unified-null-checks   Use pre-1.4 exception types in null checks instead of java.lang.NPE. See KT-22275 for more details
  -Xno-use-ir                Do not use the IR backend. Useful for a custom-built compiler where IR backend is enabled by default
  -Xprofile=<profilerPath:command:outputDir>
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.cli
+
+import java.io.File
+import java.io.FileInputStream
+import java.util.jar.JarInputStream
+import java.util.zip.ZipEntry
+import kotlin.test.assertEquals
+import kotlin.test.assertNotEquals
+import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
+import org.jetbrains.kotlin.cli.jvm.compiler.CompileEnvironmentUtil.DOS_EPOCH
+import org.jetbrains.kotlin.test.TestCaseWithTmpdir
+
+class DeterministicOutputTest : TestCaseWithTmpdir() {
+
+    fun testDeterministicOutput() {
+        val fooKt = tmpdir.resolve("foo.kt").also {
+            it.writeText("class Foo")
+        }
+
+        val firstJar = tmpdir.resolve("first.jar")
+        AbstractCliTest.executeCompilerGrabOutput(
+            K2JVMCompiler(),
+            listOf(fooKt.path, "-d", firstJar.path, "-include-runtime"))
+
+        val secondJar = tmpdir.resolve("second.jar")
+        AbstractCliTest.executeCompilerGrabOutput(
+            K2JVMCompiler(),
+            listOf(fooKt.path, "-d", secondJar.path, "-include-runtime"))
+
+        assertEquals(
+            firstJar.readBytes().toList(),
+            secondJar.readBytes().toList(),
+            "jar contents should be identical if compiler command and inputs are the same")
+        assertAllTimestampsAreReset(firstJar)
+        assertAllTimestampsAreReset(secondJar)
+    }
+
+    fun testNoResetJarTimestamps() {
+        val fooKt = tmpdir.resolve("foo.kt").also {
+            it.writeText("class Foo")
+        }
+        val jar = tmpdir.resolve("jarWithTimestamps.jar")
+        AbstractCliTest.executeCompilerGrabOutput(
+            K2JVMCompiler(),
+            listOf(fooKt.path, "-d", jar.path, "-include-runtime", "-Xno-reset-jar-timestamps"))
+
+        assertNoTimestampsAreReset(jar)
+    }
+
+    private fun assertAllTimestampsAreReset(jar: File) {
+        val zis = JarInputStream(FileInputStream(jar))
+        var entry: ZipEntry? = zis.nextEntry
+        while (entry != null) {
+            assertEquals(entry.time, DOS_EPOCH, "$entry timestamp should be reset")
+            entry = zis.nextEntry
+        }
+    }
+
+    private fun assertNoTimestampsAreReset(jar: File) {
+        val zis = JarInputStream(FileInputStream(jar))
+        var entry: ZipEntry? = zis.nextEntry
+        while (entry != null) {
+            assertNotEquals(entry.time, DOS_EPOCH, "$entry timestamp should not be reset")
+            entry = zis.nextEntry
+        }
+    }
+}