KT-45777: Add custom serialization for classpath snapshot

Also add a few commonly-used classes/methods to externalizers.kt to allow reuse. Bug: KT-45777 Test: New ClasspathSnapshotSerializerTest
2021-07-21 15:51:53 +03:00
parent 0d2a514a70
commit 16dfdb620e
9 changed files with 358 additions and 148 deletions
@@ -6,7 +6,6 @@
 package org.jetbrains.kotlin.gradle.incremental

 import org.jetbrains.kotlin.incremental.KotlinClassInfo
-import java.io.*

 /** Snapshot of a classpath. It consists of a list of [ClasspathEntrySnapshot]s. */
 class ClasspathSnapshot(val classpathEntrySnapshots: List<ClasspathEntrySnapshot>)
@@ -19,64 +18,20 @@ class ClasspathEntrySnapshot(
     * jar).
     */
    val classSnapshots: LinkedHashMap<String, ClassSnapshot>
-) : Serializable {
-
-    companion object {
-        private const val serialVersionUID = 0L
-    }
-}
+)

 /**
- * Snapshot of a class. It contains information to compute the source files that need to be recompiled during an incremental run of the
- * `KotlinCompile` task.
+ * Snapshot of a class. It contains minimal information about a class to compute the source files that need to be recompiled during an
+ * incremental run of the `KotlinCompile` task.
+ *
+ * It's important that this class contain only the minimal required information, as it will be part of the classpath snapshot of the
+ * `KotlinCompile` task and the task needs to support compile avoidance. For example, this class should contain public method signatures,
+ * and should not contain private method signatures, or method implementations.
 */
-abstract class ClassSnapshot : Serializable {
+sealed class ClassSnapshot

-    companion object {
-        private const val serialVersionUID = 0L
-    }
-}
+/** [ClassSnapshot] of a Kotlin class. */
+class KotlinClassSnapshot(val classInfo: KotlinClassInfo) : ClassSnapshot()

-/** [ClassSnapshot] of a kotlinc-generated class. */
-class KotlinClassSnapshot(val classInfo: KotlinClassInfo) : ClassSnapshot(), Serializable {
-
-    companion object {
-        private const val serialVersionUID = 0L
-    }
-}
-
-/** [ClassSnapshot] of a non-kotlinc-generated class. */
-class JavaClassSnapshot : ClassSnapshot(), Serializable {
-
-    // TODO WORK-IN-PROGRESS
-
-    companion object {
-        private const val serialVersionUID = 0L
-    }
-}
-
-/** Utility to read/write a [ClasspathSnapshot] from/to a file. */
-object ClasspathSnapshotSerializer {
-
-    fun readFromFiles(classpathEntrySnapshotFiles: List<File>): ClasspathSnapshot {
-        return ClasspathSnapshot(classpathEntrySnapshotFiles.map { ClasspathEntrySnapshotSerializer.readFromFile(it) })
-    }
-}
-
-/** Utility to read/write a [ClasspathEntrySnapshot] from/to a file. */
-object ClasspathEntrySnapshotSerializer {
-
-    fun readFromFile(classpathEntrySnapshotFile: File): ClasspathEntrySnapshot {
-        check(classpathEntrySnapshotFile.isFile) { "`${classpathEntrySnapshotFile.path}` does not exist (or is a directory)." }
-        return ObjectInputStream(FileInputStream(classpathEntrySnapshotFile).buffered()).use {
-            it.readObject() as ClasspathEntrySnapshot
-        }
-    }
-
-    fun writeToFile(classpathEntrySnapshot: ClasspathEntrySnapshot, classpathEntrySnapshotFile: File) {
-        check(classpathEntrySnapshotFile.parentFile.exists()) { "Parent dir of `${classpathEntrySnapshotFile.path}` does not exist." }
-        ObjectOutputStream(FileOutputStream(classpathEntrySnapshotFile).buffered()).use {
-            it.writeObject(classpathEntrySnapshot)
-        }
-    }
-}
+/** [ClassSnapshot] of a Java class. */
+object JavaClassSnapshot : ClassSnapshot()
@@ -0,0 +1,171 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.incremental
+
+import com.intellij.util.io.DataExternalizer
+import org.jetbrains.kotlin.incremental.KotlinClassInfo
+import org.jetbrains.kotlin.incremental.storage.*
+import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
+import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.name.FqName
+import java.io.*
+
+/** Utility to serialize a [ClasspathSnapshot]. */
+object ClasspathSnapshotSerializer {
+
+    fun load(classpathEntrySnapshotFiles: List<File>): ClasspathSnapshot {
+        return ClasspathSnapshot(classpathEntrySnapshotFiles.map {
+            ClasspathEntrySnapshotSerializer.load(it)
+        })
+    }
+}
+
+object ClasspathEntrySnapshotSerializer : DataSerializer<ClasspathEntrySnapshot> {
+
+    override fun save(output: DataOutput, snapshot: ClasspathEntrySnapshot) {
+        LinkedHashMapExternalizer(StringExternalizer, ClassSnapshotDataSerializer).save(output, snapshot.classSnapshots)
+    }
+
+    override fun read(input: DataInput): ClasspathEntrySnapshot {
+        return ClasspathEntrySnapshot(
+            classSnapshots = LinkedHashMapExternalizer(StringExternalizer, ClassSnapshotDataSerializer).read(input)
+        )
+    }
+}
+
+object ClassSnapshotDataSerializer : DataSerializer<ClassSnapshot> {
+
+    override fun save(output: DataOutput, snapshot: ClassSnapshot) {
+        output.writeString(snapshot.javaClass.name)
+        when (snapshot) {
+            is KotlinClassSnapshot -> KotlinClassSnapshotExternalizer.save(output, snapshot)
+            is JavaClassSnapshot -> JavaClassSnapshotExternalizer.save(output, snapshot)
+        }
+    }
+
+    override fun read(input: DataInput): ClassSnapshot {
+        return when (val className = input.readString()) {
+            KotlinClassSnapshot::class.java.name -> KotlinClassSnapshotExternalizer.read(input)
+            JavaClassSnapshot::class.java.name -> JavaClassSnapshotExternalizer.read(input)
+            else -> error("Unrecognized class: $className")
+        }
+    }
+}
+
+object KotlinClassSnapshotExternalizer : DataExternalizer<KotlinClassSnapshot> {
+
+    override fun save(output: DataOutput, snapshot: KotlinClassSnapshot) {
+        KotlinClassInfoExternalizer.save(output, snapshot.classInfo)
+    }
+
+    override fun read(input: DataInput): KotlinClassSnapshot {
+        return KotlinClassSnapshot(classInfo = KotlinClassInfoExternalizer.read(input))
+    }
+}
+
+object KotlinClassInfoExternalizer : DataExternalizer<KotlinClassInfo> {
+
+    override fun save(output: DataOutput, info: KotlinClassInfo) {
+        ClassIdExternalizer.save(output, info.classId)
+        output.writeInt(info.classKind.id)
+        ListExternalizer(StringExternalizer).save(output, info.classHeaderData.toList())
+        ListExternalizer(StringExternalizer).save(output, info.classHeaderStrings.toList())
+        NullableValueExternalizer(StringExternalizer).save(output, info.multifileClassName)
+        LinkedHashMapExternalizer(StringExternalizer, ConstantExternalizer).save(output, info.constantsMap)
+        LinkedHashMapExternalizer(StringExternalizer, LongExternalizer).save(output, info.inlineFunctionsMap)
+    }
+
+    override fun read(input: DataInput): KotlinClassInfo {
+        return KotlinClassInfo(
+            classId = ClassIdExternalizer.read(input),
+            classKind = KotlinClassHeader.Kind.getById(input.readInt()),
+            classHeaderData = ListExternalizer(StringExternalizer).read(input).toTypedArray(),
+            classHeaderStrings = ListExternalizer(StringExternalizer).read(input).toTypedArray(),
+            multifileClassName = NullableValueExternalizer(StringExternalizer).read(input),
+            constantsMap = LinkedHashMapExternalizer(StringExternalizer, ConstantExternalizer).read(input),
+            inlineFunctionsMap = LinkedHashMapExternalizer(StringExternalizer, LongExternalizer).read(input)
+        )
+    }
+}
+
+object ClassIdExternalizer : DataExternalizer<ClassId> {
+
+    override fun save(output: DataOutput, classId: ClassId) {
+        FqNameExternalizer.save(output, classId.packageFqName)
+        FqNameExternalizer.save(output, classId.relativeClassName)
+        output.writeBoolean(classId.isLocal)
+    }
+
+    override fun read(input: DataInput): ClassId {
+        return ClassId(
+            /* packageFqName */ FqNameExternalizer.read(input),
+            /* relativeClassName */ FqNameExternalizer.read(input),
+            /* isLocal */ input.readBoolean()
+        )
+    }
+}
+
+object FqNameExternalizer : DataExternalizer<FqName> {
+
+    override fun save(output: DataOutput, fqName: FqName) {
+        output.writeString(fqName.asString())
+    }
+
+    override fun read(input: DataInput): FqName {
+        return FqName(input.readString())
+    }
+}
+
+object JavaClassSnapshotExternalizer : DataExternalizer<JavaClassSnapshot> {
+
+    override fun save(output: DataOutput, snapshot: JavaClassSnapshot) {
+    }
+
+    override fun read(input: DataInput): JavaClassSnapshot {
+        return JavaClassSnapshot
+    }
+}
+
+interface DataSerializer<T> : DataExternalizer<T> {
+
+    fun save(file: File, value: T) {
+        return FileOutputStream(file).buffered().use {
+            it.writeValue(value)
+        }
+    }
+
+    fun load(file: File): T {
+        return FileInputStream(file).buffered().use {
+            it.readValue()
+        }
+    }
+
+    fun toByteArray(value: T): ByteArray {
+        val byteArrayOutputStream = ByteArrayOutputStream()
+        byteArrayOutputStream.buffered().use {
+            it.writeValue(value)
+        }
+        return byteArrayOutputStream.toByteArray()
+    }
+
+    fun fromByteArray(byteArray: ByteArray): T {
+        return ByteArrayInputStream(byteArray).buffered().use {
+            it.readValue()
+        }
+    }
+
+    private fun OutputStream.writeValue(value: T) {
+        DataOutputStream(this).use {
+            save(it, value)
+        }
+    }
+
+    private fun InputStream.readValue(): T {
+        return DataInputStream(this).use {
+            read(it)
+        }
+    }
+}
@@ -32,9 +32,8 @@ object ClasspathEntrySnapshotter {
 object ClassSnapshotter {

    fun snapshot(classContents: ByteArray): ClassSnapshot {
-        // TODO Add custom serialization to KotlinClassInfo then return it here. For now, use JavaClassSnapshot.
-        KotlinClassInfo.tryCreateFrom(classContents)?.let { KotlinClassSnapshot(it) }
-        return JavaClassSnapshot()
+        return KotlinClassInfo.tryCreateFrom(classContents)?.let { KotlinClassSnapshot(it) }
+            ?: JavaClassSnapshot
    }
 }

@@ -9,10 +9,10 @@ import org.gradle.api.artifacts.transform.*
 import org.gradle.api.file.FileSystemLocation
 import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.Classpath
-import org.jetbrains.kotlin.gradle.incremental.ClasspathEntrySnapshotter
 import org.jetbrains.kotlin.gradle.incremental.ClasspathEntrySnapshotSerializer
+import org.jetbrains.kotlin.gradle.incremental.ClasspathEntrySnapshotter

-/** Transform to create a snapshot ([CLASSPATH_ENTRY_SNAPSHOT_ARTIFACT_TYPE]) of a classpath entry (directory or jar). */
+/** Transform to create a snapshot of a classpath entry (directory or jar). */
@CacheableTransform
 abstract class ClasspathEntrySnapshotTransform : TransformAction<TransformParameters.None> {

@@ -25,7 +25,7 @@ abstract class ClasspathEntrySnapshotTransform : TransformAction<TransformParame
        val snapshotFile = outputs.file(CLASSPATH_ENTRY_SNAPSHOT_FILE_NAME)

        val snapshot = ClasspathEntrySnapshotter.snapshot(classpathEntry)
-        ClasspathEntrySnapshotSerializer.writeToFile(snapshot, snapshotFile)
+        ClasspathEntrySnapshotSerializer.save(snapshotFile, snapshot)
    }
 }

@@ -778,8 +778,8 @@ abstract class KotlinCompile @Inject constructor(
        val currentSnapshotFiles = classpathSnapshotProperties.classpathSnapshot.files.toList()
        val previousSnapshotFiles = getClasspathSnapshotFilesInDir(classpathSnapshotProperties.classpathSnapshotDir.get().asFile)

-        val currentSnapshot = ClasspathSnapshotSerializer.readFromFiles(currentSnapshotFiles)
-        val previousSnapshot = ClasspathSnapshotSerializer.readFromFiles(previousSnapshotFiles)
+        val currentSnapshot = ClasspathSnapshotSerializer.load(currentSnapshotFiles)
+        val previousSnapshot = ClasspathSnapshotSerializer.load(previousSnapshotFiles)

        return ClasspathChangesComputer.getChanges(currentSnapshot, previousSnapshot)
    }
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.incremental
+
+import org.junit.Assert.assertEquals
+import org.junit.Test
+
+abstract class ClasspathSnapshotSerializerTest : ClasspathSnapshotTestCommon() {
+
+    protected abstract val testSourceFile: ChangeableTestSourceFile
+
+    @Test
+    fun `test ClassSnapshotDataSerializer`() {
+        val originalSnapshot = testSourceFile.compileAndSnapshot()
+        val serializedSnapshot = ClassSnapshotDataSerializer.toByteArray(originalSnapshot)
+        val deserializedSnapshot = ClassSnapshotDataSerializer.fromByteArray(serializedSnapshot)
+
+        assertEquals(originalSnapshot.toGson(), deserializedSnapshot.toGson())
+    }
+}
+
+class KotlinClassesClasspathSnapshotSerializerTest : ClasspathSnapshotSerializerTest() {
+    override val testSourceFile = SimpleKotlinClass(tmpDir)
+}
@@ -6,7 +6,6 @@
 package org.jetbrains.kotlin.gradle.incremental

 import com.google.gson.GsonBuilder
-import org.jetbrains.kotlin.incremental.KotlinClassInfo
 import org.junit.Rule
 import org.junit.rules.TemporaryFolder
 import java.io.File
@@ -35,7 +34,7 @@ abstract class ClasspathSnapshotTestCommon {
            check(relativePath.endsWith(".class"))
        }

-        fun snapshot() = KotlinClassSnapshot(KotlinClassInfo.tryCreateFrom(asFile().readBytes())!!)
+        fun snapshot() = ClassSnapshotter.snapshot(asFile().readBytes())
    }

    open class TestSourceFile(val sourceFile: SourceFile, val tmpDir: TemporaryFolder) {
@@ -100,4 +99,4 @@ abstract class ClasspathSnapshotTestCommon {
    // Use Gson to compare objects
    private val gson by lazy { GsonBuilder().setPrettyPrinting().create() }
    protected fun Any.toGson(): String = gson.toJson(this)
-}
+}