KotlinCompile: Prepare for classpath snapshotting and diffing

This commit prepares necessary components only. The core parts of
classpath snapshotting and diffing will be implemented next.

Bug: KT-45777
Test: Existing IncrementalCompilationClasspathSnapshotJvmMultiProjectIT
      and IncrementalJavaChangeClasspathSnapshotIT

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/ClasspathChangesComputer.kt

@@ -0,0 +1,115 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.incremental
+
+import org.jetbrains.kotlin.build.report.BuildReporter
+import org.jetbrains.kotlin.build.report.ICReporter
+import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
+import org.jetbrains.kotlin.build.report.metrics.BuildMetrics
+import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
+import org.jetbrains.kotlin.build.report.metrics.BuildTime
+import org.jetbrains.kotlin.cli.common.ExitCode
+import org.jetbrains.kotlin.incremental.*
+import java.io.File
+import org.jetbrains.kotlin.gradle.incremental.ChangesCollectorResult.Success
+import org.jetbrains.kotlin.gradle.incremental.ChangesCollectorResult.Failure
+
+/** Computes [ClasspathChanges] between two [ClasspathSnapshot]s .*/
+object ClasspathChangesComputer {
+
+    fun getChanges(current: ClasspathSnapshot, previous: ClasspathSnapshot): ClasspathChanges {
+        val changesCollector = ChangesCollector()
+        return when (collectClasspathChanges(current, previous, changesCollector)) {
+            Success -> {
+                val (lookupSymbols, fqNames, _) = changesCollector.getDirtyData(emptyList(), NoOpBuildReporter)
+                ClasspathChanges.Available(lookupSymbols.toList(), fqNames.toList())
+            }
+            is Failure -> ClasspathChanges.NotAvailable.UnableToCompute
+        }
+    }
+
+    private fun collectClasspathChanges(
+        current: ClasspathSnapshot,
+        previous: ClasspathSnapshot,
+        changesCollector: ChangesCollector
+    ): ChangesCollectorResult {
+        if (current.classpathEntrySnapshots.size != previous.classpathEntrySnapshots.size) {
+            return Failure.AddedRemovedClasspathEntries
+        }
+
+        for (index in current.classpathEntrySnapshots.indices) {
+            val result = collectClasspathEntryChanges(
+                current.classpathEntrySnapshots[index],
+                previous.classpathEntrySnapshots[index],
+                changesCollector
+            )
+            if (result is Failure) {
+                return result
+            }
+        }
+        return Success
+    }
+
+    private fun collectClasspathEntryChanges(
+        current: ClasspathEntrySnapshot,
+        previous: ClasspathEntrySnapshot,
+        changesCollector: ChangesCollector
+    ): ChangesCollectorResult {
+        if (current.classSnapshots.size != previous.classSnapshots.size) {
+            return Failure.AddedRemovedClasses
+        }
+
+        for (key in current.classSnapshots.keys) {
+            val currentSnapshot = current.classSnapshots[key]!!
+            val previousSnapshot = previous.classSnapshots[key] ?: return Failure.AddedRemovedClasses
+            val result = collectClassChanges(currentSnapshot, previousSnapshot, changesCollector)
+            if (result is Failure) {
+                return result
+            }
+        }
+        return Success
+    }
+
+    private fun collectClassChanges(
+        current: ClassSnapshot,
+        previous: ClassSnapshot,
+        changesCollector: ChangesCollector
+    ): ChangesCollectorResult {
+        return Failure.NotYetImplemented
+    }
+}
+
+private sealed class ChangesCollectorResult {
+
+    object Success : ChangesCollectorResult()
+
+    sealed class Failure : ChangesCollectorResult() {
+        // TODO: Handle these cases
+        object AddedRemovedClasspathEntries : Failure()
+        object AddedRemovedClasses : Failure()
+        object NotYetImplemented : Failure()
+    }
+}
+
+private object NoOpBuildReporter : BuildReporter(NoOpICReporter, NoOpBuildMetricsReporter) {
+
+    object NoOpICReporter : ICReporter {
+        override fun report(message: () -> String) {}
+        override fun reportVerbose(message: () -> String) {}
+        override fun reportCompileIteration(incremental: Boolean, sourceFiles: Collection<File>, exitCode: ExitCode) {}
+        override fun reportMarkDirtyClass(affectedFiles: Iterable<File>, classFqName: String) {}
+        override fun reportMarkDirtyMember(affectedFiles: Iterable<File>, scope: String, name: String) {}
+        override fun reportMarkDirty(affectedFiles: Iterable<File>, reason: String) {}
+    }
+
+    object NoOpBuildMetricsReporter : BuildMetricsReporter {
+        override fun startMeasure(metric: BuildTime, startNs: Long) {}
+        override fun endMeasure(metric: BuildTime, endNs: Long) {}
+        override fun addAttribute(attribute: BuildAttribute) {}
+        override fun getMetrics(): BuildMetrics = BuildMetrics()
+        override fun addMetrics(metrics: BuildMetrics?) {}
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/ClasspathSnapshot.kt

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.incremental
+
+import java.io.*
+
+/** Snapshot of a classpath. It consists of a list of [ClasspathEntrySnapshot]s. */
+class ClasspathSnapshot(val classpathEntrySnapshots: List<ClasspathEntrySnapshot>)
+
+/** Snapshot of a classpath entry (directory or jar). It consists of a list of [ClassSnapshot]s. */
+class ClasspathEntrySnapshot(
+
+    /**
+     * Maps (Unix-like) relative paths of classes to their snapshots. The paths are relative to the containing classpath entry (directory or
+     * jar).
+     */
+    val classSnapshots: LinkedHashMap<String, ClassSnapshot>
+) : Serializable {
+
+    companion object {
+        private const val serialVersionUID = 0L
+    }
+}
+
+/**
+ * Snapshot of a class. It contains information to compute the source files that need to be recompiled during an incremental run of the
+ * `KotlinCompile` task.
+ */
+class ClassSnapshot : Serializable {
+
+    // TODO WORK-IN-PROGRESS
+
+    companion object {
+        private const val serialVersionUID = 0L
+    }
+}
+
+/** Utility to read/write a [ClasspathSnapshot] from/to a file. */
+object ClasspathSnapshotSerializer {
+
+    fun readFromFiles(classpathEntrySnapshotFiles: List<File>): ClasspathSnapshot {
+        return ClasspathSnapshot(classpathEntrySnapshotFiles.map { ClasspathEntrySnapshotSerializer.readFromFile(it) })
+    }
+}
+
+/** Utility to read/write a [ClasspathEntrySnapshot] from/to a file. */
+object ClasspathEntrySnapshotSerializer {
+
+    fun readFromFile(classpathEntrySnapshotFile: File): ClasspathEntrySnapshot {
+        check(classpathEntrySnapshotFile.isFile) { "`${classpathEntrySnapshotFile.path}` does not exist (or is a directory)." }
+        return ObjectInputStream(FileInputStream(classpathEntrySnapshotFile).buffered()).use {
+            it.readObject() as ClasspathEntrySnapshot
+        }
+    }
+
+    fun writeToFile(classpathEntrySnapshot: ClasspathEntrySnapshot, classpathEntrySnapshotFile: File) {
+        check(classpathEntrySnapshotFile.parentFile.exists()) { "Parent dir of `${classpathEntrySnapshotFile.path}` does not exist." }
+        ObjectOutputStream(FileOutputStream(classpathEntrySnapshotFile).buffered()).use {
+            it.writeObject(classpathEntrySnapshot)
+        }
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/ClasspathSnapshotter.kt

@@ -0,0 +1,117 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.incremental
+
+import org.jetbrains.kotlin.gradle.incremental.ClasspathEntryContentsReader.Companion.DEFAULT_CLASS_FILTER
+import java.io.File
+import java.util.zip.ZipInputStream
+
+/** Computes a [ClasspathEntrySnapshot] of a classpath entry (directory or jar). */
+@Suppress("SpellCheckingInspection")
+object ClasspathEntrySnapshotter {
+
+    fun snapshot(classpathEntry: File): ClasspathEntrySnapshot {
+        val pathsToContents: LinkedHashMap<String, ByteArray> =
+            ClasspathEntryContentsReader.from(classpathEntry).readContents(DEFAULT_CLASS_FILTER)
+
+        val pathsToSnapshots = LinkedHashMap<String, ClassSnapshot>()
+        pathsToContents.mapValuesTo(pathsToSnapshots) { (invariantSeparatorsRelativePath, classContents) ->
+            ClassSnapshotter.snapshot(invariantSeparatorsRelativePath, classContents)
+        }
+
+        return ClasspathEntrySnapshot(pathsToSnapshots)
+    }
+}
+
+/** Computes a [ClassSnapshot] of a class. */
+@Suppress("SpellCheckingInspection")
+object ClassSnapshotter {
+
+    fun snapshot(invariantSeparatorsRelativePath: String, classContents: ByteArray): ClassSnapshot {
+        // TODO WORK-IN-PROGRESS
+        return ClassSnapshot()
+    }
+}
+
+/** Utility to read the contents of a classpath entry (directory or jar). */
+sealed class ClasspathEntryContentsReader {
+
+    companion object {
+
+        val DEFAULT_CLASS_FILTER = { invariantSeparatorsRelativePath: String, isDirectory: Boolean ->
+            !isDirectory
+                    && invariantSeparatorsRelativePath.endsWith(".class", ignoreCase = true)
+                    && !invariantSeparatorsRelativePath.endsWith("module-info.class", ignoreCase = true)
+                    && !invariantSeparatorsRelativePath.startsWith("meta-inf", ignoreCase = true)
+        }
+
+        /** Creates a [ClasspathEntryContentsReader] for the given classpath entry (directory or jar). */
+        fun from(classpathEntry: File): ClasspathEntryContentsReader {
+            return if (classpathEntry.isDirectory) {
+                DirectoryContentsReader(classpathEntry)
+            } else {
+                JarContentsReader(classpathEntry)
+            }
+        }
+    }
+
+    /**
+     * Returns a map from (Unix-like) relative paths of classes to their contents. The paths are relative to the containing classpath entry
+     * (directory or jar).
+     *
+     * The map entries need to satisfy the given filter.
+     *
+     * The map entries are sorted based on their (Unix-like) relative paths (to ensure deterministic results across filesystems).
+     *
+     * Note: If a jar has duplicate entries, only one of them will be used (there is no guarantee which one will be used).
+     */
+    abstract fun readContents(filter: ((invariantSeparatorsRelativePath: String, isDirectory: Boolean) -> Boolean)? = null):
+            LinkedHashMap<String, ByteArray>
+}
+
+/** Utility to read the contents of a directory. */
+class DirectoryContentsReader(private val directory: File) : ClasspathEntryContentsReader() {
+
+    init {
+        check(directory.isDirectory)
+    }
+
+    override fun readContents(
+        filter: ((invariantSeparatorsRelativePath: String, isDirectory: Boolean) -> Boolean)?
+    ): LinkedHashMap<String, ByteArray> {
+        val relativePathsToContents: MutableList<Pair<String, ByteArray>> = mutableListOf()
+        directory.walk().forEach {
+            val invariantSeparatorsRelativePath = it.relativeTo(directory).invariantSeparatorsPath
+            if (filter == null || filter(invariantSeparatorsRelativePath, it.isDirectory)) {
+                relativePathsToContents.add(invariantSeparatorsRelativePath to it.readBytes())
+            }
+        }
+        return relativePathsToContents.sortedBy { it.first }.toMap(LinkedHashMap())
+    }
+}
+
+/** Utility to read the contents of a jar. */
+class JarContentsReader(private val jarFile: File) : ClasspathEntryContentsReader() {
+
+    init {
+        check(jarFile.path.endsWith(".jar", ignoreCase = true))
+    }
+
+    override fun readContents(
+        filter: ((invariantSeparatorsRelativePath: String, isDirectory: Boolean) -> Boolean)?
+    ): LinkedHashMap<String, ByteArray> {
+        val relativePathsToContents: MutableList<Pair<String, ByteArray>> = mutableListOf()
+        ZipInputStream(jarFile.inputStream().buffered()).use { zipInputStream ->
+            while (true) {
+                val entry = zipInputStream.nextEntry ?: break
+                if (filter == null || filter(entry.name, entry.isDirectory)) {
+                    relativePathsToContents.add(entry.name to zipInputStream.readBytes())
+                }
+            }
+        }
+        return relativePathsToContents.sortedBy { it.first }.toMap(LinkedHashMap())
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt

@@ -9,7 +9,8 @@ import org.gradle.api.artifacts.transform.*
 import org.gradle.api.file.FileSystemLocation
 import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.Classpath
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile.Companion.CLASSPATH_ENTRY_SNAPSHOT_ARTIFACT_TYPE
+import org.jetbrains.kotlin.gradle.incremental.ClasspathEntrySnapshotter
+import org.jetbrains.kotlin.gradle.incremental.ClasspathEntrySnapshotSerializer
 
 /** Transform to create a snapshot ([CLASSPATH_ENTRY_SNAPSHOT_ARTIFACT_TYPE]) of a classpath entry (directory or jar). */
 @CacheableTransform
@@ -23,9 +24,8 @@ abstract class ClasspathEntrySnapshotTransform : TransformAction<TransformParame
         val classpathEntry = inputArtifact.get().asFile
         val snapshotFile = outputs.file(CLASSPATH_ENTRY_SNAPSHOT_FILE_NAME)
 
-        // TODO WORK-IN-PROGRESS
+        val snapshot = ClasspathEntrySnapshotter.snapshot(classpathEntry)
-        val snapshot = "[Place holder - actual snapshot will be inserted in an upcoming change]"
+        ClasspathEntrySnapshotSerializer.writeToFile(snapshot, snapshotFile)
-        snapshotFile.writeText(snapshot)
     }
 }
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt

@@ -34,9 +34,8 @@ import org.jetbrains.kotlin.compilerRunner.*
 import org.jetbrains.kotlin.compilerRunner.GradleCompilerRunner.Companion.normalizeForFlagFile
 import org.jetbrains.kotlin.daemon.common.MultiModuleICSettings
 import org.jetbrains.kotlin.gradle.dsl.*
+import org.jetbrains.kotlin.gradle.incremental.*
 import org.jetbrains.kotlin.gradle.incremental.ChangedFiles
-import org.jetbrains.kotlin.gradle.incremental.IncrementalModuleInfoBuildService
-import org.jetbrains.kotlin.gradle.incremental.IncrementalModuleInfoProvider
 import org.jetbrains.kotlin.gradle.internal.*
 import org.jetbrains.kotlin.gradle.internal.tasks.TaskConfigurator
 import org.jetbrains.kotlin.gradle.internal.tasks.TaskWithLocalState
@@ -730,8 +729,10 @@ abstract class KotlinCompile @Inject constructor(
         val currentSnapshotFiles = classpathSnapshotProperties.classpathSnapshot.files.toList()
         val previousSnapshotFiles = getClasspathSnapshotFilesInDir(classpathSnapshotProperties.classpathSnapshotDir.get().asFile)
 
-        // TODO WORK-IN-PROGRESS
+        val currentSnapshot = ClasspathSnapshotSerializer.readFromFiles(currentSnapshotFiles)
-        return ClasspathChanges.NotAvailable.UnableToCompute
+        val previousSnapshot = ClasspathSnapshotSerializer.readFromFiles(previousSnapshotFiles)
+
+        return ClasspathChangesComputer.getChanges(currentSnapshot, previousSnapshot)
     }
 
     /**