From ecf60013655d66e946268c8285a31beafbf8abaf Mon Sep 17 00:00:00 2001 From: Hung Nguyen Date: Wed, 26 May 2021 21:05:22 +0100 Subject: [PATCH] KotlinCompile: Prepare for classpath snapshotting and diffing This commit prepares necessary components only. The core parts of classpath snapshotting and diffing will be implemented next. Bug: KT-45777 Test: Existing IncrementalCompilationClasspathSnapshotJvmMultiProjectIT and IncrementalJavaChangeClasspathSnapshotIT --- .../incremental/ClasspathChangesComputer.kt | 115 +++++++++++++++++ .../gradle/incremental/ClasspathSnapshot.kt | 65 ++++++++++ .../incremental/ClasspathSnapshotter.kt | 117 ++++++++++++++++++ .../ClasspathEntrySnapshotTransform.kt | 8 +- .../jetbrains/kotlin/gradle/tasks/Tasks.kt | 9 +- 5 files changed, 306 insertions(+), 8 deletions(-) create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/ClasspathChangesComputer.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/ClasspathSnapshot.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/ClasspathSnapshotter.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/ClasspathChangesComputer.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/ClasspathChangesComputer.kt new file mode 100644 index 00000000000..6a9328c437e --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/ClasspathChangesComputer.kt @@ -0,0 +1,115 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.incremental + +import org.jetbrains.kotlin.build.report.BuildReporter +import org.jetbrains.kotlin.build.report.ICReporter +import org.jetbrains.kotlin.build.report.metrics.BuildAttribute +import org.jetbrains.kotlin.build.report.metrics.BuildMetrics +import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter +import org.jetbrains.kotlin.build.report.metrics.BuildTime +import org.jetbrains.kotlin.cli.common.ExitCode +import org.jetbrains.kotlin.incremental.* +import java.io.File +import org.jetbrains.kotlin.gradle.incremental.ChangesCollectorResult.Success +import org.jetbrains.kotlin.gradle.incremental.ChangesCollectorResult.Failure + +/** Computes [ClasspathChanges] between two [ClasspathSnapshot]s .*/ +object ClasspathChangesComputer { + + fun getChanges(current: ClasspathSnapshot, previous: ClasspathSnapshot): ClasspathChanges { + val changesCollector = ChangesCollector() + return when (collectClasspathChanges(current, previous, changesCollector)) { + Success -> { + val (lookupSymbols, fqNames, _) = changesCollector.getDirtyData(emptyList(), NoOpBuildReporter) + ClasspathChanges.Available(lookupSymbols.toList(), fqNames.toList()) + } + is Failure -> ClasspathChanges.NotAvailable.UnableToCompute + } + } + + private fun collectClasspathChanges( + current: ClasspathSnapshot, + previous: ClasspathSnapshot, + changesCollector: ChangesCollector + ): ChangesCollectorResult { + if (current.classpathEntrySnapshots.size != previous.classpathEntrySnapshots.size) { + return Failure.AddedRemovedClasspathEntries + } + + for (index in current.classpathEntrySnapshots.indices) { + val result = collectClasspathEntryChanges( + current.classpathEntrySnapshots[index], + previous.classpathEntrySnapshots[index], + changesCollector + ) + if (result is Failure) { + return result + } + } + return Success + } + + private fun collectClasspathEntryChanges( + current: ClasspathEntrySnapshot, + previous: ClasspathEntrySnapshot, + changesCollector: ChangesCollector + ): ChangesCollectorResult { + if (current.classSnapshots.size != previous.classSnapshots.size) { + return Failure.AddedRemovedClasses + } + + for (key in current.classSnapshots.keys) { + val currentSnapshot = current.classSnapshots[key]!! + val previousSnapshot = previous.classSnapshots[key] ?: return Failure.AddedRemovedClasses + val result = collectClassChanges(currentSnapshot, previousSnapshot, changesCollector) + if (result is Failure) { + return result + } + } + return Success + } + + private fun collectClassChanges( + current: ClassSnapshot, + previous: ClassSnapshot, + changesCollector: ChangesCollector + ): ChangesCollectorResult { + return Failure.NotYetImplemented + } +} + +private sealed class ChangesCollectorResult { + + object Success : ChangesCollectorResult() + + sealed class Failure : ChangesCollectorResult() { + // TODO: Handle these cases + object AddedRemovedClasspathEntries : Failure() + object AddedRemovedClasses : Failure() + object NotYetImplemented : Failure() + } +} + +private object NoOpBuildReporter : BuildReporter(NoOpICReporter, NoOpBuildMetricsReporter) { + + object NoOpICReporter : ICReporter { + override fun report(message: () -> String) {} + override fun reportVerbose(message: () -> String) {} + override fun reportCompileIteration(incremental: Boolean, sourceFiles: Collection, exitCode: ExitCode) {} + override fun reportMarkDirtyClass(affectedFiles: Iterable, classFqName: String) {} + override fun reportMarkDirtyMember(affectedFiles: Iterable, scope: String, name: String) {} + override fun reportMarkDirty(affectedFiles: Iterable, reason: String) {} + } + + object NoOpBuildMetricsReporter : BuildMetricsReporter { + override fun startMeasure(metric: BuildTime, startNs: Long) {} + override fun endMeasure(metric: BuildTime, endNs: Long) {} + override fun addAttribute(attribute: BuildAttribute) {} + override fun getMetrics(): BuildMetrics = BuildMetrics() + override fun addMetrics(metrics: BuildMetrics?) {} + } +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/ClasspathSnapshot.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/ClasspathSnapshot.kt new file mode 100644 index 00000000000..2c3bef05d80 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/ClasspathSnapshot.kt @@ -0,0 +1,65 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.incremental + +import java.io.* + +/** Snapshot of a classpath. It consists of a list of [ClasspathEntrySnapshot]s. */ +class ClasspathSnapshot(val classpathEntrySnapshots: List) + +/** Snapshot of a classpath entry (directory or jar). It consists of a list of [ClassSnapshot]s. */ +class ClasspathEntrySnapshot( + + /** + * Maps (Unix-like) relative paths of classes to their snapshots. The paths are relative to the containing classpath entry (directory or + * jar). + */ + val classSnapshots: LinkedHashMap +) : Serializable { + + companion object { + private const val serialVersionUID = 0L + } +} + +/** + * Snapshot of a class. It contains information to compute the source files that need to be recompiled during an incremental run of the + * `KotlinCompile` task. + */ +class ClassSnapshot : Serializable { + + // TODO WORK-IN-PROGRESS + + companion object { + private const val serialVersionUID = 0L + } +} + +/** Utility to read/write a [ClasspathSnapshot] from/to a file. */ +object ClasspathSnapshotSerializer { + + fun readFromFiles(classpathEntrySnapshotFiles: List): ClasspathSnapshot { + return ClasspathSnapshot(classpathEntrySnapshotFiles.map { ClasspathEntrySnapshotSerializer.readFromFile(it) }) + } +} + +/** Utility to read/write a [ClasspathEntrySnapshot] from/to a file. */ +object ClasspathEntrySnapshotSerializer { + + fun readFromFile(classpathEntrySnapshotFile: File): ClasspathEntrySnapshot { + check(classpathEntrySnapshotFile.isFile) { "`${classpathEntrySnapshotFile.path}` does not exist (or is a directory)." } + return ObjectInputStream(FileInputStream(classpathEntrySnapshotFile).buffered()).use { + it.readObject() as ClasspathEntrySnapshot + } + } + + fun writeToFile(classpathEntrySnapshot: ClasspathEntrySnapshot, classpathEntrySnapshotFile: File) { + check(classpathEntrySnapshotFile.parentFile.exists()) { "Parent dir of `${classpathEntrySnapshotFile.path}` does not exist." } + ObjectOutputStream(FileOutputStream(classpathEntrySnapshotFile).buffered()).use { + it.writeObject(classpathEntrySnapshot) + } + } +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/ClasspathSnapshotter.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/ClasspathSnapshotter.kt new file mode 100644 index 00000000000..437b6686f68 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/incremental/ClasspathSnapshotter.kt @@ -0,0 +1,117 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.incremental + +import org.jetbrains.kotlin.gradle.incremental.ClasspathEntryContentsReader.Companion.DEFAULT_CLASS_FILTER +import java.io.File +import java.util.zip.ZipInputStream + +/** Computes a [ClasspathEntrySnapshot] of a classpath entry (directory or jar). */ +@Suppress("SpellCheckingInspection") +object ClasspathEntrySnapshotter { + + fun snapshot(classpathEntry: File): ClasspathEntrySnapshot { + val pathsToContents: LinkedHashMap = + ClasspathEntryContentsReader.from(classpathEntry).readContents(DEFAULT_CLASS_FILTER) + + val pathsToSnapshots = LinkedHashMap() + pathsToContents.mapValuesTo(pathsToSnapshots) { (invariantSeparatorsRelativePath, classContents) -> + ClassSnapshotter.snapshot(invariantSeparatorsRelativePath, classContents) + } + + return ClasspathEntrySnapshot(pathsToSnapshots) + } +} + +/** Computes a [ClassSnapshot] of a class. */ +@Suppress("SpellCheckingInspection") +object ClassSnapshotter { + + fun snapshot(invariantSeparatorsRelativePath: String, classContents: ByteArray): ClassSnapshot { + // TODO WORK-IN-PROGRESS + return ClassSnapshot() + } +} + +/** Utility to read the contents of a classpath entry (directory or jar). */ +sealed class ClasspathEntryContentsReader { + + companion object { + + val DEFAULT_CLASS_FILTER = { invariantSeparatorsRelativePath: String, isDirectory: Boolean -> + !isDirectory + && invariantSeparatorsRelativePath.endsWith(".class", ignoreCase = true) + && !invariantSeparatorsRelativePath.endsWith("module-info.class", ignoreCase = true) + && !invariantSeparatorsRelativePath.startsWith("meta-inf", ignoreCase = true) + } + + /** Creates a [ClasspathEntryContentsReader] for the given classpath entry (directory or jar). */ + fun from(classpathEntry: File): ClasspathEntryContentsReader { + return if (classpathEntry.isDirectory) { + DirectoryContentsReader(classpathEntry) + } else { + JarContentsReader(classpathEntry) + } + } + } + + /** + * Returns a map from (Unix-like) relative paths of classes to their contents. The paths are relative to the containing classpath entry + * (directory or jar). + * + * The map entries need to satisfy the given filter. + * + * The map entries are sorted based on their (Unix-like) relative paths (to ensure deterministic results across filesystems). + * + * Note: If a jar has duplicate entries, only one of them will be used (there is no guarantee which one will be used). + */ + abstract fun readContents(filter: ((invariantSeparatorsRelativePath: String, isDirectory: Boolean) -> Boolean)? = null): + LinkedHashMap +} + +/** Utility to read the contents of a directory. */ +class DirectoryContentsReader(private val directory: File) : ClasspathEntryContentsReader() { + + init { + check(directory.isDirectory) + } + + override fun readContents( + filter: ((invariantSeparatorsRelativePath: String, isDirectory: Boolean) -> Boolean)? + ): LinkedHashMap { + val relativePathsToContents: MutableList> = mutableListOf() + directory.walk().forEach { + val invariantSeparatorsRelativePath = it.relativeTo(directory).invariantSeparatorsPath + if (filter == null || filter(invariantSeparatorsRelativePath, it.isDirectory)) { + relativePathsToContents.add(invariantSeparatorsRelativePath to it.readBytes()) + } + } + return relativePathsToContents.sortedBy { it.first }.toMap(LinkedHashMap()) + } +} + +/** Utility to read the contents of a jar. */ +class JarContentsReader(private val jarFile: File) : ClasspathEntryContentsReader() { + + init { + check(jarFile.path.endsWith(".jar", ignoreCase = true)) + } + + override fun readContents( + filter: ((invariantSeparatorsRelativePath: String, isDirectory: Boolean) -> Boolean)? + ): LinkedHashMap { + val relativePathsToContents: MutableList> = mutableListOf() + ZipInputStream(jarFile.inputStream().buffered()).use { zipInputStream -> + while (true) { + val entry = zipInputStream.nextEntry ?: break + if (filter == null || filter(entry.name, entry.isDirectory)) { + relativePathsToContents.add(entry.name to zipInputStream.readBytes()) + } + } + } + return relativePathsToContents.sortedBy { it.first }.toMap(LinkedHashMap()) + } +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt index 7ee258de00c..5998f843eff 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt @@ -9,7 +9,8 @@ import org.gradle.api.artifacts.transform.* import org.gradle.api.file.FileSystemLocation import org.gradle.api.provider.Provider import org.gradle.api.tasks.Classpath -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile.Companion.CLASSPATH_ENTRY_SNAPSHOT_ARTIFACT_TYPE +import org.jetbrains.kotlin.gradle.incremental.ClasspathEntrySnapshotter +import org.jetbrains.kotlin.gradle.incremental.ClasspathEntrySnapshotSerializer /** Transform to create a snapshot ([CLASSPATH_ENTRY_SNAPSHOT_ARTIFACT_TYPE]) of a classpath entry (directory or jar). */ @CacheableTransform @@ -23,9 +24,8 @@ abstract class ClasspathEntrySnapshotTransform : TransformAction