From 83c53b7ce390e6cb7e45186b05fef45a5984003e Mon Sep 17 00:00:00 2001 From: "Alexander.Likhachev" Date: Mon, 31 Jul 2023 13:52:38 +0200 Subject: [PATCH] [BT] Implement classpath snapshotting in the Build Tools API #KT-57565 In Progress --- .../api/kotlin-build-tools-api.api | 11 ++- .../buildtools/api/CompilationService.kt | 3 +- .../api/jvm/ClasspathEntrySnapshot.kt | 18 ++++- .../kotlin-build-tools-impl/build.gradle.kts | 7 +- .../buildtools/internal/ClassSnapshotImpl.kt | 13 ++++ .../internal/ClasspathEntrySnapshotImpl.kt | 31 ++++++++ .../internal/CompilationServiceImpl.kt | 7 +- .../btapi/BuildToolsApiCompilationWork.kt | 2 +- ...ToolsApiClasspathEntrySnapshotTransform.kt | 76 +++++++++++++++++++ .../configuration/KotlinCompileConfig.kt | 58 +++++++++++++- 10 files changed, 213 insertions(+), 13 deletions(-) create mode 100644 compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/ClassSnapshotImpl.kt create mode 100644 compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/ClasspathEntrySnapshotImpl.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/BuildToolsApiClasspathEntrySnapshotTransform.kt diff --git a/compiler/build-tools/kotlin-build-tools-api/api/kotlin-build-tools-api.api b/compiler/build-tools/kotlin-build-tools-api/api/kotlin-build-tools-api.api index c19de666b2b..391359f453d 100644 --- a/compiler/build-tools/kotlin-build-tools-api/api/kotlin-build-tools-api.api +++ b/compiler/build-tools/kotlin-build-tools-api/api/kotlin-build-tools-api.api @@ -9,7 +9,7 @@ public final class org/jetbrains/kotlin/buildtools/api/CompilationResult : java/ public abstract interface class org/jetbrains/kotlin/buildtools/api/CompilationService { public static final field Companion Lorg/jetbrains/kotlin/buildtools/api/CompilationService$Companion; - public abstract fun calculateClasspathSnapshot (Ljava/io/File;)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathEntrySnapshot; + public abstract fun calculateClasspathSnapshot (Ljava/io/File;Lorg/jetbrains/kotlin/buildtools/api/jvm/ClassSnapshotGranularity;)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathEntrySnapshot; public abstract fun compileJvm (Lorg/jetbrains/kotlin/buildtools/api/ProjectId;Lorg/jetbrains/kotlin/buildtools/api/CompilerExecutionStrategyConfiguration;Lorg/jetbrains/kotlin/buildtools/api/jvm/JvmCompilationConfiguration;Ljava/util/List;Ljava/util/List;)Lorg/jetbrains/kotlin/buildtools/api/CompilationResult; public abstract fun finishProjectCompilation (Lorg/jetbrains/kotlin/buildtools/api/ProjectId;)V public static fun loadImplementation (Ljava/lang/ClassLoader;)Lorg/jetbrains/kotlin/buildtools/api/CompilationService; @@ -82,6 +82,10 @@ public final class org/jetbrains/kotlin/buildtools/api/SourcesChanges$Unknown : public static final field INSTANCE Lorg/jetbrains/kotlin/buildtools/api/SourcesChanges$Unknown; } +public abstract interface class org/jetbrains/kotlin/buildtools/api/jvm/AccessibleClassSnapshot : org/jetbrains/kotlin/buildtools/api/jvm/ClassSnapshot { + public abstract fun getClassAbiHash ()J +} + public abstract interface class org/jetbrains/kotlin/buildtools/api/jvm/ClassSnapshot { } @@ -93,7 +97,7 @@ public final class org/jetbrains/kotlin/buildtools/api/jvm/ClassSnapshotGranular } public abstract interface class org/jetbrains/kotlin/buildtools/api/jvm/ClasspathEntrySnapshot { - public abstract fun getClassSnapshots ()Ljava/util/LinkedHashMap; + public abstract fun getClassSnapshots ()Ljava/util/Map; public abstract fun saveSnapshot (Ljava/io/File;)V } @@ -115,6 +119,9 @@ public abstract interface class org/jetbrains/kotlin/buildtools/api/jvm/Classpat public abstract fun useProjectDir (Ljava/io/File;)Lorg/jetbrains/kotlin/buildtools/api/jvm/ClasspathSnapshotBasedIncrementalJvmCompilationConfiguration; } +public abstract interface class org/jetbrains/kotlin/buildtools/api/jvm/InaccessibleClassSnapshot : org/jetbrains/kotlin/buildtools/api/jvm/ClassSnapshot { +} + public abstract interface class org/jetbrains/kotlin/buildtools/api/jvm/IncrementalCompilationApproachParameters { } diff --git a/compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/CompilationService.kt b/compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/CompilationService.kt index 231be161bf8..11b86ea4df2 100644 --- a/compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/CompilationService.kt +++ b/compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/CompilationService.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.buildtools.api +import org.jetbrains.kotlin.buildtools.api.jvm.ClassSnapshotGranularity import org.jetbrains.kotlin.buildtools.api.jvm.ClasspathEntrySnapshot import org.jetbrains.kotlin.buildtools.api.jvm.JvmCompilationConfiguration import java.io.File @@ -28,7 +29,7 @@ public interface CompilationService { /** * TODO KT-57565 */ - public fun calculateClasspathSnapshot(classpathEntry: File): ClasspathEntrySnapshot + public fun calculateClasspathSnapshot(classpathEntry: File, granularity: ClassSnapshotGranularity): ClasspathEntrySnapshot /** * Provides a default [CompilerExecutionStrategyConfiguration] allowing to use it as is or customizing for specific requirements. diff --git a/compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/jvm/ClasspathEntrySnapshot.kt b/compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/jvm/ClasspathEntrySnapshot.kt index 14a3d73529b..d979b0c1281 100644 --- a/compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/jvm/ClasspathEntrySnapshot.kt +++ b/compiler/build-tools/kotlin-build-tools-api/src/main/kotlin/org/jetbrains/kotlin/buildtools/api/jvm/ClasspathEntrySnapshot.kt @@ -15,7 +15,7 @@ import java.io.File */ @ExperimentalBuildToolsApi public interface ClasspathEntrySnapshot { - public val classSnapshots: LinkedHashMap + public val classSnapshots: Map public fun saveSnapshot(path: File) } @@ -26,6 +26,18 @@ public interface ClasspathEntrySnapshot { * This interface is not intended to be implemented by the API consumers. */ @ExperimentalBuildToolsApi -public interface ClassSnapshot { - // ... TODO: KT-57565, it will expose some part of org.jetbrains.kotlin.incremental.classpathDiff.ClassSnapshot hierarchy +public sealed interface ClassSnapshot + +/** + * [ClassSnapshot] of an inaccessible class. + * + * A class is inaccessible if it can't be referenced from other source files (and therefore any changes in an inaccessible class will not + * require recompilation of other source files). + */ +@ExperimentalBuildToolsApi +public interface InaccessibleClassSnapshot : ClassSnapshot + +@ExperimentalBuildToolsApi +public interface AccessibleClassSnapshot : ClassSnapshot { + public val classAbiHash: Long } \ No newline at end of file diff --git a/compiler/build-tools/kotlin-build-tools-impl/build.gradle.kts b/compiler/build-tools/kotlin-build-tools-impl/build.gradle.kts index fccb2dc949b..c06ebadd866 100644 --- a/compiler/build-tools/kotlin-build-tools-impl/build.gradle.kts +++ b/compiler/build-tools/kotlin-build-tools-impl/build.gradle.kts @@ -13,13 +13,18 @@ dependencies { compileOnly(project(":kotlin-daemon-client")) compileOnly(project(":compiler:incremental-compilation-impl")) compileOnly(project(":kotlin-compiler-runner-unshaded")) + compileOnly(intellijCore()) runtimeOnly(project(":kotlin-compiler-embeddable")) runtimeOnly(project(":kotlin-compiler-runner")) } publish() -standardPublicJars() +runtimeJar(rewriteDefaultJarDepsToShadedCompiler { + from(mainSourceSet.output) +}) +sourcesJar() +javadocJar() kotlin { compilerOptions { diff --git a/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/ClassSnapshotImpl.kt b/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/ClassSnapshotImpl.kt new file mode 100644 index 00000000000..a9ea709ea46 --- /dev/null +++ b/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/ClassSnapshotImpl.kt @@ -0,0 +1,13 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.buildtools.internal + +import org.jetbrains.kotlin.buildtools.api.jvm.AccessibleClassSnapshot +import org.jetbrains.kotlin.buildtools.api.jvm.InaccessibleClassSnapshot + +class AccessibleClassSnapshotImpl(override val classAbiHash: Long) : AccessibleClassSnapshot + +object InaccessibleClassSnapshotImpl : InaccessibleClassSnapshot diff --git a/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/ClasspathEntrySnapshotImpl.kt b/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/ClasspathEntrySnapshotImpl.kt new file mode 100644 index 00000000000..9242a2cbc9d --- /dev/null +++ b/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/ClasspathEntrySnapshotImpl.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.buildtools.internal + +import org.jetbrains.kotlin.buildtools.api.jvm.ClassSnapshot +import org.jetbrains.kotlin.buildtools.api.jvm.ClasspathEntrySnapshot +import org.jetbrains.kotlin.incremental.classpathDiff.AccessibleClassSnapshot +import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathEntrySnapshotExternalizer +import org.jetbrains.kotlin.incremental.classpathDiff.InaccessibleClassSnapshot +import org.jetbrains.kotlin.incremental.storage.saveToFile +import java.io.File + +class ClasspathEntrySnapshotImpl( + private val origin: org.jetbrains.kotlin.incremental.classpathDiff.ClasspathEntrySnapshot, +) : + ClasspathEntrySnapshot { + override val classSnapshots: Map + get() = origin.classSnapshots.mapValues { + when (val snapshot = it.value) { + is AccessibleClassSnapshot -> AccessibleClassSnapshotImpl(snapshot.classAbiHash) + is InaccessibleClassSnapshot -> InaccessibleClassSnapshotImpl + } + } + + override fun saveSnapshot(path: File) { + ClasspathEntrySnapshotExternalizer.saveToFile(path, origin) + } +} \ No newline at end of file diff --git a/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/CompilationServiceImpl.kt b/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/CompilationServiceImpl.kt index 690fb2d871e..fc20a28138a 100644 --- a/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/CompilationServiceImpl.kt +++ b/compiler/build-tools/kotlin-build-tools-impl/src/main/kotlin/org/jetbrains/kotlin/buildtools/internal/CompilationServiceImpl.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.buildtools.internal +import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter import org.jetbrains.kotlin.buildtools.api.* import org.jetbrains.kotlin.buildtools.api.jvm.* import org.jetbrains.kotlin.cli.common.ExitCode @@ -18,6 +19,7 @@ import org.jetbrains.kotlin.daemon.client.BasicCompilerServicesWithResultsFacade import org.jetbrains.kotlin.daemon.common.CompilerId import org.jetbrains.kotlin.daemon.common.configureDaemonJVMOptions import org.jetbrains.kotlin.daemon.common.filterExtractProps +import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathEntrySnapshotter import java.io.File import java.net.URLClassLoader import java.util.concurrent.ConcurrentHashMap @@ -36,9 +38,8 @@ private fun getCurrentClasspath() = (CompilationServiceImpl::class.java.classLoa internal object CompilationServiceImpl : CompilationService { private val buildIdToSessionFlagFile: MutableMap = ConcurrentHashMap() - override fun calculateClasspathSnapshot(classpathEntry: File): ClasspathEntrySnapshot { - TODO("Calculating classpath snapshots via the Build Tools API is not yet implemented: KT-57565") - } + override fun calculateClasspathSnapshot(classpathEntry: File, granularity: ClassSnapshotGranularity) = + ClasspathEntrySnapshotImpl(ClasspathEntrySnapshotter.snapshot(classpathEntry, granularity, DoNothingBuildMetricsReporter)) override fun makeCompilerExecutionStrategyConfiguration() = CompilerExecutionStrategyConfigurationImpl() diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt index 0d976081d03..1fac1356cc1 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt @@ -142,7 +142,7 @@ internal abstract class BuildToolsApiCompilationWork : WorkAction { + + abstract class Parameters : TransformParameters { + @get:Internal + abstract val gradleUserHomeDir: DirectoryProperty + + @get:Internal + internal abstract val classLoadersCachingService: Property + + @get:Classpath + internal abstract val classpath: ConfigurableFileCollection + } + + @get:Classpath + @get:InputArtifact + abstract val inputArtifact: Provider + + override fun transform(outputs: TransformOutputs) { + val classpathEntryInputDirOrJar = inputArtifact.get().asFile + val snapshotOutputFile = outputs.file(classpathEntryInputDirOrJar.name.replace('.', '_') + "-snapshot.bin") + + val granularity = getClassSnapshotGranularity(classpathEntryInputDirOrJar, parameters.gradleUserHomeDir.get().asFile) + + try { + val classLoader = parameters.classLoadersCachingService.get() + .getClassLoader(parameters.classpath, SharedApiClassesClassLoaderProvider) + val compilationService = CompilationService.loadImplementation(classLoader) + val snapshot = compilationService.calculateClasspathSnapshot(classpathEntryInputDirOrJar, granularity) + snapshot.saveSnapshot(snapshotOutputFile) + } catch (e: Throwable) { + throw e + } + } + + /** + * Determines the [ClassSnapshotGranularity] when taking a snapshot of the given [classpathEntryDirOrJar]. + * + * As mentioned in [ClassSnapshotGranularity]'s kdoc, we will take [CLASS_LEVEL] snapshots for classes that are infrequently changed + * (e.g., external libraries which are typically stored/transformed inside the Gradle user home, or a few hard-coded cases), and take + * [CLASS_MEMBER_LEVEL] snapshots for the others. + */ + private fun getClassSnapshotGranularity(classpathEntryDirOrJar: File, gradleUserHomeDir: File): ClassSnapshotGranularity { + return if ( + classpathEntryDirOrJar.startsWith(gradleUserHomeDir) || + classpathEntryDirOrJar.name == "android.jar" + ) CLASS_LEVEL + else CLASS_MEMBER_LEVEL + } +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/configuration/KotlinCompileConfig.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/configuration/KotlinCompileConfig.kt index 2d7fbd571cd..6fe8a236b3f 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/configuration/KotlinCompileConfig.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/configuration/KotlinCompileConfig.kt @@ -6,11 +6,17 @@ package org.jetbrains.kotlin.gradle.tasks.configuration import org.gradle.api.Project +import org.gradle.api.artifacts.transform.TransformSpec import org.gradle.api.attributes.Attribute +import org.gradle.api.file.FileCollection +import org.gradle.api.provider.Provider import org.jetbrains.kotlin.gradle.dsl.KotlinAndroidProjectExtension import org.jetbrains.kotlin.gradle.dsl.KotlinJvmProjectExtension import org.jetbrains.kotlin.gradle.dsl.KotlinTopLevelExtension +import org.jetbrains.kotlin.gradle.internal.ClassLoadersCachingBuildService +import org.jetbrains.kotlin.gradle.internal.transforms.BuildToolsApiClasspathEntrySnapshotTransform import org.jetbrains.kotlin.gradle.internal.transforms.ClasspathEntrySnapshotTransform +import org.jetbrains.kotlin.gradle.plugin.BUILD_TOOLS_API_CLASSPATH_CONFIGURATION_NAME import org.jetbrains.kotlin.gradle.plugin.KotlinCompilationInfo import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJvmAndroidCompilation import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJvmCompilation @@ -18,6 +24,7 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinWithJavaCompilation import org.jetbrains.kotlin.gradle.utils.markResolvable import org.jetbrains.kotlin.gradle.plugin.tcsOrNull import org.jetbrains.kotlin.gradle.report.BuildMetricsService +import org.jetbrains.kotlin.gradle.tasks.DefaultKotlinJavaToolchain import org.jetbrains.kotlin.gradle.tasks.KotlinCompile import org.jetbrains.kotlin.gradle.utils.providerWithLazyConvention @@ -29,7 +36,9 @@ internal open class BaseKotlinCompileConfig : AbstractKotl configureTaskProvider { taskProvider -> val useClasspathSnapshot = propertiesProvider.useClasspathSnapshot val classpathConfiguration = if (useClasspathSnapshot) { - registerTransformsOnce(project) + val runKotlinCompilerViaBuildToolsApi = propertiesProvider.runKotlinCompilerViaBuildToolsApi + val jvmToolchain = taskProvider.flatMap { it.defaultKotlinJavaToolchain } + registerTransformsOnce(project, jvmToolchain, runKotlinCompilerViaBuildToolsApi) // Note: Creating configurations should be done during build configuration, not task configuration, to avoid issues with // composite builds (e.g., https://issuetracker.google.com/183952598). project.configurations.detachedConfiguration( @@ -108,12 +117,24 @@ internal open class BaseKotlinCompileConfig : AbstractKotl const val CLASSPATH_ENTRY_SNAPSHOT_ARTIFACT_TYPE = "classpath-entry-snapshot" } - private fun registerTransformsOnce(project: Project) { + private fun registerTransformsOnce( + project: Project, + jvmToolchain: Provider, + runKotlinCompilerViaBuildToolsApi: Boolean, + ) { if (project.extensions.extraProperties.has(TRANSFORMS_REGISTERED)) { return } project.extensions.extraProperties[TRANSFORMS_REGISTERED] = true + if (runKotlinCompilerViaBuildToolsApi) { + registerBuildToolsApiTransformations(project, jvmToolchain) + } else { + registerOldTransformations(project) + } + } + + private fun registerOldTransformations(project: Project) { val buildMetricsService = BuildMetricsService.registerIfAbsent(project) project.dependencies.registerTransform(ClasspathEntrySnapshotTransform::class.java) { it.from.attribute(ARTIFACT_TYPE_ATTRIBUTE, JAR_ARTIFACT_TYPE) @@ -129,4 +150,37 @@ internal open class BaseKotlinCompileConfig : AbstractKotl } } + private fun TransformSpec.configureCommonParameters( + classLoadersCachingService: Provider, + classpath: Provider, + jvmToolchain: Provider, + ) { + parameters.gradleUserHomeDir.set(project.gradle.gradleUserHomeDir) + parameters.classLoadersCachingService.set(classLoadersCachingService) + parameters.classpath.from(classpath) + // add some tools.jar in order to reuse some of the classloaders required for compilation + parameters.classpath.from(jvmToolchain.map { toolchain -> + if (toolchain.currentJvmJdkToolsJar.isPresent) { + setOf(toolchain.currentJvmJdkToolsJar.get()) + } else { + emptySet() + } + }) + } + + private fun registerBuildToolsApiTransformations(project: Project, jvmToolchain: Provider) { + val classLoadersCachingService = ClassLoadersCachingBuildService.registerIfAbsent(project) + val classpath = project.configurations.named(BUILD_TOOLS_API_CLASSPATH_CONFIGURATION_NAME) + project.dependencies.registerTransform(BuildToolsApiClasspathEntrySnapshotTransform::class.java) { + it.from.attribute(ARTIFACT_TYPE_ATTRIBUTE, JAR_ARTIFACT_TYPE) + it.to.attribute(ARTIFACT_TYPE_ATTRIBUTE, CLASSPATH_ENTRY_SNAPSHOT_ARTIFACT_TYPE) + it.configureCommonParameters(classLoadersCachingService, classpath, jvmToolchain) + } + project.dependencies.registerTransform(BuildToolsApiClasspathEntrySnapshotTransform::class.java) { + it.from.attribute(ARTIFACT_TYPE_ATTRIBUTE, DIRECTORY_ARTIFACT_TYPE) + it.to.attribute(ARTIFACT_TYPE_ATTRIBUTE, CLASSPATH_ENTRY_SNAPSHOT_ARTIFACT_TYPE) + it.configureCommonParameters(classLoadersCachingService, classpath, jvmToolchain) + } + } + } \ No newline at end of file