From 5699cf6784b9ae5adac324530ff8c7d6e6318911 Mon Sep 17 00:00:00 2001 From: "Alexander.Likhachev" Date: Mon, 20 Mar 2023 21:50:37 +0100 Subject: [PATCH] [Gradle] Add `ClassLoadersCachingBuildService` #KT-57397 In Progress --- .../GradleKotlinCompilerRunner.kt | 6 +- .../btapi/BuildToolsApiCompilationWork.kt | 2 + .../GradleBuildToolsApiCompilerRunner.kt | 9 ++- .../ClassLoadersCachingBuildService.kt | 81 +++++++++++++++++++ .../gradle/tasks/AbstractKotlinCompile.kt | 3 + .../AbstractKotlinCompileConfig.kt | 3 + .../kotlin/gradle/utils/gradleUtils.kt | 14 ++++ 7 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/internal/ClassLoadersCachingBuildService.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt index 12e35b1f9d5..4fbf5e75cf4 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt @@ -8,6 +8,7 @@ package org.jetbrains.kotlin.compilerRunner import org.gradle.api.Project import org.gradle.api.invocation.Gradle import org.gradle.api.logging.Logger +import org.gradle.api.provider.Property import org.gradle.api.tasks.bundling.AbstractArchiveTask import org.gradle.api.tasks.bundling.Zip import org.gradle.jvm.tasks.Jar @@ -27,6 +28,7 @@ import org.jetbrains.kotlin.daemon.common.filterExtractProps import org.jetbrains.kotlin.gradle.dsl.KotlinJsProjectExtension import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull +import org.jetbrains.kotlin.gradle.internal.ClassLoadersCachingBuildService import org.jetbrains.kotlin.gradle.logging.kotlinDebug import org.jetbrains.kotlin.gradle.logging.kotlinInfo import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation @@ -60,7 +62,8 @@ internal fun createGradleCompilerRunner( compilerExecutionSettings: CompilerExecutionSettings, buildMetricsReporter: BuildMetricsReporter, workerExecutor: WorkerExecutor, - runViaBuildToolsApi: Boolean + runViaBuildToolsApi: Boolean, + cachedClassLoadersService: Property ): GradleCompilerRunner { return if (runViaBuildToolsApi) { GradleBuildToolsApiCompilerRunner( @@ -69,6 +72,7 @@ internal fun createGradleCompilerRunner( compilerExecutionSettings, buildMetricsReporter, workerExecutor, + cachedClassLoadersService, ) } else { GradleCompilerRunnerWithWorkers( diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt index 4b43f36422c..50072440558 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/BuildToolsApiCompilationWork.kt @@ -12,10 +12,12 @@ import org.gradle.workers.WorkAction import org.gradle.workers.WorkParameters import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter import org.jetbrains.kotlin.compilerRunner.GradleKotlinCompilerWorkArguments +import org.jetbrains.kotlin.gradle.internal.ClassLoadersCachingBuildService import java.io.File internal abstract class BuildToolsApiCompilationWork : WorkAction { internal interface BuildToolsApiCompilationParameters : WorkParameters { + val classLoadersCachingService: Property val compilerWorkArguments: Property val taskOutputsToRestore: ListProperty val snapshotsDir: DirectoryProperty diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/GradleBuildToolsApiCompilerRunner.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/GradleBuildToolsApiCompilerRunner.kt index b39c168c76d..2f3a8d15e73 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/GradleBuildToolsApiCompilerRunner.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/btapi/GradleBuildToolsApiCompilerRunner.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.compilerRunner.btapi +import org.gradle.api.provider.Provider import org.gradle.workers.WorkQueue import org.gradle.workers.WorkerExecutor import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter @@ -12,6 +13,7 @@ import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric import org.jetbrains.kotlin.compilerRunner.CompilerExecutionSettings import org.jetbrains.kotlin.compilerRunner.GradleCompilerRunner import org.jetbrains.kotlin.compilerRunner.GradleKotlinCompilerWorkArguments +import org.jetbrains.kotlin.gradle.internal.ClassLoadersCachingBuildService import org.jetbrains.kotlin.gradle.tasks.GradleCompileTaskProvider import org.jetbrains.kotlin.gradle.tasks.TaskOutputsBackup import java.io.File @@ -21,17 +23,20 @@ internal class GradleBuildToolsApiCompilerRunner( jdkToolsJar: File?, compilerExecutionSettings: CompilerExecutionSettings, buildMetrics: BuildMetricsReporter, - private val workerExecutor: WorkerExecutor + private val workerExecutor: WorkerExecutor, + private val cachedClassLoadersService: Provider ) : GradleCompilerRunner(taskProvider, jdkToolsJar, compilerExecutionSettings, buildMetrics) { + + override fun runCompilerAsync( workArgs: GradleKotlinCompilerWorkArguments, taskOutputsBackup: TaskOutputsBackup? ): WorkQueue { - buildMetrics.addTimeMetric(BuildPerformanceMetric.CALL_WORKER) val workQueue = workerExecutor.noIsolation() workQueue.submit(BuildToolsApiCompilationWork::class.java) { params -> params.compilerWorkArguments.set(workArgs) + params.classLoadersCachingService.set(cachedClassLoadersService) if (taskOutputsBackup != null) { params.taskOutputsToRestore.set(taskOutputsBackup.outputsToRestore) params.buildDir.set(taskOutputsBackup.buildDirectory) diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/internal/ClassLoadersCachingBuildService.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/internal/ClassLoadersCachingBuildService.kt new file mode 100644 index 00000000000..bd817018136 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/internal/ClassLoadersCachingBuildService.kt @@ -0,0 +1,81 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.internal + +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.api.logging.Logging +import org.gradle.api.provider.Property +import org.gradle.api.services.BuildService +import org.gradle.api.services.BuildServiceParameters +import org.gradle.api.tasks.Internal +import org.jetbrains.kotlin.gradle.tasks.withType +import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject +import org.jetbrains.kotlin.gradle.utils.registerClassLoaderScopedBuildService +import java.io.File +import java.net.URLClassLoader +import java.util.concurrent.ConcurrentHashMap + +internal interface UsesClassLoadersCachingBuildService : Task { + @get:Internal + val classLoadersCachingService: Property +} + +/** + * A [BuildService] for caching [ClassLoader] instances + */ +internal abstract class ClassLoadersCachingBuildService : BuildService, AutoCloseable { + // The service could be used by multiple tasks in parallel, so the map have to be synchronized + private val classLoaders = ConcurrentHashMap() + private val logger = Logging.getLogger(javaClass) + + fun getClassLoader( + classpath: Iterable, + parentClassLoaderProvider: ParentClassLoaderProvider = DefaultParentClassLoaderProvider() + ) = + classLoaders.computeIfAbsent(ClassLoaderCacheKey(classpath, parentClassLoaderProvider)) { + logger.debug("Creating a new classloader for classpath $classpath") + URLClassLoader(classpath.map { it.toURI().toURL() }.toTypedArray(), parentClassLoaderProvider.getClassLoader()) + } + + override fun close() { + logger.debug("Clearing ${classLoaders.size} cached classloaders") + classLoaders.clear() + } + + companion object { + fun registerIfAbsent(project: Project) = + project.gradle.registerClassLoaderScopedBuildService(ClassLoadersCachingBuildService::class).also { serviceProvider -> + SingleActionPerProject.run(project, UsesClassLoadersCachingBuildService::class.java.name) { + project.tasks.withType().configureEach { task -> + task.usesService(serviceProvider) + } + } + } + } +} + +private data class ClassLoaderCacheKey( + val classpath: Iterable, + val parentClassLoaderProvider: ParentClassLoaderProvider, +) + +/** + * A provider of the parent [ClassLoader] for a newly created ClassLoader instances. + * + * An implementation must override `equals` and `hashCode`! It's used as a part of a [Map] key + */ +internal fun interface ParentClassLoaderProvider { + fun getClassLoader(): ClassLoader +} + +private class DefaultParentClassLoaderProvider : ParentClassLoaderProvider { + override fun getClassLoader(): ClassLoader = javaClass.classLoader + + override fun hashCode() = javaClass.hashCode() + + override fun equals(other: Any?) = other is DefaultParentClassLoaderProvider && other.javaClass == javaClass +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/AbstractKotlinCompile.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/AbstractKotlinCompile.kt index 60b9aed1da5..5afd0e8af94 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/AbstractKotlinCompile.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/AbstractKotlinCompile.kt @@ -32,6 +32,7 @@ import org.jetbrains.kotlin.compilerRunner.createGradleCompilerRunner import org.jetbrains.kotlin.daemon.common.MultiModuleICSettings import org.jetbrains.kotlin.gradle.dsl.KotlinCommonCompilerOptions import org.jetbrains.kotlin.gradle.incremental.UsesIncrementalModuleInfoBuildService +import org.jetbrains.kotlin.gradle.internal.UsesClassLoadersCachingBuildService import org.jetbrains.kotlin.gradle.internal.tasks.allOutputFiles import org.jetbrains.kotlin.gradle.logging.GradleKotlinLogger import org.jetbrains.kotlin.gradle.logging.kotlinDebug @@ -61,6 +62,7 @@ abstract class AbstractKotlinCompile @Inject constr UsesCompilerSystemPropertiesService, UsesVariantImplementationFactories, UsesBuildFinishedListenerService, + UsesClassLoadersCachingBuildService, BaseKotlinCompile { init { @@ -184,6 +186,7 @@ abstract class AbstractKotlinCompile @Inject constr params.first, workerExecutor, runViaBuildToolsApi.get(), + classLoadersCachingService, ) } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/configuration/AbstractKotlinCompileConfig.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/configuration/AbstractKotlinCompileConfig.kt index 878ff568012..ea8d6ac6f4d 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/configuration/AbstractKotlinCompileConfig.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/configuration/AbstractKotlinCompileConfig.kt @@ -17,6 +17,7 @@ import org.jetbrains.kotlin.compilerRunner.GradleCompilerRunner import org.jetbrains.kotlin.gradle.dsl.KotlinTopLevelExtension import org.jetbrains.kotlin.gradle.dsl.topLevelExtension import org.jetbrains.kotlin.gradle.incremental.IncrementalModuleInfoBuildService +import org.jetbrains.kotlin.gradle.internal.ClassLoadersCachingBuildService import org.jetbrains.kotlin.gradle.internal.KaptGenerateStubsTask import org.jetbrains.kotlin.gradle.plugin.* import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider @@ -69,6 +70,7 @@ internal abstract class AbstractKotlinCompileConfig val propertiesProvider = project.kotlinPropertiesProvider @@ -123,6 +125,7 @@ internal abstract class AbstractKotlinCompileConfig, P : BuildServiceParameters> Gradle.registerClassLoaderScopedBuildService( + serviceClass: KClass, + configureAction: Action> = Action { }, +): Provider { + val serviceName = "${serviceClass.simpleName}_${serviceClass.java.classLoader.hashCode()}" + return sharedServices.registerIfAbsent(serviceName, serviceClass.java, configureAction) +}