[Gradle] Add ClassLoadersCachingBuildService

#KT-57397 In Progress
2023-03-20 21:50:37 +01:00
parent bb635d1bdb
commit 5699cf6784
7 changed files with 115 additions and 3 deletions
@@ -8,6 +8,7 @@ package org.jetbrains.kotlin.compilerRunner
 import org.gradle.api.Project
 import org.gradle.api.invocation.Gradle
 import org.gradle.api.logging.Logger
+import org.gradle.api.provider.Property
 import org.gradle.api.tasks.bundling.AbstractArchiveTask
 import org.gradle.api.tasks.bundling.Zip
 import org.gradle.jvm.tasks.Jar
@@ -27,6 +28,7 @@ import org.jetbrains.kotlin.daemon.common.filterExtractProps
 import org.jetbrains.kotlin.gradle.dsl.KotlinJsProjectExtension
 import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension
 import org.jetbrains.kotlin.gradle.dsl.multiplatformExtensionOrNull
+import org.jetbrains.kotlin.gradle.internal.ClassLoadersCachingBuildService
 import org.jetbrains.kotlin.gradle.logging.kotlinDebug
 import org.jetbrains.kotlin.gradle.logging.kotlinInfo
 import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation
@@ -60,7 +62,8 @@ internal fun createGradleCompilerRunner(
    compilerExecutionSettings: CompilerExecutionSettings,
    buildMetricsReporter: BuildMetricsReporter,
    workerExecutor: WorkerExecutor,
-    runViaBuildToolsApi: Boolean
+    runViaBuildToolsApi: Boolean,
+    cachedClassLoadersService: Property<ClassLoadersCachingBuildService>
 ): GradleCompilerRunner {
    return if (runViaBuildToolsApi) {
        GradleBuildToolsApiCompilerRunner(
@@ -69,6 +72,7 @@ internal fun createGradleCompilerRunner(
            compilerExecutionSettings,
            buildMetricsReporter,
            workerExecutor,
+            cachedClassLoadersService,
        )
    } else {
        GradleCompilerRunnerWithWorkers(
@@ -12,10 +12,12 @@ import org.gradle.workers.WorkAction
 import org.gradle.workers.WorkParameters
 import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
 import org.jetbrains.kotlin.compilerRunner.GradleKotlinCompilerWorkArguments
+import org.jetbrains.kotlin.gradle.internal.ClassLoadersCachingBuildService
 import java.io.File

 internal abstract class BuildToolsApiCompilationWork : WorkAction<BuildToolsApiCompilationWork.BuildToolsApiCompilationParameters> {
    internal interface BuildToolsApiCompilationParameters : WorkParameters {
+        val classLoadersCachingService: Property<ClassLoadersCachingBuildService>
        val compilerWorkArguments: Property<GradleKotlinCompilerWorkArguments>
        val taskOutputsToRestore: ListProperty<File>
        val snapshotsDir: DirectoryProperty
@@ -5,6 +5,7 @@

 package org.jetbrains.kotlin.compilerRunner.btapi

+import org.gradle.api.provider.Provider
 import org.gradle.workers.WorkQueue
 import org.gradle.workers.WorkerExecutor
 import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
@@ -12,6 +13,7 @@ import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric
 import org.jetbrains.kotlin.compilerRunner.CompilerExecutionSettings
 import org.jetbrains.kotlin.compilerRunner.GradleCompilerRunner
 import org.jetbrains.kotlin.compilerRunner.GradleKotlinCompilerWorkArguments
+import org.jetbrains.kotlin.gradle.internal.ClassLoadersCachingBuildService
 import org.jetbrains.kotlin.gradle.tasks.GradleCompileTaskProvider
 import org.jetbrains.kotlin.gradle.tasks.TaskOutputsBackup
 import java.io.File
@@ -21,17 +23,20 @@ internal class GradleBuildToolsApiCompilerRunner(
    jdkToolsJar: File?,
    compilerExecutionSettings: CompilerExecutionSettings,
    buildMetrics: BuildMetricsReporter,
-    private val workerExecutor: WorkerExecutor
+    private val workerExecutor: WorkerExecutor,
+    private val cachedClassLoadersService: Provider<ClassLoadersCachingBuildService>
 ) : GradleCompilerRunner(taskProvider, jdkToolsJar, compilerExecutionSettings, buildMetrics) {
+
+
    override fun runCompilerAsync(
        workArgs: GradleKotlinCompilerWorkArguments,
        taskOutputsBackup: TaskOutputsBackup?
    ): WorkQueue {
-
        buildMetrics.addTimeMetric(BuildPerformanceMetric.CALL_WORKER)
        val workQueue = workerExecutor.noIsolation()
        workQueue.submit(BuildToolsApiCompilationWork::class.java) { params ->
            params.compilerWorkArguments.set(workArgs)
+            params.classLoadersCachingService.set(cachedClassLoadersService)
            if (taskOutputsBackup != null) {
                params.taskOutputsToRestore.set(taskOutputsBackup.outputsToRestore)
                params.buildDir.set(taskOutputsBackup.buildDirectory)
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.internal
+
+import org.gradle.api.Project
+import org.gradle.api.Task
+import org.gradle.api.logging.Logging
+import org.gradle.api.provider.Property
+import org.gradle.api.services.BuildService
+import org.gradle.api.services.BuildServiceParameters
+import org.gradle.api.tasks.Internal
+import org.jetbrains.kotlin.gradle.tasks.withType
+import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject
+import org.jetbrains.kotlin.gradle.utils.registerClassLoaderScopedBuildService
+import java.io.File
+import java.net.URLClassLoader
+import java.util.concurrent.ConcurrentHashMap
+
+internal interface UsesClassLoadersCachingBuildService : Task {
+    @get:Internal
+    val classLoadersCachingService: Property<ClassLoadersCachingBuildService>
+}
+
+/**
+ * A [BuildService] for caching [ClassLoader] instances
+ */
+internal abstract class ClassLoadersCachingBuildService : BuildService<BuildServiceParameters.None>, AutoCloseable {
+    // The service could be used by multiple tasks in parallel, so the map have to be synchronized
+    private val classLoaders = ConcurrentHashMap<ClassLoaderCacheKey, ClassLoader>()
+    private val logger = Logging.getLogger(javaClass)
+
+    fun getClassLoader(
+        classpath: Iterable<File>,
+        parentClassLoaderProvider: ParentClassLoaderProvider = DefaultParentClassLoaderProvider()
+    ) =
+        classLoaders.computeIfAbsent(ClassLoaderCacheKey(classpath, parentClassLoaderProvider)) {
+            logger.debug("Creating a new classloader for classpath $classpath")
+            URLClassLoader(classpath.map { it.toURI().toURL() }.toTypedArray(), parentClassLoaderProvider.getClassLoader())
+        }
+
+    override fun close() {
+        logger.debug("Clearing ${classLoaders.size} cached classloaders")
+        classLoaders.clear()
+    }
+
+    companion object {
+        fun registerIfAbsent(project: Project) =
+            project.gradle.registerClassLoaderScopedBuildService(ClassLoadersCachingBuildService::class).also { serviceProvider ->
+                SingleActionPerProject.run(project, UsesClassLoadersCachingBuildService::class.java.name) {
+                    project.tasks.withType<UsesClassLoadersCachingBuildService>().configureEach { task ->
+                        task.usesService(serviceProvider)
+                    }
+                }
+            }
+    }
+}
+
+private data class ClassLoaderCacheKey(
+    val classpath: Iterable<File>,
+    val parentClassLoaderProvider: ParentClassLoaderProvider,
+)
+
+/**
+ * A provider of the parent [ClassLoader] for a newly created ClassLoader instances.
+ *
+ * An implementation must override `equals` and `hashCode`! It's used as a part of a [Map] key
+ */
+internal fun interface ParentClassLoaderProvider {
+    fun getClassLoader(): ClassLoader
+}
+
+private class DefaultParentClassLoaderProvider : ParentClassLoaderProvider {
+    override fun getClassLoader(): ClassLoader = javaClass.classLoader
+
+    override fun hashCode() = javaClass.hashCode()
+
+    override fun equals(other: Any?) = other is DefaultParentClassLoaderProvider && other.javaClass == javaClass
+}
@@ -32,6 +32,7 @@ import org.jetbrains.kotlin.compilerRunner.createGradleCompilerRunner
 import org.jetbrains.kotlin.daemon.common.MultiModuleICSettings
 import org.jetbrains.kotlin.gradle.dsl.KotlinCommonCompilerOptions
 import org.jetbrains.kotlin.gradle.incremental.UsesIncrementalModuleInfoBuildService
+import org.jetbrains.kotlin.gradle.internal.UsesClassLoadersCachingBuildService
 import org.jetbrains.kotlin.gradle.internal.tasks.allOutputFiles
 import org.jetbrains.kotlin.gradle.logging.GradleKotlinLogger
 import org.jetbrains.kotlin.gradle.logging.kotlinDebug
@@ -61,6 +62,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> @Inject constr
    UsesCompilerSystemPropertiesService,
    UsesVariantImplementationFactories,
    UsesBuildFinishedListenerService,
+    UsesClassLoadersCachingBuildService,
    BaseKotlinCompile {

    init {
@@ -184,6 +186,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> @Inject constr
                                    params.first,
                                    workerExecutor,
                                    runViaBuildToolsApi.get(),
+                                    classLoadersCachingService,
                                )
                            }
                    }
@@ -17,6 +17,7 @@ import org.jetbrains.kotlin.compilerRunner.GradleCompilerRunner
 import org.jetbrains.kotlin.gradle.dsl.KotlinTopLevelExtension
 import org.jetbrains.kotlin.gradle.dsl.topLevelExtension
 import org.jetbrains.kotlin.gradle.incremental.IncrementalModuleInfoBuildService
+import org.jetbrains.kotlin.gradle.internal.ClassLoadersCachingBuildService
 import org.jetbrains.kotlin.gradle.internal.KaptGenerateStubsTask
 import org.jetbrains.kotlin.gradle.plugin.*
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider
@@ -69,6 +70,7 @@ internal abstract class AbstractKotlinCompileConfig<TASK : AbstractKotlinCompile
                GradleCompilerRunner.buildModulesInfo(project.gradle)
            })
        val buildFinishedListenerService = BuildFinishedListenerService.registerIfAbsent(project)
+        val cachedClassLoadersService = ClassLoadersCachingBuildService.registerIfAbsent(project)
        configureTask { task ->
            val propertiesProvider = project.kotlinPropertiesProvider

@@ -123,6 +125,7 @@ internal abstract class AbstractKotlinCompileConfig<TASK : AbstractKotlinCompile
                task.compilerOptions.useK2.value(true)
            }
            task.runViaBuildToolsApi.convention(propertiesProvider.runKotlinCompilerViaBuildToolsApi).finalizeValueOnRead()
+            task.classLoadersCachingService.value(cachedClassLoadersService).disallowChanges()
        }
    }

@@ -5,10 +5,24 @@

 package org.jetbrains.kotlin.gradle.utils

+import org.gradle.api.Action
 import org.gradle.api.Project
 import org.gradle.api.invocation.Gradle
+import org.gradle.api.provider.Provider
+import org.gradle.api.services.BuildService
+import org.gradle.api.services.BuildServiceParameters
+import org.gradle.api.services.BuildServiceSpec
+import kotlin.reflect.KClass

 val Gradle.projectCacheDir
    get() = startParameter.projectCacheDir ?: this.rootProject.projectDir.resolve(".gradle")

 internal val Project.compositeBuildRootProject: Project get() = generateSequence(project.gradle) { it.parent }.last().rootProject
+
+internal fun <T : BuildService<P>, P : BuildServiceParameters> Gradle.registerClassLoaderScopedBuildService(
+    serviceClass: KClass<T>,
+    configureAction: Action<BuildServiceSpec<P>> = Action { },
+): Provider<T> {
+    val serviceName = "${serviceClass.simpleName}_${serviceClass.java.classLoader.hashCode()}"
+    return sharedServices.registerIfAbsent(serviceName, serviceClass.java, configureAction)
+}