From c283c761308306f4a686c67e3b18fd8844ebc275 Mon Sep 17 00:00:00 2001 From: Ilya Goncharov Date: Wed, 13 Jan 2021 16:02:06 +0300 Subject: [PATCH] [Gradle, JS] Add CacheBuilder for precache libraries --- .../gradle/targets/js/ir/KotlinJsIrLink.kt | 135 ++++++++++++++++++ .../targets/native/tasks/KotlinNativeTasks.kt | 61 ++------ .../jetbrains/kotlin/gradle/tasks/Tasks.kt | 3 +- .../kotlin/gradle/tasks/TasksProvider.kt | 1 + .../kotlin/gradle/utils/cacheKlibUtils.kt | 72 ++++++++++ 5 files changed, 221 insertions(+), 51 deletions(-) create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/utils/cacheKlibUtils.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/ir/KotlinJsIrLink.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/ir/KotlinJsIrLink.kt index 870f68fb196..547f673e478 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/ir/KotlinJsIrLink.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/ir/KotlinJsIrLink.kt @@ -5,19 +5,33 @@ package org.jetbrains.kotlin.gradle.targets.js.ir +import org.gradle.api.artifacts.Configuration +import org.gradle.api.artifacts.ResolvedDependency import org.gradle.api.file.ConfigurableFileCollection import org.gradle.api.file.DirectoryProperty import org.gradle.api.file.FileTree import org.gradle.api.model.ObjectFactory +import org.gradle.api.logging.Logger import org.gradle.api.tasks.* import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments +import org.jetbrains.kotlin.compilerRunner.GradleCompilerEnvironment +import org.jetbrains.kotlin.compilerRunner.GradleCompilerRunner +import org.jetbrains.kotlin.compilerRunner.KotlinNativeCompilerRunner +import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl import org.jetbrains.kotlin.gradle.dsl.KotlinJsOptions import org.jetbrains.kotlin.gradle.dsl.KotlinJsOptionsImpl import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.KotlinCompilationData +import org.jetbrains.kotlin.gradle.internal.tasks.allOutputFiles +import org.jetbrains.kotlin.gradle.logging.GradlePrintingMessageCollector +import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation +import org.jetbrains.kotlin.gradle.report.ReportingSettings import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode.DEVELOPMENT import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode.PRODUCTION import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile +import org.jetbrains.kotlin.gradle.utils.* +import org.jetbrains.kotlin.library.* +import java.io.File import javax.inject.Inject @CacheableTask @@ -37,6 +51,10 @@ abstract class KotlinJsIrLink @Inject constructor( } } + @Transient + @get:Internal + internal lateinit var compilation: KotlinCompilation<*> + // Link tasks are not affected by compiler plugin override val pluginClasspath: ConfigurableFileCollection = project.objects.fileCollection() @@ -47,6 +65,12 @@ abstract class KotlinJsIrLink @Inject constructor( @Internal override fun getSource(): FileTree = super.getSource() + private val buildDir = project.buildDir + + private val compileClasspathConfiguration by lazy { + project.configurations.getByName(compilation.compileDependencyConfigurationName) + } + @get:SkipWhenEmpty @get:InputDirectory @get:PathSensitive(PathSensitivity.RELATIVE) @@ -63,6 +87,17 @@ abstract class KotlinJsIrLink @Inject constructor( } DEVELOPMENT -> { kotlinOptions.configureOptions(GENERATE_D_TS) + CacheBuilder( + buildDir, + compileClasspathConfiguration, + libraryFilter, + compilerRunner, + { createCompilerArgs() }, + computedCompilerClasspath, + logger, + objects.fileCollection(), + reportingSettings + ).buildCompilerArgs() } } super.setupCompilerArgs(args, defaultsOnly, ignoreClasspathResolutionErrors) @@ -74,3 +109,103 @@ abstract class KotlinJsIrLink @Inject constructor( "$ENTRY_IR_MODULE=${entryModule.get().asFile.canonicalPath}" } } + + +internal class CacheBuilder( + private val buildDir: File, + private val compileClasspath: Configuration, + private val libraryFilter: (File) -> Boolean, + private val compilerRunner: GradleCompilerRunner, + private val compilerArgsFactory: () -> K2JSCompilerArguments, + private val computedCompilerClasspath: List, + private val logger: Logger, + private val outputFiles: FileCollection, + private val reportingSettings: ReportingSettings + +) { + val rootCacheDirectory by lazy { + buildDir.resolve("klib/cache") + } + + fun buildCompilerArgs(): List { + val visitedDependencies = mutableSetOf() + val allCacheDirectories = mutableSetOf() + + compileClasspath.resolvedConfiguration.firstLevelModuleDependencies + .forEach { dependency -> + ensureDependencyCached(dependency, visitedDependencies) + (listOf(dependency) + getAllDependencies(dependency)) + .forEach { + val cacheDirectory = getCacheDirectory(rootCacheDirectory, dependency) + if (cacheDirectory.exists()) { + allCacheDirectories.add(cacheDirectory.normalize().absolutePath) + } + } + } + + return allCacheDirectories + .map { "-Xcache-directory=${it}" } + } + + private fun ensureDependencyCached( + dependency: ResolvedDependency, + visitedDependency: MutableSet + ) { + if (dependency in visitedDependency) return + visitedDependency.add(dependency) + + dependency.children + .forEach { ensureDependencyCached(it, visitedDependency) } + + val artifactsToAddToCache = dependency.moduleArtifacts + .filter { libraryFilter(it.file) } + + if (artifactsToAddToCache.isEmpty()) return + + val dependenciesCacheDirectories = getDependenciesCacheDirectories( + rootCacheDirectory, + dependency + ) ?: return + + val cacheDirectory = getCacheDirectory(rootCacheDirectory, dependency) + cacheDirectory.mkdirs() + + for (library in artifactsToAddToCache) { + if (cacheDirectory.listFilesOrEmpty().isNotEmpty()) + continue + + val compilerArgs = compilerArgsFactory() + compilerArgs.includes = library.file.normalize().absolutePath + compilerArgs.outputFile = cacheDirectory.resolve("${library.name}.js").normalize().absolutePath + compilerArgs.irProduceJs = true + dependenciesCacheDirectories.forEach { + compilerArgs.freeArgs += "-Xcache-directory=${it.absolutePath}" + } + + compilerArgs.libraries = getAllDependencies(dependency) + .flatMap { it.moduleArtifacts } + .map { it.file } + .filter { it.exists() && libraryFilter(it) } + .distinct() + .joinToString(File.pathSeparator) { it.normalize().absolutePath } + + val messageCollector = GradlePrintingMessageCollector(logger, false) + val outputItemCollector = OutputItemsCollectorImpl() + val environment = GradleCompilerEnvironment( + computedCompilerClasspath, + messageCollector, + outputItemCollector, + outputFiles = outputFiles, + reportingSettings = reportingSettings + ) + + compilerRunner + .runJsCompilerAsync( + emptyList(), + emptyList(), + compilerArgs, + environment + ) + } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt index 7910c87833d..f8ff81c8adf 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt @@ -31,9 +31,10 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.KotlinNativeFragmentMetadataC import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.isMainCompilationData import org.jetbrains.kotlin.gradle.plugin.sources.DefaultLanguageSettingsBuilder import org.jetbrains.kotlin.gradle.targets.native.internal.isAllowCommonizer -import org.jetbrains.kotlin.gradle.utils.getValue +import org.jetbrains.kotlin.gradle.utils.* import org.jetbrains.kotlin.gradle.utils.klibModuleName import org.jetbrains.kotlin.gradle.utils.listFilesOrEmpty +import org.jetbrains.kotlin.gradle.utils.newProperty import org.jetbrains.kotlin.konan.library.KLIB_INTEROP_IR_PROVIDER_IDENTIFIER import org.jetbrains.kotlin.konan.properties.resolvablePropertyList import org.jetbrains.kotlin.konan.properties.saveToFile @@ -696,38 +697,10 @@ internal class CacheBuilder(val project: Project, val binary: NativeBinary, val getRootCacheDirectory(File(project.konanHome), konanTarget, debuggable, konanCacheKind) } - private fun getAllDependencies(dependency: ResolvedDependency): Set { - val allDependencies = mutableSetOf() - - fun traverseAllDependencies(dependency: ResolvedDependency) { - if (dependency in allDependencies) - return - allDependencies.add(dependency) - dependency.children.forEach { traverseAllDependencies(it) } - } - - dependency.children.forEach { traverseAllDependencies(it) } - return allDependencies - } - - private fun ByteArray.toHexString() = joinToString("") { (0xFF and it.toInt()).toString(16).padStart(2, '0') } - - private fun computeDependenciesHash(dependency: ResolvedDependency): String { - val allArtifactsPaths = - (dependency.moduleArtifacts + getAllDependencies(dependency).flatMap { it.moduleArtifacts }) - .map { it.file.absolutePath } - .distinct() - .sortedBy { it } - .joinToString("|") { it } - val digest = MessageDigest.getInstance("SHA-256") - val hash = digest.digest(allArtifactsPaths.toByteArray(StandardCharsets.UTF_8)) - return hash.toHexString() - } - - private fun getCacheDirectory(dependency: ResolvedDependency): File { - val moduleCacheDirectory = File(rootCacheDirectory, dependency.moduleName) - val versionCacheDirectory = File(moduleCacheDirectory, dependency.moduleVersion) - return File(versionCacheDirectory, computeDependenciesHash(dependency)) + private fun getCacheDirectory( + dependency: ResolvedDependency + ): File { + return getCacheDirectory(rootCacheDirectory, dependency) } private fun needCache(libraryPath: String) = @@ -742,16 +715,11 @@ internal class CacheBuilder(val project: Project, val binary: NativeBinary, val val artifactsToAddToCache = dependency.moduleArtifacts.filter { needCache(it.file.absolutePath) } if (artifactsToAddToCache.isEmpty()) return - val dependenciesCacheDirectories = getAllDependencies(dependency) - .map { childDependency -> - val hasKlibs = childDependency.moduleArtifacts.any { it.file.absolutePath.endsWith(".klib") } - val cacheDirectory = getCacheDirectory(childDependency) - // We can only compile klib to cache if all of its dependencies are also cached. - if (hasKlibs && !cacheDirectory.exists()) - return - cacheDirectory - } - .filter { it.exists() } + val dependenciesCacheDirectories = getDependenciesCacheDirectories( + rootCacheDirectory, + dependency + ) ?: return + val cacheDirectory = getCacheDirectory(dependency) cacheDirectory.mkdirs() @@ -883,13 +851,6 @@ internal class CacheBuilder(val project: Project, val binary: NativeBinary, val } } - private class GradleLoggerAdapter(private val gradleLogger: Logger) : KLogger { - override fun log(message: String) = gradleLogger.info(message) - override fun warning(message: String) = gradleLogger.warn(message) - override fun error(message: String) = kotlin.error(message) - override fun fatal(message: String): Nothing = kotlin.error(message) - } - companion object { internal fun getRootCacheDirectory(konanHome: File, target: KonanTarget, debuggable: Boolean, cacheKind: NativeCacheKind): File { require(cacheKind != NativeCacheKind.NONE) { "Usupported cache kind: ${NativeCacheKind.NONE}" } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt index cb656844bc8..dac78cd7dec 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt @@ -695,7 +695,8 @@ abstract class Kotlin2JsCompile @Inject constructor( // 1) purely pre-IR backend // 2) purely IR backend // 3) hybrid pre-IR and IR backend. Can only accept libraries with both JS and IR parts. - private val libraryFilter: (File) -> Boolean + @get:Internal + protected val libraryFilter: (File) -> Boolean get() = if (kotlinOptions.isIrBackendEnabled()) { if (kotlinOptions.isPreIrBackendDisabled()) { ::isKotlinLibrary diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/TasksProvider.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/TasksProvider.kt index 566ce8302c0..90cac9a1095 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/TasksProvider.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/TasksProvider.kt @@ -122,6 +122,7 @@ internal open class KotlinTasksProvider { val properties = PropertiesProvider(project) val taskClass = taskOrWorkersTask(properties) val result = project.registerTask(name, taskClass) { + it.compilation = compilation configureAction(it) KotlinJsIrLink.Configurator(compilation).configure(it) } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/utils/cacheKlibUtils.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/utils/cacheKlibUtils.kt new file mode 100644 index 00000000000..ac2ae0a5d8b --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/utils/cacheKlibUtils.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.utils + +import org.gradle.api.artifacts.ResolvedDependency +import org.gradle.api.logging.Logger +import java.io.File +import java.nio.charset.StandardCharsets +import java.security.MessageDigest + +fun getCacheDirectory( + rootCacheDirectory: File, + dependency: ResolvedDependency +): File { + val moduleCacheDirectory = File(rootCacheDirectory, dependency.moduleName) + val versionCacheDirectory = File(moduleCacheDirectory, dependency.moduleVersion) + return File(versionCacheDirectory, computeDependenciesHash(dependency)) +} + +private fun ByteArray.toHexString() = joinToString("") { (0xFF and it.toInt()).toString(16).padStart(2, '0') } + +private fun computeDependenciesHash(dependency: ResolvedDependency): String { + val allArtifactsPaths = + (dependency.moduleArtifacts + getAllDependencies(dependency).flatMap { it.moduleArtifacts }) + .map { it.file.absolutePath } + .distinct() + .sortedBy { it } + .joinToString("|") { it } + val digest = MessageDigest.getInstance("SHA-256") + val hash = digest.digest(allArtifactsPaths.toByteArray(StandardCharsets.UTF_8)) + return hash.toHexString() +} + +fun getDependenciesCacheDirectories( + rootCacheDirectory: File, + dependency: ResolvedDependency +): List? { + return getAllDependencies(dependency) + .map { childDependency -> + val hasKlibs = childDependency.moduleArtifacts.any { it.file.absolutePath.endsWith(".klib") } + val cacheDirectory = getCacheDirectory(rootCacheDirectory, childDependency) + // We can only compile klib to cache if all of its dependencies are also cached. + if (hasKlibs && !cacheDirectory.exists()) + return null + cacheDirectory + } + .filter { it.exists() } +} + +fun getAllDependencies(dependency: ResolvedDependency): Set { + val allDependencies = mutableSetOf() + + fun traverseAllDependencies(dependency: ResolvedDependency) { + if (dependency in allDependencies) + return + allDependencies.add(dependency) + dependency.children.forEach { traverseAllDependencies(it) } + } + + dependency.children.forEach { traverseAllDependencies(it) } + return allDependencies +} + +internal class GradleLoggerAdapter(private val gradleLogger: Logger) : org.jetbrains.kotlin.util.Logger { + override fun log(message: String) = gradleLogger.info(message) + override fun warning(message: String) = gradleLogger.warn(message) + override fun error(message: String) = kotlin.error(message) + override fun fatal(message: String): Nothing = kotlin.error(message) +} \ No newline at end of file