[Gradle, JS] Add CacheBuilder for precache libraries

2021-01-13 16:02:06 +03:00
parent 420bc6cbe8
commit c283c76130
5 changed files with 221 additions and 51 deletions
@@ -5,19 +5,33 @@

 package org.jetbrains.kotlin.gradle.targets.js.ir

+import org.gradle.api.artifacts.Configuration
+import org.gradle.api.artifacts.ResolvedDependency
 import org.gradle.api.file.ConfigurableFileCollection
 import org.gradle.api.file.DirectoryProperty
 import org.gradle.api.file.FileTree
 import org.gradle.api.model.ObjectFactory
+import org.gradle.api.logging.Logger
 import org.gradle.api.tasks.*
 import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
+import org.jetbrains.kotlin.compilerRunner.GradleCompilerEnvironment
+import org.jetbrains.kotlin.compilerRunner.GradleCompilerRunner
+import org.jetbrains.kotlin.compilerRunner.KotlinNativeCompilerRunner
+import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl
 import org.jetbrains.kotlin.gradle.dsl.KotlinJsOptions
 import org.jetbrains.kotlin.gradle.dsl.KotlinJsOptionsImpl
 import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.KotlinCompilationData
+import org.jetbrains.kotlin.gradle.internal.tasks.allOutputFiles
+import org.jetbrains.kotlin.gradle.logging.GradlePrintingMessageCollector
+import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation
+import org.jetbrains.kotlin.gradle.report.ReportingSettings
 import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode
 import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode.DEVELOPMENT
 import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode.PRODUCTION
 import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile
+import org.jetbrains.kotlin.gradle.utils.*
+import org.jetbrains.kotlin.library.*
+import java.io.File
 import javax.inject.Inject

@CacheableTask
@@ -37,6 +51,10 @@ abstract class KotlinJsIrLink @Inject constructor(
        }
    }

+    @Transient
+    @get:Internal
+    internal lateinit var compilation: KotlinCompilation<*>
+
    // Link tasks are not affected by compiler plugin
    override val pluginClasspath: ConfigurableFileCollection = project.objects.fileCollection()

@@ -47,6 +65,12 @@ abstract class KotlinJsIrLink @Inject constructor(
    @Internal
    override fun getSource(): FileTree = super.getSource()

+    private val buildDir = project.buildDir
+
+    private val compileClasspathConfiguration by lazy {
+        project.configurations.getByName(compilation.compileDependencyConfigurationName)
+    }
+
    @get:SkipWhenEmpty
    @get:InputDirectory
    @get:PathSensitive(PathSensitivity.RELATIVE)
@@ -63,6 +87,17 @@ abstract class KotlinJsIrLink @Inject constructor(
            }
            DEVELOPMENT -> {
                kotlinOptions.configureOptions(GENERATE_D_TS)
+                CacheBuilder(
+                    buildDir,
+                    compileClasspathConfiguration,
+                    libraryFilter,
+                    compilerRunner,
+                    { createCompilerArgs() },
+                    computedCompilerClasspath,
+                    logger,
+                    objects.fileCollection(),
+                    reportingSettings
+                ).buildCompilerArgs()
            }
        }
        super.setupCompilerArgs(args, defaultsOnly, ignoreClasspathResolutionErrors)
@@ -74,3 +109,103 @@ abstract class KotlinJsIrLink @Inject constructor(
                "$ENTRY_IR_MODULE=${entryModule.get().asFile.canonicalPath}"
    }
 }
+
+
+internal class CacheBuilder(
+    private val buildDir: File,
+    private val compileClasspath: Configuration,
+    private val libraryFilter: (File) -> Boolean,
+    private val compilerRunner: GradleCompilerRunner,
+    private val compilerArgsFactory: () -> K2JSCompilerArguments,
+    private val computedCompilerClasspath: List<File>,
+    private val logger: Logger,
+    private val outputFiles: FileCollection,
+    private val reportingSettings: ReportingSettings
+
+) {
+    val rootCacheDirectory by lazy {
+        buildDir.resolve("klib/cache")
+    }
+
+    fun buildCompilerArgs(): List<String> {
+        val visitedDependencies = mutableSetOf<ResolvedDependency>()
+        val allCacheDirectories = mutableSetOf<String>()
+
+        compileClasspath.resolvedConfiguration.firstLevelModuleDependencies
+            .forEach { dependency ->
+                ensureDependencyCached(dependency, visitedDependencies)
+                (listOf(dependency) + getAllDependencies(dependency))
+                    .forEach {
+                        val cacheDirectory = getCacheDirectory(rootCacheDirectory, dependency)
+                        if (cacheDirectory.exists()) {
+                            allCacheDirectories.add(cacheDirectory.normalize().absolutePath)
+                        }
+                    }
+            }
+
+        return allCacheDirectories
+            .map { "-Xcache-directory=${it}" }
+    }
+
+    private fun ensureDependencyCached(
+        dependency: ResolvedDependency,
+        visitedDependency: MutableSet<ResolvedDependency>
+    ) {
+        if (dependency in visitedDependency) return
+        visitedDependency.add(dependency)
+
+        dependency.children
+            .forEach { ensureDependencyCached(it, visitedDependency) }
+
+        val artifactsToAddToCache = dependency.moduleArtifacts
+            .filter { libraryFilter(it.file) }
+
+        if (artifactsToAddToCache.isEmpty()) return
+
+        val dependenciesCacheDirectories = getDependenciesCacheDirectories(
+            rootCacheDirectory,
+            dependency
+        ) ?: return
+
+        val cacheDirectory = getCacheDirectory(rootCacheDirectory, dependency)
+        cacheDirectory.mkdirs()
+
+        for (library in artifactsToAddToCache) {
+            if (cacheDirectory.listFilesOrEmpty().isNotEmpty())
+                continue
+
+            val compilerArgs = compilerArgsFactory()
+            compilerArgs.includes = library.file.normalize().absolutePath
+            compilerArgs.outputFile = cacheDirectory.resolve("${library.name}.js").normalize().absolutePath
+            compilerArgs.irProduceJs = true
+            dependenciesCacheDirectories.forEach {
+                compilerArgs.freeArgs += "-Xcache-directory=${it.absolutePath}"
+            }
+
+            compilerArgs.libraries = getAllDependencies(dependency)
+                .flatMap { it.moduleArtifacts }
+                .map { it.file }
+                .filter { it.exists() && libraryFilter(it) }
+                .distinct()
+                .joinToString(File.pathSeparator) { it.normalize().absolutePath }
+
+            val messageCollector = GradlePrintingMessageCollector(logger, false)
+            val outputItemCollector = OutputItemsCollectorImpl()
+            val environment = GradleCompilerEnvironment(
+                computedCompilerClasspath,
+                messageCollector,
+                outputItemCollector,
+                outputFiles = outputFiles,
+                reportingSettings = reportingSettings
+            )
+
+            compilerRunner
+                .runJsCompilerAsync(
+                    emptyList(),
+                    emptyList(),
+                    compilerArgs,
+                    environment
+                )
+        }
+    }
+}
@@ -31,9 +31,10 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.KotlinNativeFragmentMetadataC
 import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.isMainCompilationData
 import org.jetbrains.kotlin.gradle.plugin.sources.DefaultLanguageSettingsBuilder
 import org.jetbrains.kotlin.gradle.targets.native.internal.isAllowCommonizer
-import org.jetbrains.kotlin.gradle.utils.getValue
+import org.jetbrains.kotlin.gradle.utils.*
 import org.jetbrains.kotlin.gradle.utils.klibModuleName
 import org.jetbrains.kotlin.gradle.utils.listFilesOrEmpty
+import org.jetbrains.kotlin.gradle.utils.newProperty
 import org.jetbrains.kotlin.konan.library.KLIB_INTEROP_IR_PROVIDER_IDENTIFIER
 import org.jetbrains.kotlin.konan.properties.resolvablePropertyList
 import org.jetbrains.kotlin.konan.properties.saveToFile
@@ -696,38 +697,10 @@ internal class CacheBuilder(val project: Project, val binary: NativeBinary, val
        getRootCacheDirectory(File(project.konanHome), konanTarget, debuggable, konanCacheKind)
    }

-    private fun getAllDependencies(dependency: ResolvedDependency): Set<ResolvedDependency> {
-        val allDependencies = mutableSetOf<ResolvedDependency>()
-
-        fun traverseAllDependencies(dependency: ResolvedDependency) {
-            if (dependency in allDependencies)
-                return
-            allDependencies.add(dependency)
-            dependency.children.forEach { traverseAllDependencies(it) }
-        }
-
-        dependency.children.forEach { traverseAllDependencies(it) }
-        return allDependencies
-    }
-
-    private fun ByteArray.toHexString() = joinToString("") { (0xFF and it.toInt()).toString(16).padStart(2, '0') }
-
-    private fun computeDependenciesHash(dependency: ResolvedDependency): String {
-        val allArtifactsPaths =
-            (dependency.moduleArtifacts + getAllDependencies(dependency).flatMap { it.moduleArtifacts })
-                .map { it.file.absolutePath }
-                .distinct()
-                .sortedBy { it }
-                .joinToString("|") { it }
-        val digest = MessageDigest.getInstance("SHA-256")
-        val hash = digest.digest(allArtifactsPaths.toByteArray(StandardCharsets.UTF_8))
-        return hash.toHexString()
-    }
-
-    private fun getCacheDirectory(dependency: ResolvedDependency): File {
-        val moduleCacheDirectory = File(rootCacheDirectory, dependency.moduleName)
-        val versionCacheDirectory = File(moduleCacheDirectory, dependency.moduleVersion)
-        return File(versionCacheDirectory, computeDependenciesHash(dependency))
+    private fun getCacheDirectory(
+        dependency: ResolvedDependency
+    ): File {
+        return getCacheDirectory(rootCacheDirectory, dependency)
    }

    private fun needCache(libraryPath: String) =
@@ -742,16 +715,11 @@ internal class CacheBuilder(val project: Project, val binary: NativeBinary, val
        val artifactsToAddToCache = dependency.moduleArtifacts.filter { needCache(it.file.absolutePath) }
        if (artifactsToAddToCache.isEmpty()) return

-        val dependenciesCacheDirectories = getAllDependencies(dependency)
-            .map { childDependency ->
-                val hasKlibs = childDependency.moduleArtifacts.any { it.file.absolutePath.endsWith(".klib") }
-                val cacheDirectory = getCacheDirectory(childDependency)
-                // We can only compile klib to cache if all of its dependencies are also cached.
-                if (hasKlibs && !cacheDirectory.exists())
-                    return
-                cacheDirectory
-            }
-            .filter { it.exists() }
+        val dependenciesCacheDirectories = getDependenciesCacheDirectories(
+            rootCacheDirectory,
+            dependency
+        ) ?: return
+
        val cacheDirectory = getCacheDirectory(dependency)
        cacheDirectory.mkdirs()

@@ -883,13 +851,6 @@ internal class CacheBuilder(val project: Project, val binary: NativeBinary, val
        }
    }

-    private class GradleLoggerAdapter(private val gradleLogger: Logger) : KLogger {
-        override fun log(message: String) = gradleLogger.info(message)
-        override fun warning(message: String) = gradleLogger.warn(message)
-        override fun error(message: String) = kotlin.error(message)
-        override fun fatal(message: String): Nothing = kotlin.error(message)
-    }
-
    companion object {
        internal fun getRootCacheDirectory(konanHome: File, target: KonanTarget, debuggable: Boolean, cacheKind: NativeCacheKind): File {
            require(cacheKind != NativeCacheKind.NONE) { "Usupported cache kind: ${NativeCacheKind.NONE}" }
@@ -695,7 +695,8 @@ abstract class Kotlin2JsCompile @Inject constructor(
    //  1) purely pre-IR backend
    //  2) purely IR backend
    //  3) hybrid pre-IR and IR backend. Can only accept libraries with both JS and IR parts.
-    private val libraryFilter: (File) -> Boolean
+    @get:Internal
+    protected val libraryFilter: (File) -> Boolean
        get() = if (kotlinOptions.isIrBackendEnabled()) {
            if (kotlinOptions.isPreIrBackendDisabled()) {
                ::isKotlinLibrary
@@ -122,6 +122,7 @@ internal open class KotlinTasksProvider {
        val properties = PropertiesProvider(project)
        val taskClass = taskOrWorkersTask<KotlinJsIrLink, KotlinJsIrLinkWithWorkers>(properties)
        val result = project.registerTask(name, taskClass) {
+            it.compilation = compilation
            configureAction(it)
            KotlinJsIrLink.Configurator(compilation).configure(it)
        }
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.utils
+
+import org.gradle.api.artifacts.ResolvedDependency
+import org.gradle.api.logging.Logger
+import java.io.File
+import java.nio.charset.StandardCharsets
+import java.security.MessageDigest
+
+fun getCacheDirectory(
+    rootCacheDirectory: File,
+    dependency: ResolvedDependency
+): File {
+    val moduleCacheDirectory = File(rootCacheDirectory, dependency.moduleName)
+    val versionCacheDirectory = File(moduleCacheDirectory, dependency.moduleVersion)
+    return File(versionCacheDirectory, computeDependenciesHash(dependency))
+}
+
+private fun ByteArray.toHexString() = joinToString("") { (0xFF and it.toInt()).toString(16).padStart(2, '0') }
+
+private fun computeDependenciesHash(dependency: ResolvedDependency): String {
+    val allArtifactsPaths =
+        (dependency.moduleArtifacts + getAllDependencies(dependency).flatMap { it.moduleArtifacts })
+            .map { it.file.absolutePath }
+            .distinct()
+            .sortedBy { it }
+            .joinToString("|") { it }
+    val digest = MessageDigest.getInstance("SHA-256")
+    val hash = digest.digest(allArtifactsPaths.toByteArray(StandardCharsets.UTF_8))
+    return hash.toHexString()
+}
+
+fun getDependenciesCacheDirectories(
+    rootCacheDirectory: File,
+    dependency: ResolvedDependency
+): List<File>? {
+    return getAllDependencies(dependency)
+        .map { childDependency ->
+            val hasKlibs = childDependency.moduleArtifacts.any { it.file.absolutePath.endsWith(".klib") }
+            val cacheDirectory = getCacheDirectory(rootCacheDirectory, childDependency)
+            // We can only compile klib to cache if all of its dependencies are also cached.
+            if (hasKlibs && !cacheDirectory.exists())
+                return null
+            cacheDirectory
+        }
+        .filter { it.exists() }
+}
+
+fun getAllDependencies(dependency: ResolvedDependency): Set<ResolvedDependency> {
+    val allDependencies = mutableSetOf<ResolvedDependency>()
+
+    fun traverseAllDependencies(dependency: ResolvedDependency) {
+        if (dependency in allDependencies)
+            return
+        allDependencies.add(dependency)
+        dependency.children.forEach { traverseAllDependencies(it) }
+    }
+
+    dependency.children.forEach { traverseAllDependencies(it) }
+    return allDependencies
+}
+
+internal class GradleLoggerAdapter(private val gradleLogger: Logger) : org.jetbrains.kotlin.util.Logger {
+    override fun log(message: String) = gradleLogger.info(message)
+    override fun warning(message: String) = gradleLogger.warn(message)
+    override fun error(message: String) = kotlin.error(message)
+    override fun fatal(message: String): Nothing = kotlin.error(message)
+}