[Gradle, JS] Use file hasher for ProcessedFilesCache

Gradle's FileHasher can cache calculated hashes saving execution time #KT-45774 In Progress
2021-04-02 17:29:23 +03:00
parent 6e333c4f3f
commit 4416f2eb3f
8 changed files with 62 additions and 26 deletions
@@ -9,9 +9,8 @@ import com.google.gson.GsonBuilder
 import com.google.gson.stream.JsonReader
 import com.google.gson.stream.JsonToken
 import com.google.gson.stream.JsonWriter
-import org.gradle.api.logging.Logger
+import org.gradle.internal.hash.FileHasher
 import java.io.File
-import java.security.MessageDigest

 /**
 * Cache for preventing processing some files twice.
@@ -24,6 +23,7 @@ import java.security.MessageDigest
 * @param version When updating logic in `compute`, `version` should be increased to invalidate cache
 */
 internal open class ProcessedFilesCache(
+    val fileHasher: FileHasher,
    val projectDir: File,
    val targetDir: File,
    stateFileName: String,
@@ -192,19 +192,7 @@ internal open class ProcessedFilesCache(
        file: File,
        compute: () -> File?
    ): String? {
-        val md = MessageDigest.getInstance("MD5")
-        val buffer = ByteArray(4048)
-        file.inputStream().use { input ->
-            while (true) {
-                val len = input.read(buffer)
-                if (len < 0) {
-                    break
-                }
-                md.update(buffer, 0, len)
-            }
-        }
-
-        val hash = md.digest()
+        val hash = fileHasher.hash(file).toByteArray()
        val old = state[hash]

        if (old != null) {
@@ -16,6 +16,7 @@ import org.jetbrains.kotlin.gradle.targets.js.npm.KotlinNpmResolutionManager
 import org.jetbrains.kotlin.gradle.targets.js.npm.NpmApi
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolver.PACKAGE_JSON_UMBRELLA_TASK_NAME
 import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinNpmInstallTask
+import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinNpmCachesSetup
 import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.RootPackageJsonTask
 import org.jetbrains.kotlin.gradle.targets.js.yarn.Yarn
 import org.jetbrains.kotlin.gradle.tasks.internal.CleanableStore
@@ -66,6 +67,9 @@ open class NodeJsRootExtension(@Transient val rootProject: Project) : Configurat
    val rootPackageJsonTaskProvider: TaskProvider<RootPackageJsonTask>?
        get() = rootProject?.tasks?.withType(RootPackageJsonTask::class.java)?.named(RootPackageJsonTask.NAME)

+    val npmCachesSetupTaskProvider: TaskProvider<out KotlinNpmCachesSetup>?
+        get() = rootProject?.tasks?.withType(KotlinNpmCachesSetup::class.java)?.named(KotlinNpmCachesSetup.NAME)
+
    val rootPackageDir: File by lazy {
        rootProject.buildDir.resolve("js")
    }
@@ -13,6 +13,7 @@ import org.jetbrains.kotlin.gradle.targets.js.MultiplePluginDeclarationDetector
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension.Companion.EXTENSION_NAME
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolver.PACKAGE_JSON_UMBRELLA_TASK_NAME
 import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinNpmInstallTask
+import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinNpmCachesSetup
 import org.jetbrains.kotlin.gradle.targets.js.yarn.YarnPlugin
 import org.jetbrains.kotlin.gradle.tasks.CleanDataTask
 import org.jetbrains.kotlin.gradle.tasks.registerTask
@@ -48,6 +49,11 @@ open class NodeJsRootPlugin : Plugin<Project> {
            it.mustRunAfter(rootClean)
        }

+        registerTask<KotlinNpmCachesSetup>(KotlinNpmCachesSetup.NAME) {
+            it.group = TASKS_GROUP_NAME
+            it.description = "Setup file hasher for caches"
+        }
+
        registerTask<Task>(PACKAGE_JSON_UMBRELLA_TASK_NAME)

        YarnPlugin.apply(project)
@@ -7,9 +7,9 @@ package org.jetbrains.kotlin.gradle.targets.js.npm

 import com.google.gson.GsonBuilder
 import org.gradle.api.file.DirectoryProperty
-import org.gradle.api.invocation.Gradle
 import org.gradle.api.services.BuildService
 import org.gradle.api.services.BuildServiceParameters
+import org.gradle.internal.hash.FileHasher
 import org.jetbrains.kotlin.gradle.internal.ProcessedFilesCache
 import java.io.File

@@ -26,12 +26,17 @@ internal abstract class AbstractNodeModulesCache : AutoCloseable, BuildService<A
        const val STATE_FILE_NAME = ".visited"
    }

-    private val cache = ProcessedFilesCache(
-        parameters.rootProjectDir.get().asFile,
-        parameters.cacheDir.get().asFile,
-        STATE_FILE_NAME,
-        "9"
-    )
+    lateinit var fileHasher: FileHasher
+
+    private val cache by lazy {
+        ProcessedFilesCache(
+            fileHasher,
+            parameters.rootProjectDir.get().asFile,
+            parameters.cacheDir.get().asFile,
+            STATE_FILE_NAME,
+            "9"
+        )
+    }

    @Synchronized
    fun get(
@@ -6,7 +6,6 @@
 package org.jetbrains.kotlin.gradle.targets.js.npm

 import org.gradle.api.Incubating
-import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.internal.project.ProjectInternal
 import org.gradle.api.logging.Logger
@@ -14,7 +14,6 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.isMain
 import org.jetbrains.kotlin.gradle.targets.js.dukat.DukatRootResolverPlugin
 import org.jetbrains.kotlin.gradle.targets.js.ir.KotlinJsIrCompilation
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension
-import org.jetbrains.kotlin.gradle.targets.js.nodejs.TasksRequirements
 import org.jetbrains.kotlin.gradle.targets.js.npm.CompositeNodeModulesCache
 import org.jetbrains.kotlin.gradle.targets.js.npm.GradleNodeModulesCache
 import org.jetbrains.kotlin.gradle.targets.js.npm.KotlinNpmResolutionManager
@@ -69,7 +68,7 @@ internal class KotlinRootNpmResolver internal constructor(
    private val archiveOperations by lazy { ArchiveOperationsCompat(rootProject) }
    private val fs by lazy { FileSystemOperationsCompat(rootProject) }

-    private val gradleNodeModulesProvider: Provider<GradleNodeModulesCache> =
+    internal val gradleNodeModulesProvider: Provider<GradleNodeModulesCache> =
        rootProject.gradle.sharedServices.registerIfAbsent("gradle-node-modules", GradleNodeModulesCache::class.java) {
            it.parameters.cacheDir.set(nodeJs.nodeModulesGradleCacheDir)
            it.parameters.rootProjectDir.set(rootProject.projectDir)
@@ -81,7 +80,7 @@ internal class KotlinRootNpmResolver internal constructor(
            it.fs = fs
        }

-    private val compositeNodeModulesProvider: Provider<CompositeNodeModulesCache> =
+    internal val compositeNodeModulesProvider: Provider<CompositeNodeModulesCache> =
        rootProject.gradle.sharedServices.registerIfAbsent("composite-node-modules", CompositeNodeModulesCache::class.java) {
            it.parameters.cacheDir.set(nodeJs.nodeModulesGradleCacheDir)
            it.parameters.rootProjectDir.set(rootProject.projectDir)
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.targets.js.npm.tasks
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.tasks.TaskAction
+import org.gradle.internal.hash.FileHasher
+import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootPlugin
+import javax.inject.Inject
+
+abstract class KotlinNpmCachesSetup : DefaultTask() {
+    @get:Inject
+    open val fileHasher: FileHasher
+        get() = throw UnsupportedOperationException()
+
+    @Transient
+    private val nodeJs = NodeJsRootPlugin.apply(project.rootProject)
+    private val gradleNodeModules = nodeJs.npmResolutionManager.resolver.gradleNodeModulesProvider
+    private val compositeNodeModules = nodeJs.npmResolutionManager.resolver.compositeNodeModulesProvider
+
+    @TaskAction
+    fun setup() {
+        gradleNodeModules.get().fileHasher = fileHasher
+        compositeNodeModules.get().fileHasher = fileHasher
+    }
+
+    companion object {
+        const val NAME = "kotlinNpmCachesSetup"
+    }
+}
@@ -96,6 +96,7 @@ open class KotlinPackageJsonTask : DefaultTask() {

            val rootClean = project.rootProject.tasks.named(BasePlugin.CLEAN_TASK_NAME)
            val npmInstallTask = nodeJs.npmInstallTaskProvider
+            val npmCachesSetupTask = nodeJs.npmCachesSetupTaskProvider
            val packageJsonTaskName = npmProject.packageJsonTaskName
            val packageJsonUmbrella = nodeJs.packageJsonUmbrellaTaskProvider
            val packageJsonTask = project.registerTask<KotlinPackageJsonTask>(packageJsonTaskName) { task ->
@@ -105,6 +106,7 @@ open class KotlinPackageJsonTask : DefaultTask() {
                task.group = NodeJsRootPlugin.TASKS_GROUP_NAME

                task.dependsOn(target.project.provider { task.findDependentTasks() })
+                task.dependsOn(npmCachesSetupTask)
                task.mustRunAfter(rootClean)
            }
            packageJsonUmbrella.configure { task ->