From 60da9281a2ca3f48d0d993c4ff144a13c984a9a2 Mon Sep 17 00:00:00 2001 From: Ilya Goncharov Date: Fri, 16 Oct 2020 14:46:07 +0300 Subject: [PATCH] [Gradle, JS] Next step of configurstion cache --- .../gradle/internal/ProcessedFilesCache.kt | 55 +++++++------------ .../js/npm/AbstractNodeModulesCache.kt | 16 ++++-- .../js/npm/CompositeNodeModuleBuilder.kt | 10 +--- .../js/npm/CompositeNodeModulesCache.kt | 2 +- .../targets/js/npm/GradleNodeModuleBuilder.kt | 14 +++-- .../targets/js/npm/GradleNodeModulesCache.kt | 9 ++- .../resolver/KotlinCompilationNpmResolver.kt | 20 +++++-- 7 files changed, 64 insertions(+), 62 deletions(-) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/ProcessedFilesCache.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/ProcessedFilesCache.kt index f039f0d7f59..d45053abca8 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/ProcessedFilesCache.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/ProcessedFilesCache.kt @@ -9,9 +9,7 @@ import com.google.gson.GsonBuilder import com.google.gson.stream.JsonReader import com.google.gson.stream.JsonToken import com.google.gson.stream.JsonWriter -import org.gradle.api.Project -import org.gradle.api.internal.project.ProjectInternal -import org.gradle.internal.hash.FileHasher +import org.gradle.api.logging.Logger import java.io.File /** @@ -24,14 +22,15 @@ import java.io.File * * @param version When updating logic in `compute`, `version` should be increased to invalidate cache */ +// CHECK internal open class ProcessedFilesCache( - val project: Project, + val logger: Logger, +// val hasher: FileHasher, + val projectDir: File, val targetDir: File, stateFileName: String, val version: String ) : AutoCloseable { - private val hasher = (project as ProjectInternal).services.get(FileHasher::class.java) - private val gson = GsonBuilder().setPrettyPrinting().create() private fun readFrom(json: JsonReader): State? { val result = State() @@ -72,7 +71,7 @@ internal open class ProcessedFilesCache( json.name("items") json.obj { byHash.forEach { - json.name(it.key.contents.toHex()) + json.name(it.key.toHex()) json.obj { json.name("src").value(it.value.src) json.name("target") @@ -96,26 +95,12 @@ internal open class ProcessedFilesCache( endObject() } - fun ByteArray.toHex(): String { - val result = CharArray(size * 2) { ' ' } - var i = 0 - forEach { - val n = it.toInt() - result[i++] = Character.forDigit(n shr 4 and 0xF, 16) - result[i++] = Character.forDigit(n and 0xF, 16) - } - return String(result) + fun Int.toHex(): String { + return toString(16) } - private fun decodeHexString(hexString: String): ByteArray { - check(hexString.length % 2 == 0) - val bytes = ByteArray(hexString.length / 2) - var i = 0 - var o = 0 - while (i < hexString.length) { - bytes[o++] = hexToByte(hexString[i++], hexString[i++]) - } - return bytes + private fun decodeHexString(hexString: String): Int { + return Integer.parseInt(hexString, 16) } private fun hexToByte(a: Char, b: Char): Byte = ((a.toDigit() shl 4) + b.toDigit()).toByte() @@ -140,13 +125,13 @@ internal open class ProcessedFilesCache( } private class State { - val byHash = mutableMapOf() + val byHash = mutableMapOf() val byTarget = mutableMapOf() - operator fun get(elementHash: ByteArray) = byHash[ByteArrayWrapper(elementHash)] + operator fun get(elementHash: Int) = byHash[elementHash] - operator fun set(elementHash: ByteArray, element: Element) { - byHash[ByteArrayWrapper(elementHash)] = element + operator fun set(elementHash: Int, element: Element) { + byHash[elementHash] = element val target = element.target if (target != null) { byTarget[target] = element @@ -175,9 +160,9 @@ internal open class ProcessedFilesCache( state = (if (stateFile.exists()) { try { - gson.newJsonReader(stateFile.reader()).use { readFrom(it) } + GsonBuilder().setPrettyPrinting().create().newJsonReader(stateFile.reader()).use { readFrom(it) } } catch (e: Throwable) { - project.logger.warn("Cannot read $stateFile", e) + logger.warn("Cannot read $stateFile", e) if (targetDir.exists()) { targetDir.deleteRecursively() } @@ -195,19 +180,19 @@ internal open class ProcessedFilesCache( file: File, compute: () -> File? ): String? { - val hash = hasher.hash(file).toByteArray() + val hash = file.hashCode() val old = state[hash] if (old != null) { if (checkTarget(old.target)) return old.target - else project.logger.warn("Cannot find ${File(targetDir.relativeTo(project.projectDir), old.target!!)}, rebuilding") + else logger.warn("Cannot find ${File(targetDir.relativeTo(projectDir), old.target!!)}, rebuilding") } val key = compute()?.relativeTo(targetDir)?.toString() val existedTarget = state.byTarget[key] if (key != null && existedTarget != null) { if (!File(existedTarget.src).exists()) { - project.logger.warn("Removing cache for removed source `${existedTarget.src}`") + logger.warn("Removing cache for removed source `${existedTarget.src}`") state.remove(existedTarget) } } @@ -223,7 +208,7 @@ internal open class ProcessedFilesCache( override fun close() { stateFile.parentFile.mkdirs() - gson.newJsonWriter(stateFile.writer()).use { + GsonBuilder().setPrettyPrinting().create().newJsonWriter(stateFile.writer()).use { state.writeTo(it) } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/AbstractNodeModulesCache.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/AbstractNodeModulesCache.kt index d599aa7b047..712905f6a73 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/AbstractNodeModulesCache.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/AbstractNodeModulesCache.kt @@ -6,8 +6,8 @@ package org.jetbrains.kotlin.gradle.targets.js.npm import com.google.gson.GsonBuilder -import org.gradle.api.Project -import org.gradle.api.artifacts.ResolvedDependency +import org.gradle.api.internal.project.ProjectInternal +import org.gradle.internal.hash.FileHasher import org.jetbrains.kotlin.gradle.internal.ProcessedFilesCache import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension import java.io.File @@ -15,14 +15,20 @@ import java.io.File /** * Cache for storing already created [GradleNodeModule]s */ -internal abstract class AbstractNodeModulesCache(val nodeJs: NodeJsRootExtension) : AutoCloseable { +internal abstract class AbstractNodeModulesCache(nodeJs: NodeJsRootExtension) : AutoCloseable { companion object { const val STATE_FILE_NAME = ".visited" } - val project: Project get() = nodeJs.rootProject internal val dir = nodeJs.nodeModulesGradleCacheDir - private val cache = ProcessedFilesCache(project, dir, STATE_FILE_NAME, "9") + private val cache = ProcessedFilesCache( + nodeJs.rootProject.logger, +// (nodeJs.rootProject as ProjectInternal).services.get(FileHasher::class.java), + nodeJs.rootProject.projectDir, + dir, + STATE_FILE_NAME, + "9" + ) @Synchronized fun get( diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/CompositeNodeModuleBuilder.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/CompositeNodeModuleBuilder.kt index ab2067a4261..5ddecea9b26 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/CompositeNodeModuleBuilder.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/CompositeNodeModuleBuilder.kt @@ -5,17 +5,14 @@ package org.jetbrains.kotlin.gradle.targets.js.npm -import org.gradle.api.Project -import org.gradle.api.artifacts.ResolvedDependency import java.io.File /** * Creates fake NodeJS module directory from given composite [dependency]. */ internal class CompositeNodeModuleBuilder( - val project: Project, val srcDir: File, - val cache: CompositeNodeModulesCache + val cacheDir: File ) { var srcPackageJsonFile: File = srcDir @@ -29,15 +26,14 @@ internal class CompositeNodeModuleBuilder( // yarn requires semver packageJson.version = fixSemver(packageJson.version) - val container = cache.dir - val importedPackageDir = importedPackageDir(container, packageJson.name, packageJson.version) + val importedPackageDir = importedPackageDir(cacheDir, packageJson.name, packageJson.version) packageJson.main = srcDir.parentFile.resolve(packageJson.main!!) .relativeToOrNull(importedPackageDir) ?.path ?: throw IllegalStateException("Unable to link composite builds for Kotlin/JS which have different roots") - return makeNodeModule(container, packageJson) + return makeNodeModule(cacheDir, packageJson) } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/CompositeNodeModulesCache.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/CompositeNodeModulesCache.kt index f95681dd59a..e03b844b2c6 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/CompositeNodeModulesCache.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/CompositeNodeModulesCache.kt @@ -17,7 +17,7 @@ internal class CompositeNodeModulesCache(nodeJs: NodeJsRootExtension) : Abstract version: String, file: File ): File? { - val module = CompositeNodeModuleBuilder(project, file, this) + val module = CompositeNodeModuleBuilder(file, dir) return module.rebuild() } } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/GradleNodeModuleBuilder.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/GradleNodeModuleBuilder.kt index 2fa2385c8d5..14fac9f9a86 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/GradleNodeModuleBuilder.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/GradleNodeModuleBuilder.kt @@ -5,7 +5,8 @@ package org.jetbrains.kotlin.gradle.targets.js.npm -import org.gradle.api.Project +import org.gradle.api.file.ArchiveOperations +import org.gradle.api.file.FileSystemOperations import org.jetbrains.kotlin.gradle.targets.js.JS import org.jetbrains.kotlin.gradle.targets.js.JS_MAP import org.jetbrains.kotlin.gradle.targets.js.META_JS @@ -16,11 +17,12 @@ import java.io.File * Creates fake NodeJS module directory from given gradle [dependency]. */ internal class GradleNodeModuleBuilder( - val project: Project, + val fs: FileSystemOperations, + val archiveOperations: ArchiveOperations, val moduleName: String, val moduleVersion: String, val srcFiles: Collection, - val cache: GradleNodeModulesCache + val cacheDir: File ) { private var srcPackageJsonFile: File? = null private val files = mutableListOf() @@ -29,7 +31,7 @@ internal class GradleNodeModuleBuilder( srcFiles.forEach { srcFile -> when { isKotlinJsRuntimeFile(srcFile) -> files.add(srcFile) - srcFile.isCompatibleArchive -> project.zipTree(srcFile).forEach { innerFile -> + srcFile.isCompatibleArchive -> archiveOperations.zipTree(srcFile).forEach { innerFile -> when { innerFile.name == NpmProject.PACKAGE_JSON -> srcPackageJsonFile = innerFile isKotlinJsRuntimeFile(innerFile) -> files.add(innerFile) @@ -61,8 +63,8 @@ internal class GradleNodeModuleBuilder( val actualFiles = files.filterNot { it.name.endsWith(".$META_JS") } - return makeNodeModule(cache.dir, packageJson) { nodeModule -> - project.copy { copy -> + return makeNodeModule(cacheDir, packageJson) { nodeModule -> + fs.copy { copy -> copy.from(actualFiles) copy.into(nodeModule) } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/GradleNodeModulesCache.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/GradleNodeModulesCache.kt index 410e19df075..9f350db323f 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/GradleNodeModulesCache.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/GradleNodeModulesCache.kt @@ -5,7 +5,9 @@ package org.jetbrains.kotlin.gradle.targets.js.npm -import org.gradle.api.artifacts.ResolvedDependency +import org.gradle.api.file.ArchiveOperations +import org.gradle.api.file.FileSystemOperations +import org.gradle.api.internal.project.ProjectInternal import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension import java.io.File @@ -13,12 +15,15 @@ import java.io.File * Cache for storing already created [GradleNodeModule]s */ internal class GradleNodeModulesCache(nodeJs: NodeJsRootExtension) : AbstractNodeModulesCache(nodeJs) { + private val fs = (nodeJs.rootProject as ProjectInternal).services.get(FileSystemOperations::class.java) + private val archiveOperations = (nodeJs.rootProject as ProjectInternal).services.get(ArchiveOperations::class.java) + override fun buildImportedPackage( name: String, version: String, file: File ): File? { - val module = GradleNodeModuleBuilder(project, name, version, listOf(file), this) + val module = GradleNodeModuleBuilder(fs, archiveOperations, name, version, listOf(file), dir) module.visitArtifacts() return module.rebuild() } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/resolver/KotlinCompilationNpmResolver.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/resolver/KotlinCompilationNpmResolver.kt index 8c56410ae00..b3c3c495a51 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/resolver/KotlinCompilationNpmResolver.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/resolver/KotlinCompilationNpmResolver.kt @@ -51,6 +51,14 @@ internal class KotlinCompilationNpmResolver( @Transient val resolver = projectResolver.resolver + private val gradleNodeModules by lazy { + resolver.gradleNodeModules + } + + private val compositeNodeModules by lazy { + resolver.compositeNodeModules + } + @Transient val npmProject = compilation.npmProject @@ -365,13 +373,13 @@ internal class KotlinCompilationNpmResolver( ?: error("Unresolved dependent npm package: ${this@KotlinCompilationNpmResolver} -> $it") } val importedExternalGradleDependencies = fileExternalGradleDependencies.mapNotNull { - resolver.gradleNodeModules.get(it.dependencyName, it.dependencyVersion, it.file) + gradleNodeModules.get(it.dependencyName, it.dependencyVersion, it.file) } + fileCollectionDependencies.flatMap { dependency -> dependency.files // Gradle can hash with FileHasher only files and only existed files .filter { it.isFile } .map { file -> - resolver.gradleNodeModules.get( + gradleNodeModules.get( file.name, dependency.version ?: "0.0.1", file @@ -382,7 +390,7 @@ internal class KotlinCompilationNpmResolver( val compositeDependencies = internalCompositeDependencies.flatMap { dependency -> dependency.getPackages() .map { file -> - resolver.compositeNodeModules.get( + compositeNodeModules.get( dependency.dependency.moduleName, dependency.dependency.moduleVersion, file @@ -391,10 +399,10 @@ internal class KotlinCompilationNpmResolver( } .filterNotNull() - val toolsNpmDependencies = nodeJs.taskRequirements - .getCompilationNpmRequirements(compilation) +// val toolsNpmDependencies = nodeJs.taskRequirements +// .getCompilationNpmRequirements(compilation) - val allNpmDependencies = externalNpmDependencies + toolsNpmDependencies + val allNpmDependencies = externalNpmDependencies/* + toolsNpmDependencies*/ val packageJson = packageJson( npmProject,