azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Gradle, JS] Next step of configurstion cache

Browse Source
This commit is contained in:
Ilya Goncharov
2020-10-16 14:46:07 +03:00
committed by Alexander Likhachev
parent 19e59fe770
commit 60da9281a2
7 changed files with 64 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/ProcessedFilesCache.kt
+20 -35
View File
@@ -9,9 +9,7 @@ import com.google.gson.GsonBuilder
import com.google.gson.stream.JsonReader
import com.google.gson.stream.JsonToken
import com.google.gson.stream.JsonWriter
import org.gradle.api.Project
import org.gradle.api.internal.project.ProjectInternal
import org.gradle.internal.hash.FileHasher
import org.gradle.api.logging.Logger
import java.io.File
/**
@@ -24,14 +22,15 @@ import java.io.File
*
* @param version When updating logic in `compute`, `version` should be increased to invalidate cache
*/
// CHECK
internal open class ProcessedFilesCache(
val project: Project,
val logger: Logger,
// val hasher: FileHasher,
val projectDir: File,
val targetDir: File,
stateFileName: String,
val version: String
) : AutoCloseable {
private val hasher = (project as ProjectInternal).services.get(FileHasher::class.java)
private val gson = GsonBuilder().setPrettyPrinting().create()
private fun readFrom(json: JsonReader): State? {
val result = State()
@@ -72,7 +71,7 @@ internal open class ProcessedFilesCache(
json.name("items")
json.obj {
byHash.forEach {
json.name(it.key.contents.toHex())
json.name(it.key.toHex())
json.obj {
json.name("src").value(it.value.src)
json.name("target")
@@ -96,26 +95,12 @@ internal open class ProcessedFilesCache(
endObject()
}
fun ByteArray.toHex(): String {
val result = CharArray(size * 2) { ' ' }
var i = 0
forEach {
val n = it.toInt()
result[i++] = Character.forDigit(n shr 4 and 0xF, 16)
result[i++] = Character.forDigit(n and 0xF, 16)
}
return String(result)
fun Int.toHex(): String {
return toString(16)
}
private fun decodeHexString(hexString: String): ByteArray {
check(hexString.length % 2 == 0)
val bytes = ByteArray(hexString.length / 2)
var i = 0
var o = 0
while (i < hexString.length) {
bytes[o++] = hexToByte(hexString[i++], hexString[i++])
}
return bytes
private fun decodeHexString(hexString: String): Int {
return Integer.parseInt(hexString, 16)
}
private fun hexToByte(a: Char, b: Char): Byte = ((a.toDigit() shl 4) + b.toDigit()).toByte()
@@ -140,13 +125,13 @@ internal open class ProcessedFilesCache(
}
private class State {
val byHash = mutableMapOf<ByteArrayWrapper, Element>()
val byHash = mutableMapOf<Int, Element>()
val byTarget = mutableMapOf<String, Element>()
operator fun get(elementHash: ByteArray) = byHash[ByteArrayWrapper(elementHash)]
operator fun get(elementHash: Int) = byHash[elementHash]
operator fun set(elementHash: ByteArray, element: Element) {
byHash[ByteArrayWrapper(elementHash)] = element
operator fun set(elementHash: Int, element: Element) {
byHash[elementHash] = element
val target = element.target
if (target != null) {
byTarget[target] = element
@@ -175,9 +160,9 @@ internal open class ProcessedFilesCache(
state = (if (stateFile.exists()) {
try {
gson.newJsonReader(stateFile.reader()).use { readFrom(it) }
GsonBuilder().setPrettyPrinting().create().newJsonReader(stateFile.reader()).use { readFrom(it) }
} catch (e: Throwable) {
project.logger.warn("Cannot read $stateFile", e)
logger.warn("Cannot read $stateFile", e)
if (targetDir.exists()) {
targetDir.deleteRecursively()
}
@@ -195,19 +180,19 @@ internal open class ProcessedFilesCache(
file: File,
compute: () -> File?
): String? {
val hash = hasher.hash(file).toByteArray()
val hash = file.hashCode()
val old = state[hash]
if (old != null) {
if (checkTarget(old.target)) return old.target
else project.logger.warn("Cannot find ${File(targetDir.relativeTo(project.projectDir), old.target!!)}, rebuilding")
else logger.warn("Cannot find ${File(targetDir.relativeTo(projectDir), old.target!!)}, rebuilding")
}
val key = compute()?.relativeTo(targetDir)?.toString()
val existedTarget = state.byTarget[key]
if (key != null && existedTarget != null) {
if (!File(existedTarget.src).exists()) {
project.logger.warn("Removing cache for removed source `${existedTarget.src}`")
logger.warn("Removing cache for removed source `${existedTarget.src}`")
state.remove(existedTarget)
}
}
@@ -223,7 +208,7 @@ internal open class ProcessedFilesCache(
override fun close() {
stateFile.parentFile.mkdirs()
gson.newJsonWriter(stateFile.writer()).use {
GsonBuilder().setPrettyPrinting().create().newJsonWriter(stateFile.writer()).use {
state.writeTo(it)
}
}
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/AbstractNodeModulesCache.kt
+11 -5
View File
@@ -6,8 +6,8 @@
package org.jetbrains.kotlin.gradle.targets.js.npm
import com.google.gson.GsonBuilder
import org.gradle.api.Project
import org.gradle.api.artifacts.ResolvedDependency
import org.gradle.api.internal.project.ProjectInternal
import org.gradle.internal.hash.FileHasher
import org.jetbrains.kotlin.gradle.internal.ProcessedFilesCache
import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension
import java.io.File
@@ -15,14 +15,20 @@ import java.io.File
/**
* Cache for storing already created [GradleNodeModule]s
*/
internal abstract class AbstractNodeModulesCache(val nodeJs: NodeJsRootExtension) : AutoCloseable {
internal abstract class AbstractNodeModulesCache(nodeJs: NodeJsRootExtension) : AutoCloseable {
companion object {
const val STATE_FILE_NAME = ".visited"
}
val project: Project get() = nodeJs.rootProject
internal val dir = nodeJs.nodeModulesGradleCacheDir
private val cache = ProcessedFilesCache(project, dir, STATE_FILE_NAME, "9")
private val cache = ProcessedFilesCache(
nodeJs.rootProject.logger,
// (nodeJs.rootProject as ProjectInternal).services.get(FileHasher::class.java),
nodeJs.rootProject.projectDir,
dir,
STATE_FILE_NAME,
"9"
)
@Synchronized
fun get(
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/CompositeNodeModuleBuilder.kt
+3 -7
View File
@@ -5,17 +5,14 @@
package org.jetbrains.kotlin.gradle.targets.js.npm
import org.gradle.api.Project
import org.gradle.api.artifacts.ResolvedDependency
import java.io.File
/**
* Creates fake NodeJS module directory from given composite [dependency].
*/
internal class CompositeNodeModuleBuilder(
val project: Project,
val srcDir: File,
val cache: CompositeNodeModulesCache
val cacheDir: File
) {
var srcPackageJsonFile: File = srcDir
@@ -29,15 +26,14 @@ internal class CompositeNodeModuleBuilder(
// yarn requires semver
packageJson.version = fixSemver(packageJson.version)
val container = cache.dir
val importedPackageDir = importedPackageDir(container, packageJson.name, packageJson.version)
val importedPackageDir = importedPackageDir(cacheDir, packageJson.name, packageJson.version)
packageJson.main = srcDir.parentFile.resolve(packageJson.main!!)
.relativeToOrNull(importedPackageDir)
?.path
?: throw IllegalStateException("Unable to link composite builds for Kotlin/JS which have different roots")
return makeNodeModule(container, packageJson)
return makeNodeModule(cacheDir, packageJson)
}
}
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/CompositeNodeModulesCache.kt
+1 -1
View File
@@ -17,7 +17,7 @@ internal class CompositeNodeModulesCache(nodeJs: NodeJsRootExtension) : Abstract
version: String,
file: File
): File? {
val module = CompositeNodeModuleBuilder(project, file, this)
val module = CompositeNodeModuleBuilder(file, dir)
return module.rebuild()
}
}
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/GradleNodeModuleBuilder.kt
+8 -6
View File
@@ -5,7 +5,8 @@
package org.jetbrains.kotlin.gradle.targets.js.npm
import org.gradle.api.Project
import org.gradle.api.file.ArchiveOperations
import org.gradle.api.file.FileSystemOperations
import org.jetbrains.kotlin.gradle.targets.js.JS
import org.jetbrains.kotlin.gradle.targets.js.JS_MAP
import org.jetbrains.kotlin.gradle.targets.js.META_JS
@@ -16,11 +17,12 @@ import java.io.File
* Creates fake NodeJS module directory from given gradle [dependency].
*/
internal class GradleNodeModuleBuilder(
val project: Project,
val fs: FileSystemOperations,
val archiveOperations: ArchiveOperations,
val moduleName: String,
val moduleVersion: String,
val srcFiles: Collection<File>,
val cache: GradleNodeModulesCache
val cacheDir: File
) {
private var srcPackageJsonFile: File? = null
private val files = mutableListOf<File>()
@@ -29,7 +31,7 @@ internal class GradleNodeModuleBuilder(
srcFiles.forEach { srcFile ->
when {
isKotlinJsRuntimeFile(srcFile) -> files.add(srcFile)
srcFile.isCompatibleArchive -> project.zipTree(srcFile).forEach { innerFile ->
srcFile.isCompatibleArchive -> archiveOperations.zipTree(srcFile).forEach { innerFile ->
when {
innerFile.name == NpmProject.PACKAGE_JSON -> srcPackageJsonFile = innerFile
isKotlinJsRuntimeFile(innerFile) -> files.add(innerFile)
@@ -61,8 +63,8 @@ internal class GradleNodeModuleBuilder(
val actualFiles = files.filterNot { it.name.endsWith(".$META_JS") }
return makeNodeModule(cache.dir, packageJson) { nodeModule ->
project.copy { copy ->
return makeNodeModule(cacheDir, packageJson) { nodeModule ->
fs.copy { copy ->
copy.from(actualFiles)
copy.into(nodeModule)
}
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/GradleNodeModulesCache.kt
+7 -2
View File
@@ -5,7 +5,9 @@
package org.jetbrains.kotlin.gradle.targets.js.npm
import org.gradle.api.artifacts.ResolvedDependency
import org.gradle.api.file.ArchiveOperations
import org.gradle.api.file.FileSystemOperations
import org.gradle.api.internal.project.ProjectInternal
import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension
import java.io.File
@@ -13,12 +15,15 @@ import java.io.File
* Cache for storing already created [GradleNodeModule]s
*/
internal class GradleNodeModulesCache(nodeJs: NodeJsRootExtension) : AbstractNodeModulesCache(nodeJs) {
private val fs = (nodeJs.rootProject as ProjectInternal).services.get(FileSystemOperations::class.java)
private val archiveOperations = (nodeJs.rootProject as ProjectInternal).services.get(ArchiveOperations::class.java)
override fun buildImportedPackage(
name: String,
version: String,
file: File
): File? {
val module = GradleNodeModuleBuilder(project, name, version, listOf(file), this)
val module = GradleNodeModuleBuilder(fs, archiveOperations, name, version, listOf(file), dir)
module.visitArtifacts()
return module.rebuild()
}
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/resolver/KotlinCompilationNpmResolver.kt
+14 -6
View File
@@ -51,6 +51,14 @@ internal class KotlinCompilationNpmResolver(
@Transient
val resolver = projectResolver.resolver
private val gradleNodeModules by lazy {
resolver.gradleNodeModules
}
private val compositeNodeModules by lazy {
resolver.compositeNodeModules
}
@Transient
val npmProject = compilation.npmProject
@@ -365,13 +373,13 @@ internal class KotlinCompilationNpmResolver(
?: error("Unresolved dependent npm package: ${this@KotlinCompilationNpmResolver} -> $it")
}
val importedExternalGradleDependencies = fileExternalGradleDependencies.mapNotNull {
resolver.gradleNodeModules.get(it.dependencyName, it.dependencyVersion, it.file)
gradleNodeModules.get(it.dependencyName, it.dependencyVersion, it.file)
} + fileCollectionDependencies.flatMap { dependency ->
dependency.files
// Gradle can hash with FileHasher only files and only existed files
.filter { it.isFile }
.map { file ->
resolver.gradleNodeModules.get(
gradleNodeModules.get(
file.name,
dependency.version ?: "0.0.1",
file
@@ -382,7 +390,7 @@ internal class KotlinCompilationNpmResolver(
val compositeDependencies = internalCompositeDependencies.flatMap { dependency ->
dependency.getPackages()
.map { file ->
resolver.compositeNodeModules.get(
compositeNodeModules.get(
dependency.dependency.moduleName,
dependency.dependency.moduleVersion,
file
@@ -391,10 +399,10 @@ internal class KotlinCompilationNpmResolver(
}
.filterNotNull()
val toolsNpmDependencies = nodeJs.taskRequirements
.getCompilationNpmRequirements(compilation)
// val toolsNpmDependencies = nodeJs.taskRequirements
// .getCompilationNpmRequirements(compilation)
val allNpmDependencies = externalNpmDependencies + toolsNpmDependencies
val allNpmDependencies = externalNpmDependencies/* + toolsNpmDependencies*/
val packageJson = packageJson(
npmProject,