From 4f4fc74b16622a87567b5dd44de2b8af4966d6fa Mon Sep 17 00:00:00 2001 From: Sergey Rostov Date: Sat, 13 Apr 2019 14:41:26 +0300 Subject: [PATCH] Gradle, yarn: get recursively resolved dependencies #KT-31018 Fixed #KT-31016 Fixed #KT-31017 Fixed --- .../gradle/dsl/KotlinProjectExtension.kt | 3 + .../gradle/plugin/KotlinPluginWrapper.kt | 1 + .../targets/js/KotlinJsTargetConfigurator.kt | 2 +- .../kotlin/gradle/targets/js/npm/NpmApi.kt | 15 ++- .../gradle/targets/js/npm/NpmDependency.kt | 50 +++++++-- .../gradle/targets/js/npm/NpmResolver.kt | 5 +- .../gradle/targets/js/npm/PackageJson.kt | 6 +- .../kotlin/gradle/targets/js/yarn/Yarn.kt | 103 +++++++++++++++--- .../kotlin/gradle/targets/js/yarn/YarnLock.kt | 19 +++- .../org.jetbrains.kotlin.nodejs.properties | 1 + 10 files changed, 164 insertions(+), 41 deletions(-) create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/resources/META-INF/gradle-plugins/org.jetbrains.kotlin.nodejs.properties diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinProjectExtension.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinProjectExtension.kt index 69bbe76e342..7e085e5d360 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinProjectExtension.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinProjectExtension.kt @@ -35,6 +35,9 @@ internal fun Project.createKotlinExtension(extensionClass: KClass ) + /** + * Change contents of root's project `package.json`. + * @return true if `package.json` is required even it is empty + */ @Suppress("EXPOSED_PARAMETER_TYPE") - fun hookRootPackage(rootProject: Project, rootPackageJson: PackageJson, allWorkspaces: Collection) { + fun hookRootPackage( + rootProject: Project, + rootPackageJson: PackageJson, + allWorkspaces: Collection + ): Boolean = false - } - - open val hoistGradleNodeModules - get() = false + open fun getHoistGradleNodeModules(project: Project) = false companion object { fun resolveOperationDescription(packageManagerTitle: String): String = diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmDependency.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmDependency.kt index 8f8f6513a00..78c80e4eb83 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmDependency.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmDependency.kt @@ -10,51 +10,74 @@ import org.gradle.api.artifacts.Dependency import org.gradle.api.artifacts.FileCollectionDependency import org.gradle.api.artifacts.SelfResolvingDependency import org.gradle.api.file.FileCollection +import org.gradle.api.internal.artifacts.DependencyResolveContext +import org.gradle.api.internal.artifacts.ResolvableDependency import org.gradle.api.internal.artifacts.dependencies.SelfResolvingDependencyInternal import org.gradle.api.tasks.TaskDependency -import java.io.File import org.gradle.internal.component.local.model.DefaultLibraryBinaryIdentifier import org.jetbrains.kotlin.gradle.targets.js.npm.NpmResolver.ResolutionCallResult.* +import java.io.File data class NpmDependency( - private val project: Project, + internal val project: Project, private val org: String?, private val name: String, private val version: String ) : SelfResolvingDependency, SelfResolvingDependencyInternal, + ResolvableDependency, FileCollectionDependency { override fun getGroup(): String? = org + internal val dependencies = mutableSetOf() + override fun resolve(): MutableSet { + resolveProject() ?: return mutableSetOf() + + val all = mutableSetOf() + val visited = mutableSetOf() + + fun visit(item: NpmDependency) { + if (item in visited) return + visited.add(item) + item.tryFindNodeModule()?.let { + all.add(it) + } + } + + visit(this) + + return all + } + + internal fun resolveProject(): NpmResolver.ResolvedProject? { val result = NpmResolver.resolve(project) return when (result) { - is AlreadyInProgress -> mutableSetOf() + is AlreadyInProgress -> null is AlreadyResolved -> { check(this in result.resolution.dependencies) { "Project hierarchy is already resolved in NPM without $this" } - tryFindNodeModule() + result.resolution } is ResolvedNow -> { check(this in result.resolution.dependencies) { "$this was not visited during resolution" } - tryFindNodeModule() + result.resolution } } } - private fun tryFindNodeModule(): MutableSet { - // todo: transitive dependencies + private fun tryFindNodeModule(): File? { var p = project do { - val result = p.rootDir.resolve("node_modules/$key") - if (result.exists()) return mutableSetOf(result) - p = project.parent ?: return mutableSetOf() + val result = NpmProjectLayout[p].nodeModulesDir.resolve(key) + if (result.exists()) return result + p = project.parent ?: return null } while (true) } @@ -62,6 +85,13 @@ data class NpmDependency( override fun toString() = "$key: $version" + override fun resolve(context: DependencyResolveContext) { + resolve() + dependencies.forEach { + context.add(it) + } + } + override fun resolve(transitive: Boolean): MutableSet = resolve() override fun getFiles(): FileCollection = project.files(resolve()) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmResolver.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmResolver.kt index daa658137c8..748a895be73 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmResolver.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmResolver.kt @@ -54,10 +54,9 @@ internal class NpmResolver private constructor(val rootProject: Project) { class ResolvedNow(val resolution: ResolvedProject) : ResolutionCallResult() } - private val nodeJs = NodeJsPlugin.apply(rootProject).root - private val npmProject = rootProject.npmProject + private val nodeJs = NodeJsPlugin.apply(rootProject) private val packageManager = nodeJs.packageManager - private val hoistGradleNodeModules = npmProject.hoistGradleNodeModules + private val hoistGradleNodeModules = packageManager.getHoistGradleNodeModules(rootProject) private val npmPackages = mutableListOf() private val gson = GsonBuilder() .setPrettyPrinting() diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/PackageJson.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/PackageJson.kt index a4b81fdaee2..c4fac80b7a6 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/PackageJson.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/PackageJson.kt @@ -10,11 +10,15 @@ class PackageJson( var version: String ) { val empty: Boolean - get() = private == null && workspaces == null && dependencies.isEmpty() + get() = private == null && + workspaces == null && + dependencies.isEmpty() && + devDependencies.isEmpty() var private: Boolean? = null var workspaces: Collection? = null + val devDependencies = mutableMapOf() val dependencies = mutableMapOf() } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/Yarn.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/Yarn.kt index 3919d59b367..cb96d8b52dd 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/Yarn.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/Yarn.kt @@ -6,55 +6,119 @@ package org.jetbrains.kotlin.gradle.targets.js.yarn import org.gradle.api.Project +import org.gradle.api.internal.project.ProjectInternal +import org.gradle.internal.hash.FileHasher +import org.jetbrains.kotlin.daemon.common.toHexString import org.jetbrains.kotlin.gradle.internal.execWithProgress import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin -import org.jetbrains.kotlin.gradle.targets.js.npm.NpmApi -import org.jetbrains.kotlin.gradle.targets.js.npm.NpmProjectLayout -import org.jetbrains.kotlin.gradle.targets.js.npm.NpmResolver -import org.jetbrains.kotlin.gradle.targets.js.npm.PackageJson -import org.slf4j.LoggerFactory +import org.jetbrains.kotlin.gradle.targets.js.npm.* +import java.io.File object Yarn : NpmApi { - private val log = LoggerFactory.getLogger("org.jetbrains.kotlin.gradle.targets.js.yarn.Yarn") - override fun setup(project: Project) { - YarnPlugin[project].executeSetup() + YarnPlugin.apply(project).executeSetup() } + val Project.packageJsonHashFile: File + get() = buildDir.resolve("package.json.hash") + fun yarnExec( project: Project, description: String, vararg args: String, npmProjectLayout: NpmProjectLayout = NpmProjectLayout[project] ) { - val nodeJsEnv = NodeJsPlugin[project].buildEnv() - val yarnEnv = YarnPlugin[project].buildEnv() + val nodeJsEnv = NodeJsPlugin.apply(project).environment + val yarnEnv = YarnPlugin.apply(project).environment + + val packageJsonHashFile = project.packageJsonHashFile + val packageJsonHash = if (packageJsonHashFile.exists()) packageJsonHashFile.readText() else null + + val hasher = (project as ProjectInternal).services.get(FileHasher::class.java) + val hash = hasher.hash(npmProjectLayout.packageJsonFile).toByteArray().toHexString() + + if (packageJsonHash == hash) return + + packageJsonHashFile.writeText(hash) + + val nodeWorkDir = npmProjectLayout.nodeWorkDir project.execWithProgress(description) { exec -> exec.executable = nodeJsEnv.nodeExecutable exec.args = listOf(yarnEnv.home.resolve("bin/yarn.js").absolutePath) + args - exec.workingDir = npmProjectLayout.nodeWorkDir + exec.workingDir = nodeWorkDir + } + } + + private fun yarnLockReadTransitiveDependencies( + nodeWorkDir: File, + srcDependenciesList: Collection + ) { + val yarnLock = nodeWorkDir.resolve("yarn.lock") + if (yarnLock.isFile) { + val byKey = YarnLock.parse(yarnLock).entries.associateBy { it.key } + val visited = mutableSetOf() + + fun resolveRecursively(src: NpmDependency): NpmDependency { + if (src in visited) return src + visited.add(src) + + val key = YarnLock.key(src.key, src.version) + val deps = byKey[key] + if (deps != null) { + src.dependencies.addAll(deps.dependencies.map { dep -> + resolveRecursively( + NpmDependency( + src.project, + dep.group, + dep.packageName, + dep.version ?: "*" + ) + ) + }) + } else { + // todo: [WARN] cannot find $key in yarn.lock + } + + return src + } + + srcDependenciesList.forEach { src -> + resolveRecursively(src) + } } } @Suppress("EXPOSED_PARAMETER_TYPE") override fun resolveProject(npmPackage: NpmResolver.NpmPackage) { val project = npmPackage.project - if (!YarnPlugin[project].checkUseWorkspace()) { + if (!project.yarn.useWorkspaces) { yarnExec(project, NpmApi.resolveOperationDescription("yarn for ${project.path}")) + yarnLockReadTransitiveDependencies(NpmProjectLayout[project].nodeWorkDir, npmPackage.npmDependencies) } } @Suppress("EXPOSED_PARAMETER_TYPE") - override fun hookRootPackage(rootProject: Project, rootPackageJson: PackageJson, allWorkspaces: Collection) { - if (YarnPlugin[rootProject].checkUseWorkspace()) { + override fun hookRootPackage( + rootProject: Project, + rootPackageJson: PackageJson, + allWorkspaces: Collection + ): Boolean { + if (rootProject.yarn.useWorkspaces) { rootPackageJson.private = true rootPackageJson.workspaces = allWorkspaces .filter { it.project != rootProject } - .map { it.project.rootDir.relativeTo(rootProject.rootDir).path } + .map { it.project.projectDir.relativeTo(rootProject.rootDir).path } + + return true } + + return false } + override fun getHoistGradleNodeModules(project: Project): Boolean = + project.rootProject.yarn.useWorkspaces + @Suppress("EXPOSED_PARAMETER_TYPE") override fun resolveRootProject( rootProject: Project, @@ -62,8 +126,9 @@ object Yarn : NpmApi { ) { check(rootProject == rootProject.rootProject) - if (YarnPlugin[rootProject].checkUseWorkspace()) { + if (rootProject.yarn.useWorkspaces) { yarnExec(rootProject, NpmApi.resolveOperationDescription("yarn")) + yarnLockReadTransitiveDependencies(NpmProjectLayout[rootProject].nodeWorkDir, subprojects.flatMap { it.npmDependencies }) } else { if (subprojects.any { it.project != rootProject }) { // todo: proofread message @@ -72,11 +137,15 @@ object Yarn : NpmApi { "It is recommended to enable yarn workspaces to store common NPM dependencies in root project. " + "To enable it add this to your root project: \n" + "nodeJs { manageNodeModules = true } \n" + - "yarn { useWorkspaces = true } \n" + "Note: with `manageNodeModules` enabled, your `node_modules` and `package.json` files will be managed by " + "Gradle, will be overridden during build and should be ignored in VCS." ) } } } + + override fun cleanProject(project: Project) { + super.cleanProject(project) + project.packageJsonHashFile.delete() + } } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnLock.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnLock.kt index f52935277a4..205c4821d61 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnLock.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnLock.kt @@ -6,7 +6,6 @@ package org.jetbrains.kotlin.gradle.targets.js.yarn import com.google.gson.stream.JsonReader -import org.jetbrains.kotlin.gradle.targets.js.npm.PackageJson import java.io.File import java.io.StringReader @@ -17,11 +16,23 @@ data class YarnLock(val entries: List) { val resolved: String?, val integrity: String?, val dependencies: List - ) + ) { + val packageKey = key.substringBeforeLast("@") + } data class Dependency(val key: String, val version: String?) { - val scopedName: PackageJson.ScopedName - get() = PackageJson.scopedName(key) + val group: String? + val packageName: String + + init { + if (key.contains("/")) { + group = key.substringBeforeLast("/").removePrefix("@") + packageName = key.substringAfterLast("/") + } else { + group = null + packageName = key + } + } } companion object { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/resources/META-INF/gradle-plugins/org.jetbrains.kotlin.nodejs.properties b/libraries/tools/kotlin-gradle-plugin/src/main/resources/META-INF/gradle-plugins/org.jetbrains.kotlin.nodejs.properties new file mode 100644 index 00000000000..c5b1feac01b --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/resources/META-INF/gradle-plugins/org.jetbrains.kotlin.nodejs.properties @@ -0,0 +1 @@ +implementation-class=org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin \ No newline at end of file