From 4c2c8a1788e61c3d145bdb10c7641ca5af7ecf2c Mon Sep 17 00:00:00 2001 From: Sergey Rostov Date: Wed, 10 Apr 2019 10:03:58 +0300 Subject: [PATCH] Gradle, npm: disable manageNodeModules by default #KT-30530 --- .../jetbrains/kotlin/gradle/internal/exec.kt | 43 ++++ .../kotlin/gradle/internal/progress.kt | 27 ++ .../KotlinJsCompilationTestsConfigurator.kt | 13 +- .../targets/js/nodejs/NodeJsExtension.kt | 11 +- .../targets/js/nodejs/NodeJsPlatform.kt | 2 +- .../gradle/targets/js/nodejs/NodeJsPlugin.kt | 23 +- .../targets/js/nodejs/NodeJsRootExtension.kt | 57 +++-- .../targets/js/nodejs/NodeJsSetupTask.kt | 8 +- .../kotlin/gradle/targets/js/nodejs/utils.kt | 13 +- .../targets/js/npm/GradleNodeModulesSync.kt | 241 ++++++++++++++++++ .../js/npm/NodeModulesGradleComponents.kt | 148 ----------- .../kotlin/gradle/targets/js/npm/Npm.kt | 19 -- .../kotlin/gradle/targets/js/npm/NpmApi.kt | 7 - .../gradle/targets/js/npm/NpmDependency.kt | 49 +--- .../gradle/targets/js/npm/NpmProjectLayout.kt | 35 +++ .../gradle/targets/js/npm/NpmResolver.kt | 133 +++++----- .../gradle/targets/js/npm/PackageJson.kt | 25 +- .../js/tasks/KotlinJsNodeModulesTask.kt | 63 ----- .../targets/js/tasks/KotlinNodeJsTestTask.kt | 23 +- .../kotlin/gradle/targets/js/yarn/Yarn.kt | 112 ++------ .../gradle/targets/js/yarn/YarnExtension.kt | 46 ---- .../gradle/targets/js/yarn/YarnPlugin.kt | 6 +- .../targets/js/yarn/YarnRootExtension.kt | 41 +-- .../gradle/targets/js/yarn/YarnSetupTask.kt | 4 +- 24 files changed, 541 insertions(+), 608 deletions(-) create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/exec.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/progress.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/GradleNodeModulesSync.kt delete mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NodeModulesGradleComponents.kt delete mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/Npm.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmProjectLayout.kt delete mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/tasks/KotlinJsNodeModulesTask.kt delete mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnExtension.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/exec.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/exec.kt new file mode 100644 index 00000000000..4e46125f480 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/exec.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.internal + +import org.gradle.api.Project +import org.gradle.api.internal.project.ProjectInternal +import org.gradle.process.internal.ExecAction +import org.gradle.process.internal.ExecActionFactory +import java.io.ByteArrayOutputStream +import java.io.PipedInputStream +import java.io.PipedOutputStream +import kotlin.concurrent.thread + +internal fun Project.execWithProgress(description: String, body: (ExecAction) -> Unit) { + this as ProjectInternal + + val stderr = ByteArrayOutputStream() + val stdout = StringBuilder() + val stdInPipe = PipedInputStream() + val exec = services.get(ExecActionFactory::class.java).newExecAction() + body(exec) + project.operation(description) { + exec.errorOutput = stderr + exec.standardOutput = PipedOutputStream(stdInPipe) + val outputReaderThread = thread(name = "output reader for [$description]") { + stdInPipe.reader().useLines { lines -> + lines.forEach { + stdout.appendln(it) + progress(it) + } + } + } + exec.isIgnoreExitValue = true + val result = exec.execute() + outputReaderThread.join() + if (result.exitValue != 0) { + error(stderr.toString() + "\n" + stdout) + } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/progress.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/progress.kt new file mode 100644 index 00000000000..a43c9fdaec6 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/progress.kt @@ -0,0 +1,27 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.internal + +import org.gradle.api.Project +import org.gradle.api.internal.project.ProjectInternal +import org.gradle.internal.logging.events.ProgressStartEvent +import org.gradle.internal.logging.progress.ProgressLogger + +fun Project.operation( + description: String, + initialStatus: String? = null, + body: ProgressLogger.() -> Unit +) { + val services = (project as ProjectInternal).services + val progressFactory = services.get(org.gradle.internal.logging.progress.ProgressLoggerFactory::class.java) + val operation = progressFactory.newOperation(ProgressStartEvent.BUILD_OP_CATEGORY) + operation.start(description, initialStatus) + try { + operation.body() + } finally { + operation.completed() + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/KotlinJsCompilationTestsConfigurator.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/KotlinJsCompilationTestsConfigurator.kt index 0fafea60d0d..baad4e0a76c 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/KotlinJsCompilationTestsConfigurator.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/KotlinJsCompilationTestsConfigurator.kt @@ -3,10 +3,9 @@ package org.jetbrains.kotlin.gradle.targets.js import org.gradle.api.file.FileCollection import org.gradle.language.base.plugins.LifecycleBasePlugin import org.jetbrains.kotlin.gradle.plugin.KotlinCompilationToRunnableFiles -import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsExtension import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin -import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsSetupTask -import org.jetbrains.kotlin.gradle.targets.js.tasks.* +import org.jetbrains.kotlin.gradle.targets.js.npm.NpmProjectLayout +import org.jetbrains.kotlin.gradle.targets.js.tasks.KotlinNodeJsTestTask import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile import org.jetbrains.kotlin.gradle.tasks.registerTask import org.jetbrains.kotlin.gradle.testing.internal.configureConventions @@ -51,7 +50,7 @@ internal class KotlinJsCompilationTestsConfigurator( implementation(kotlin("test-nodejs-runner")) } - val projectWithNodeJsPlugin = NodeJsPlugin.ensureAppliedInHierarchy(target.project) + val nodeJs = NodeJsPlugin[target.project] val testTask = registerTask(project, testTaskName, KotlinNodeJsTestTask::class.java) { testJs -> testJs.group = LifecycleBasePlugin.VERIFICATION_GROUP @@ -68,8 +67,6 @@ internal class KotlinJsCompilationTestsConfigurator( } testJs.nodeJsProcessOptions.workingDir = project.projectDir - - testJs.nodeModulesDir = project.rootDir.resolve("node_modules") testJs.testRuntimeNodeModules = listOf( "kotlin-test-nodejs-runner.js", "kotlin-nodejs-source-map-support.js" @@ -83,11 +80,11 @@ internal class KotlinJsCompilationTestsConfigurator( project.afterEvaluate { // defer nodeJs executable setup, as nodejs project settings may change during configuration testTask.configure { - val nodeJsSetupTask = projectWithNodeJsPlugin.tasks.findByName(NodeJsSetupTask.NAME) + val nodeJsSetupTask = nodeJs.nodeJsSetupTask it.dependsOn(nodeJsSetupTask) if (it.nodeJsProcessOptions.executable == null) { - it.nodeJsProcessOptions.executable = NodeJsExtension[projectWithNodeJsPlugin].buildEnv().nodeExec + it.nodeJsProcessOptions.executable = nodeJs.buildEnv().nodeExec } } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsExtension.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsExtension.kt index 4479d8a7b1b..336ae386247 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsExtension.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsExtension.kt @@ -9,23 +9,16 @@ import org.gradle.api.Project import org.jetbrains.kotlin.gradle.targets.js.npm.PackageJson open class NodeJsExtension(val project: Project) { - val root: NodeJsRootExtension - get() = this as? NodeJsRootExtension ?: NodeJsRootExtension[project] - internal val packageJsonHandlers = mutableListOf Unit>() - @Suppress("unused") fun packageJson(handler: PackageJson.() -> Unit) { packageJsonHandlers.add(handler) } companion object { operator fun get(project: Project): NodeJsExtension { - NodeJsPlugin.apply(project.rootProject) + NodeJsPlugin[project.rootProject] return project.extensions.getByName(NodeJsRootExtension.NODE_JS) as NodeJsExtension } } -} - -val Project.nodeJs: NodeJsExtension - get() = NodeJsExtension[this] \ No newline at end of file +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsPlatform.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsPlatform.kt index 43a33bd2fc2..7afea8819c5 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsPlatform.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsPlatform.kt @@ -2,7 +2,7 @@ package org.jetbrains.kotlin.gradle.targets.js.nodejs /** * Provides platform and architecture names that is used to download NodeJs. - * See [NodeJsEnv.ivyDependency] that is filled in [NodeJsRootExtension.environment]. + * See [NodeJsEnv.ivyDependency] that is filled in [NodeJsRootExtension.buildEnv]. */ internal object NodeJsPlatform { private val props = System.getProperties() diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsPlugin.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsPlugin.kt index 6315cdc6a77..74b386643ca 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsPlugin.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsPlugin.kt @@ -4,7 +4,10 @@ import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.plugins.BasePlugin import org.gradle.api.tasks.Delete +import org.jetbrains.kotlin.backend.common.onlyIf import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension.Companion.NODE_JS +import org.jetbrains.kotlin.gradle.targets.js.npm.NpmProjectLayout +import org.jetbrains.kotlin.gradle.targets.js.npm.NpmResolver open class NodeJsPlugin : Plugin { override fun apply(project: Project): Unit = project.run { @@ -12,10 +15,10 @@ open class NodeJsPlugin : Plugin { "NodeJsPlugin can be applied only to root project" } - this.extensions.create(NODE_JS, NodeJsRootExtension::class.java, this) + val nodeJs = this.extensions.create(NODE_JS, NodeJsRootExtension::class.java, this) tasks.create(NodeJsSetupTask.NAME, NodeJsSetupTask::class.java) - setupCleanNodeModulesTask(project) +// setupCleanNodeModulesTask(project, nodeJs) allprojects { if (it != project) { @@ -24,13 +27,21 @@ open class NodeJsPlugin : Plugin { } } - private fun setupCleanNodeModulesTask(project: Project) { + private fun setupCleanNodeModulesTask( + project: Project, + nodeJs: NodeJsRootExtension + ) { project.tasks.create("cleanNodeModules", Delete::class.java) { it.description = "Deletes node_modules and package.json file" it.group = BasePlugin.BUILD_GROUP - it.doLast { - project.nodeJs.root.packageManager.cleanProject(project) + val npmProjectLayout = NpmProjectLayout[it.project] + + project.delete(npmProjectLayout.nodeModulesDir) + project.afterEvaluate { + if (nodeJs.manageNodeModules) { + project.delete(npmProjectLayout.packageJsonFile) + } } project.tasks.maybeCreate(BasePlugin.CLEAN_TASK_NAME).dependsOn(it) @@ -38,7 +49,7 @@ open class NodeJsPlugin : Plugin { } companion object { - fun apply(project: Project): NodeJsRootExtension { + operator fun get(project: Project): NodeJsRootExtension { val rootProject = project.rootProject rootProject.plugins.apply(NodeJsPlugin::class.java) return rootProject.extensions.getByName(NODE_JS) as NodeJsRootExtension diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsRootExtension.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsRootExtension.kt index eeb0d348ce0..8ca212bfaef 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsRootExtension.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsRootExtension.kt @@ -19,43 +19,46 @@ open class NodeJsRootExtension(project: Project) : NodeJsExtension(project) { var nodeCommand = "node" + var npmCustomVersion: String? = null + var npmCommand = "npm" + var manageNodeModules: Boolean = false var packageManager: NpmApi = Yarn val nodeJsSetupTask: NodeJsSetupTask get() = project.tasks.getByName(NodeJsSetupTask.NAME) as NodeJsSetupTask - internal val environment: NodeJsEnv - get() { - val platform = NodeJsPlatform.name - val architecture = NodeJsPlatform.architecture + internal fun buildEnv(): NodeJsEnv { + val platform = NodeJsPlatform.name + val architecture = NodeJsPlatform.architecture - val nodeDir = installationDir.resolve("node-v$nodeVersion-$platform-$architecture") - val isWindows = NodeJsPlatform.name == NodeJsPlatform.WIN - val nodeBinDir = if (isWindows) nodeDir else nodeDir.resolve("bin") + val nodeDir = installationDir.resolve("node-v$nodeVersion-$platform-$architecture") + val isWindows = NodeJsPlatform.name == NodeJsPlatform.WIN + val nodeBinDir = if (isWindows) nodeDir else nodeDir.resolve("bin") - fun getExecutable(command: String, customCommand: String, windowsExtension: String): String { - val finalCommand = if (isWindows && customCommand == command) "$command.$windowsExtension" else customCommand - return if (download) File(nodeBinDir, finalCommand).absolutePath else finalCommand - } - - fun getIvyDependency(): String { - val type = if (isWindows) "zip" else "tar.gz" - return "org.nodejs:node:$nodeVersion:$platform-$architecture@$type" - } - - return NodeJsEnv( - nodeDir = nodeDir, - nodeBinDir = nodeBinDir, - nodeExecutable = getExecutable("node", nodeCommand, "exe"), - platformName = platform, - architectureName = architecture, - ivyDependency = getIvyDependency() - ) + fun getExecutable(command: String, customCommand: String, windowsExtension: String): String { + val finalCommand = if (isWindows && customCommand == command) "$command.$windowsExtension" else customCommand + return if (download) File(nodeBinDir, finalCommand).absolutePath else finalCommand } + fun getIvyDependency(): String { + val type = if (isWindows) "zip" else "tar.gz" + return "org.nodejs:node:$nodeVersion:$platform-$architecture@$type" + } + + return NodeJsEnv( + nodeDir = nodeDir, + nodeBinDir = nodeBinDir, + nodeExec = getExecutable("node", nodeCommand, "exe"), + npmExec = getExecutable("npm", npmCommand, "cmd"), + platformName = platform, + architectureName = architecture, + ivyDependency = getIvyDependency() + ) + } + internal fun executeSetup() { - val nodeJsEnv = environment + val nodeJsEnv = buildEnv() if (download) { if (!nodeJsEnv.nodeBinDir.isDirectory) { nodeJsSetupTask.exec() @@ -66,6 +69,6 @@ open class NodeJsRootExtension(project: Project) : NodeJsExtension(project) { companion object { const val NODE_JS: String = "nodeJs" - operator fun get(project: Project) = NodeJsPlugin.apply(project) + operator fun get(project: Project): NodeJsRootExtension = NodeJsPlugin[project] } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsSetupTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsSetupTask.kt index 76315f038ad..73b89c0609c 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsSetupTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsSetupTask.kt @@ -11,8 +11,8 @@ import java.io.File import java.net.URI open class NodeJsSetupTask : DefaultTask() { - private val settings = project.nodeJs.root - private val env by lazy { settings.environment } + private val settings = NodeJsRootExtension[project] + private val env by lazy { settings.buildEnv() } init { group = NodeJsRootExtension.NODE_JS @@ -55,7 +55,7 @@ open class NodeJsSetupTask : DefaultTask() { } } - val dep = this.project.dependencies.create(ivyDependency) + val dep = this.project.dependencies.create(env.ivyDependency) val conf = this.project.configurations.detachedConfiguration(dep) conf.isTransitive = false val result = conf.resolve().single() @@ -66,7 +66,7 @@ open class NodeJsSetupTask : DefaultTask() { unpackNodeArchive(result, destination.parentFile) // parent because archive contains name already if (!env.isWindows) { - File(env.nodeExecutable).setExecutable(true) + File(env.nodeExec).setExecutable(true) } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/utils.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/utils.kt index 4b1bf71b955..6b239f406a2 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/utils.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/utils.kt @@ -5,15 +5,4 @@ package org.jetbrains.kotlin.gradle.targets.js.nodejs -import org.gradle.api.Project - -internal fun nodeJsSetupWithoutTasks(project: Project) { - val nodeJsProject = NodeJsPlugin.ensureAppliedInHierarchy(project) - val nodeJs = NodeJsExtension[nodeJsProject] - val nodeJsEnv = nodeJs.buildEnv() - if (nodeJs.download) { - if (!nodeJsEnv.nodeBinDir.isDirectory) { - (nodeJsProject.tasks.findByName(NodeJsSetupTask.NAME) as NodeJsSetupTask).exec() - } - } -} \ No newline at end of file +import org.gradle.api.Project \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/GradleNodeModulesSync.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/GradleNodeModulesSync.kt new file mode 100644 index 00000000000..4aa552f46d2 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/GradleNodeModulesSync.kt @@ -0,0 +1,241 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.targets.js.npm + +import com.google.gson.GsonBuilder +import com.google.gson.JsonParseException +import org.gradle.api.Project +import org.gradle.api.internal.project.ProjectInternal +import org.gradle.internal.hash.FileHasher +import org.jetbrains.kotlin.daemon.common.toHexString +import org.jetbrains.kotlin.gradle.dsl.KotlinCommonOptions +import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation +import org.jetbrains.kotlin.gradle.targets.js.npm.GradleNodeModulesSync.* +import org.jetbrains.kotlin.gradle.targets.js.npm.NpmProjectLayout.Companion.PACKAGE_JSON +import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile +import java.io.File + +/** + * Incrementally sync gradle JS dependencies into node_modules directory. + * + * Working by traversing giving classpath. + * For each classpath element (called [Element]) `.js` and `.js.map` files are stored into `node_modules`. + * + * To do it incrementally, state of previously traversed classpath elements are stored in + * `node_modules/.from-gradle` file. Unchanged elements will not been visited. + * + * See [Element] and [getOrCompute] for more details. + */ +internal class GradleNodeModulesSync(val project: Project) { + companion object { + const val STATE_FILE_NAME = ".from-gradle" + } + + private val hasher = (project as ProjectInternal).services.get(FileHasher::class.java) + private val npmProjectLayout = NpmProjectLayout[project] + private val nodeModulesDir = npmProjectLayout.nodeModulesDir + private val gson = GsonBuilder().setPrettyPrinting().create() + + private class State { + val contents: MutableMap = mutableMapOf() + + operator fun get(elementHash: ByteArray) = + contents[elementHash.toHexString()] + + operator fun set(elementHash: ByteArray, element: Element) { + contents[elementHash.toHexString()] = element + } + } + + /** + * Traversed element from classpath. + * Stored in state file. + * + * When classpath traversed all elements are in one of this states: + * - unchanged: was loaded from previous state and contents not changed. + * only [fileNames] are known and contents of element not visited, + * but visitor are saved in [_fillFiles] to load in next case + * - unchanged but missed in node_modules: some of [fileNames] missed in node_modules + * in this case [_fillFiles] called before [files] was getted for copy spec + * - unchanged and existed in node_modules: nothing will be done in this case + * - new: not existed in previous state. In this case element is visited, + * both [fileNames] and [_files] are filled already + * - deleted: was loaded from previous state, but not existed in current state + * only [fileNames] are known for this case, which is used to remove outdated files + * - changed: this is just combination of deleted and new state + */ + private class Element { + lateinit var source: String + val fileNames: MutableList = mutableListOf() + + @Transient + @Suppress("PropertyName") + var _files: List? = null + + @Transient + @Suppress("PropertyName") + var _fillFiles: (() -> List)? = null + + val files: List + get() { + if (_files == null) { + _files = _fillFiles!!() + } + return _files!! + } + } + + /** + * Get info of classpath element from previous state (with storing visitor for case when files are deleted in node_modules) + * or compute it now. + * + * @param file classpath element + * @param buildElement js files list provider + */ + private fun getOrCompute(file: File, buildElement: () -> List) { + val hash = hasher.hash(file).toByteArray() + val old = old[hash] + if (old != null) old._fillFiles = buildElement + new[hash] = old ?: Element().also { + it.source = file.absolutePath + it._files = buildElement() + it._files!!.mapTo(it.fileNames) { it.name } + } + } + + private lateinit var old: State + private val new = State() + + private val stateFile = nodeModulesDir.resolve(STATE_FILE_NAME) + + fun loadOldState() { + val state: State? = if (stateFile.isFile) try { + stateFile.reader().use { + gson.fromJson(it, State::class.java) + } + } catch (e: JsonParseException) { + project.logger.warn("Cannot load previous state of Gradle depencies stored in node_modules", e) + null + } else { + if (stateFile.isDirectory) stateFile.deleteRecursively() + null + } + + old = state ?: State() + } + + fun sync() { + val children = nodeModulesDir.list()?.toSet() ?: setOf() + val deleted = old.contents.keys.toMutableSet() + val absented = mutableSetOf() + + new.contents.forEach { (key, value) -> + deleted.remove(key) + if (key !in old.contents.keys) absented.add(key) + else if (value.fileNames.any { it !in children }) absented.add(key) + } + + deleted.forEach { + old.contents[it]!!.fileNames.forEach { fileName -> + nodeModulesDir.resolve(fileName).delete() + } + } + + if (absented.isNotEmpty()) { + project.copy { copy -> + absented.forEach { key -> + copy.from(new.contents[key]!!.files) + } + copy.into(nodeModulesDir) + } + } + + if (deleted.isNotEmpty() || absented.isNotEmpty()) { + saveNewState() + } + } + + private fun saveNewState() { + stateFile.writer().use { + gson.toJson(new, it) + } + } + + class TransitiveNpmDependency( + val key: String, + val version: String + ) + + fun visitCompilation( + compilation: KotlinCompilation, + project: Project, + transitiveDependencies: MutableList + ): List { + val kotlin2JsCompile = compilation.compileKotlinTask as Kotlin2JsCompile + + // classpath + kotlin2JsCompile.classpath.forEach { srcFile -> + when { + srcFile.name == PACKAGE_JSON -> visitPackageJson(srcFile, transitiveDependencies) + isKotlinJsRuntimeFile(srcFile) -> getOrCompute(srcFile) { + listOf(srcFile) + } + srcFile.isZip -> getOrCompute(srcFile) { + mutableListOf().also { files -> + this.project.zipTree(srcFile).forEach { innerFile -> + when { + innerFile.name == PACKAGE_JSON -> visitPackageJson(innerFile, transitiveDependencies) + isKotlinJsRuntimeFile(innerFile) -> files.add(innerFile) + } + } + } + } + } + } + + // output + if (kotlin2JsCompile.state.executed) { + copyCompileOutput(project, kotlin2JsCompile) + } else { + kotlin2JsCompile.doLast { + copyCompileOutput(project, kotlin2JsCompile) + } + } + + return transitiveDependencies + } + + private fun copyCompileOutput(project: Project, kotlin2JsCompile: Kotlin2JsCompile) { + project.copy { copy -> + copy.from(kotlin2JsCompile.outputFile) + copy.from(kotlin2JsCompile.outputFile.path + ".map") + copy.into(nodeModulesDir) + } + } + + private fun visitPackageJson( + innerFile: File, + transitiveDependencies: MutableList + ) { + val packageJson = innerFile.reader().use { + gson.fromJson(it, PackageJson::class.java) + } + + packageJson.dependencies.forEach { (key, version) -> + transitiveDependencies.add(TransitiveNpmDependency(key, version)) + } + } + + private val File.isZip + get() = isFile && (name.endsWith(".jar") || name.endsWith(".zip")) + + private fun isKotlinJsRuntimeFile(file: File): Boolean { + if (!file.isFile) return false + val name = file.name + return (name.endsWith(".js") && !name.endsWith(".meta.js")) + || name.endsWith(".js.map") + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NodeModulesGradleComponents.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NodeModulesGradleComponents.kt deleted file mode 100644 index f35d0def675..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NodeModulesGradleComponents.kt +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.gradle.targets.js.npm - -import com.google.gson.GsonBuilder -import com.google.gson.JsonIOException -import com.google.gson.JsonSyntaxException -import org.gradle.api.Project -import org.gradle.api.internal.project.ProjectInternal -import org.gradle.internal.hash.FileHasher -import org.jetbrains.kotlin.gradle.dsl.KotlinCommonOptions -import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation -import org.jetbrains.kotlin.gradle.targets.js.tasks.isKotlinJsRuntimeFile -import org.jetbrains.kotlin.gradle.targets.js.tasks.isZip -import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile -import java.io.File - -class NodeModulesGradleComponents(val rootProject: Project) { - val hasher = (rootProject as ProjectInternal).services.get(FileHasher::class.java) - val nodeModulesDir = rootProject.rootDir.resolve("node_modules") - val gson = GsonBuilder().setPrettyPrinting().create() - - private class State { - val contents: MutableMap = mutableMapOf() - } - - private class Element { - val fileNames: MutableList = mutableListOf() - - @Transient - val files = mutableListOf() - } - - class TransitiveNpmDependency( - val project: String, - val from: String, - val key: String, - val version: String - ) - - private lateinit var old: State - private val new = State() - - val stateFile = rootProject.buildDir.resolve("nodeModulesGradleComponents.json") - - fun loadOldState() { - val state: State? = if (stateFile.exists()) try { - stateFile.reader().use { - gson.fromJson(it, State::class.java) - } - } catch (e: JsonSyntaxException) { - // todo: warn - null - } catch (e: JsonIOException) { - // todo: warn - null - } else null - - old = state ?: State() - } - - fun sync() { - val children = nodeModulesDir.list()?.toSet() ?: setOf() - val deleted = old.contents.keys.toMutableSet() - val absense = mutableSetOf() - - new.contents.forEach { (key, value) -> - deleted.remove(key) - if (key !in old.contents.keys) absense.add(key) - else if (value.fileNames.any { it !in children }) absense.add(key) - } - - deleted.forEach { - old.contents[it]!!.fileNames.forEach { fileName -> - nodeModulesDir.resolve(fileName).delete() - } - } - - rootProject.copy { copy -> - absense.forEach { key -> - copy.from(new.contents[key]!!.files) - copy.into(nodeModulesDir) - } - } - - saveNewState() - } - - private fun saveNewState() { - stateFile.writer().use { - gson.toJson(new, it) - } - } - - private fun getOrCompute(file: File, buildElement: (Element) -> Unit) { - val hash = hasher.hash(file).toByteArray() - new.contents[hash] = old.contents[hash] ?: Element().also { - buildElement(it) - it.files.mapTo(it.fileNames) { it.name } - } - } - - fun visitCompilation( - compilation: KotlinCompilation, - project: Project, - transitiveDependencies: MutableList - ): List { - val kotlin2JsCompile = compilation.compileKotlinTask as Kotlin2JsCompile - - // classpath - kotlin2JsCompile.classpath.forEach { srcFile -> - when { - isKotlinJsRuntimeFile(srcFile) -> getOrCompute(srcFile) { element -> - element.files.add(srcFile) - } - srcFile.isZip -> getOrCompute(srcFile) { element -> - rootProject.zipTree(srcFile).forEach { innerFile -> - if (innerFile.name == "package.json") { - val packageJson = innerFile.reader().use { - gson.fromJson(it, PackageJson::class.java) - } - - packageJson.dependencies.forEach { (key, version) -> - transitiveDependencies.add(TransitiveNpmDependency(project.path, srcFile.path, key, version)) - } - } else if (isKotlinJsRuntimeFile(innerFile)) { - element.files.add(innerFile) - } - } - } - } - } - - // output - kotlin2JsCompile.doLast { - project.copy { copy -> - copy.from(kotlin2JsCompile.outputFile) - copy.from(kotlin2JsCompile.outputFile.path + ".map") - copy.into(nodeModulesDir) - } - } - - return transitiveDependencies - } -} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/Npm.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/Npm.kt deleted file mode 100644 index 0c4dc97e023..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/Npm.kt +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.gradle.targets.js.npm - -import org.gradle.api.Project -import org.jetbrains.kotlin.gradle.targets.js.nodejs.nodeJsSetupWithoutTasks - -object Npm: NpmApi { - override fun setup(project: Project) { - nodeJsSetupWithoutTasks(project) - } - - override fun resolveRootProject(project: Project) { - TODO("not implemented") //To change body of created functions use File | Settings | File Templates. - } -} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmApi.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmApi.kt index 0e30e1b7f10..cd0aa881383 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmApi.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmApi.kt @@ -27,13 +27,6 @@ interface NpmApi { open val hoistGradleNodeModules get() = false - fun cleanProject(project: Project) { - val npmProject = project.npmProject - - npmProject.nodeModulesDir.deleteRecursively() - npmProject.packageJsonFile.delete() - } - companion object { fun resolveOperationDescription(packageManagerTitle: String): String = "Resolving NPM dependencies using $packageManagerTitle" diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmDependency.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmDependency.kt index 5bdc23f800b..916ed075806 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmDependency.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmDependency.kt @@ -10,74 +10,50 @@ import org.gradle.api.artifacts.Dependency import org.gradle.api.artifacts.FileCollectionDependency import org.gradle.api.artifacts.SelfResolvingDependency import org.gradle.api.file.FileCollection -import org.gradle.api.internal.artifacts.DependencyResolveContext -import org.gradle.api.internal.artifacts.ResolvableDependency import org.gradle.api.internal.artifacts.dependencies.SelfResolvingDependencyInternal import org.gradle.api.tasks.TaskDependency +import java.io.File import org.gradle.internal.component.local.model.DefaultLibraryBinaryIdentifier import org.jetbrains.kotlin.gradle.targets.js.npm.NpmResolver.ResolutionCallResult.* -import java.io.File data class NpmDependency( - internal val project: Project, + private val project: Project, private val org: String?, private val name: String, private val version: String ) : SelfResolvingDependency, SelfResolvingDependencyInternal, - ResolvableDependency, FileCollectionDependency { override fun getGroup(): String? = org - internal val dependencies = mutableSetOf() - override fun resolve(): MutableSet { - resolveProject() ?: return mutableSetOf() - - val all = mutableSetOf() - val visited = mutableSetOf() - - fun visit(item: NpmDependency) { - if (item in visited) return - visited.add(item) - item.tryFindNodeModule()?.let { - all.add(it) - } - } - - visit(this) - - return all - } - - internal fun resolveProject(): NpmResolver.ResolvedProject? { val result = NpmResolver.resolve(project) return when (result) { - is AlreadyInProgress -> null + is AlreadyInProgress -> mutableSetOf() is AlreadyResolved -> { check(this in result.resolution.dependencies) { "Project hierarchy is already resolved in NPM without $this" } - result.resolution + tryFindNodeModule() } is ResolvedNow -> { check(this in result.resolution.dependencies) { "$this was not visited during resolution" } - result.resolution + tryFindNodeModule() } } } - private fun tryFindNodeModule(): File? { + private fun tryFindNodeModule(): MutableSet { var p = project do { - val result = NpmProject[p].nodeModulesDir.resolve(key) - if (result.exists()) return result - p = project.parent ?: return null + val result = p.rootDir.resolve("node_modules/$key") + if (result.exists()) return mutableSetOf(result) + p = project.parent ?: return mutableSetOf() } while (true) } @@ -85,13 +61,6 @@ data class NpmDependency( override fun toString() = "$key: $version" - override fun resolve(context: DependencyResolveContext) { - resolve() - dependencies.forEach { - context.add(it) - } - } - override fun resolve(transitive: Boolean): MutableSet = resolve() override fun getFiles(): FileCollection = project.files(resolve()) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmProjectLayout.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmProjectLayout.kt new file mode 100644 index 00000000000..1a8a605e87e --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmProjectLayout.kt @@ -0,0 +1,35 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.targets.js.npm + +import org.gradle.api.Project +import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin +import java.io.File + +class NpmProjectLayout(val nodeWorkDir: File) { + val nodeModulesDir + get() = nodeWorkDir.resolve(NODE_MODULES) + + val packageJsonFile: File + get() { + return nodeWorkDir.resolve(PACKAGE_JSON) + } + + companion object { + const val PACKAGE_JSON = "package.json" + const val NODE_MODULES = "node_modules" + + operator fun get(project: Project): NpmProjectLayout { + val manageNodeModules = NodeJsPlugin[project].manageNodeModules + + val nodeWorkDir = + if (manageNodeModules) project.rootDir + else project.buildDir + + return NpmProjectLayout(nodeWorkDir) + } + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmResolver.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmResolver.kt index 207997c5f12..304ff85991f 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmResolver.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmResolver.kt @@ -10,21 +10,23 @@ import com.google.gson.GsonBuilder import org.gradle.api.Project import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension import org.jetbrains.kotlin.gradle.dsl.KotlinSingleTargetExtension -import org.jetbrains.kotlin.gradle.dsl.kotlinExtensionOrNull +import org.jetbrains.kotlin.gradle.dsl.kotlinExtension import org.jetbrains.kotlin.gradle.plugin.KotlinPlatformType import org.jetbrains.kotlin.gradle.plugin.KotlinTarget +import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsExtension import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension -import org.jetbrains.kotlin.gradle.targets.js.nodejs.nodeJs import org.jetbrains.kotlin.gradle.targets.js.npm.NpmResolver.ResolutionCallResult.* /** - * Generates `package.json` file for projects with npm or js dependencies and - * runs selected [NodeJsRootExtension.packageManager] to download and install it. + * [NpmResolver] runs selected [NodeJsRootExtension.packageManager] on root project + * with configured `package.json` files in all projects. * - * All [NpmDependency] for configurations related to kotlin/js will be added to `package.json`. - * For external gradle modules, fake npm packages will be created and added to `package.json` - * as path to directory. + * `package.json` + * + * - compile(npm(...)) dependencies + * - compile(npm(...)) dependencies from external modules + * - compile(non npm(...)) dependencies transitively */ internal class NpmResolver private constructor(val rootProject: Project) { companion object { @@ -38,11 +40,7 @@ internal class NpmResolver private constructor(val rootProject: Project) { val resolved = process?.resolved return if (resolved != null) AlreadyResolved(resolved) - else { - val resolver = NpmResolver(rootProject) - check(resolver.resolve(rootProject, null)) - ResolvedNow(ResolvedProject(resolver.npmPackages)) - } + else ResolvedNow(NpmResolver(rootProject).resolve(rootProject, null)!!) } } @@ -52,10 +50,9 @@ internal class NpmResolver private constructor(val rootProject: Project) { class ResolvedNow(val resolution: ResolvedProject) : ResolutionCallResult() } - private val nodeJs = NodeJsPlugin.apply(rootProject).root - private val npmProject = rootProject.npmProject + private val nodeJs = NodeJsPlugin[rootProject] private val packageManager = nodeJs.packageManager - private val hoistGradleNodeModules = npmProject.hoistGradleNodeModules + private val hoistGradleNodeModules = packageManager.hoistGradleNodeModules private val npmPackages = mutableListOf() private val gson = GsonBuilder() .setPrettyPrinting() @@ -69,11 +66,7 @@ internal class NpmResolver private constructor(val rootProject: Project) { } } - class ResolvedProject(val npmPackages: Collection) { - val dependencies by lazy { - npmPackages.flatMapTo(mutableSetOf()) { it.npmDependencies } - } - } + class ResolvedProject(val dependencies: Set) class NpmPackage( val project: Project, @@ -81,7 +74,7 @@ internal class NpmResolver private constructor(val rootProject: Project) { val npmDependencies: Set ) { fun savePackageJson(gson: Gson) { - NpmProject[project].packageJsonFile.writer().use { + NpmProjectLayout[project].packageJsonFile.writer().use { gson.toJson(packageJson, it) } } @@ -97,29 +90,29 @@ internal class NpmResolver private constructor(val rootProject: Project) { private fun resolve( project: Project, - parentGradleModules: GradleNodeModules? - ): Boolean { + parentGradleNodeModules: GradleNodeModulesSync? + ): ResolvedProject? { val existedProcess = ProjectData[project] - if (existedProcess != null) { + val process = if (existedProcess != null) { if (existedProcess.resolved != null) error("yarn dependencies for $project already resolved") else { // called inside resolution of classpath (from visitTarget) // we should return as we are already in progress - return false + return null } } else ProjectData().also { ProjectData[project] = it } - val gradleModules = - if (hoistGradleNodeModules && parentGradleModules != null) parentGradleModules - else GradleNodeModules(project) + val gradleNodeModules = + if (hoistGradleNodeModules && parentGradleNodeModules != null) parentGradleNodeModules + else newGradleNodeModulesSync(project) project.subprojects.forEach { - resolve(it, gradleModules) + resolve(it, gradleNodeModules) } - val npmPackage = extractNpmPackage(project, gradleModules) + val npmPackage = extractNpmPackage(project, gradleNodeModules) if (npmPackage != null) { npmPackage.savePackageJson(gson) @@ -136,60 +129,61 @@ internal class NpmResolver private constructor(val rootProject: Project) { } } - if (gradleModules != parentGradleModules) { - gradleModules.close() + if (gradleNodeModules != parentGradleNodeModules) { + gradleNodeModules.sync() } - return true + val npmDependencies = npmPackage?.npmDependencies ?: setOf() + return ResolvedProject(npmDependencies).also { + process.resolved = it + } + } + + private fun newGradleNodeModulesSync(project: Project): GradleNodeModulesSync { + return GradleNodeModulesSync(project).also { + it.loadOldState() + } } private fun extractNpmPackage( project: Project, - gradleComponents: GradleNodeModules + gradleComponentsSync: GradleNodeModulesSync ): NpmPackage? { val packageJson = PackageJson(project.name, project.version.toString()) + val transitiveDependencies = mutableListOf() + + val kotlin = project.kotlinExtension val npmDependencies = mutableSetOf() - - visitNpmDependencies(project, npmDependencies, gradleComponents, packageJson) - - if (!hoistGradleNodeModules || project == project.rootProject) { - gradleComponents.modules.forEach { - val relativePath = it.path.relativeTo(NpmProject[project].nodeWorkDir) - packageJson.dependencies[it.name] = "file:$relativePath" + when (kotlin) { + is KotlinSingleTargetExtension -> visitTarget( + kotlin.target, + project, + npmDependencies, + gradleComponentsSync, + transitiveDependencies + ) + is KotlinMultiplatformExtension -> kotlin.targets.forEach { + visitTarget(it, project, npmDependencies, gradleComponentsSync, transitiveDependencies) } } - project.nodeJs.packageJsonHandlers.forEach { + npmDependencies.forEach { + packageJson.dependencies[it.key] = chooseVersion(packageJson.dependencies[it.key], it.version) + } + + transitiveDependencies.forEach { + packageJson.dependencies[it.key] = chooseVersion(packageJson.dependencies[it.key], it.version) + } + + NodeJsExtension[project].packageJsonHandlers.forEach { it(packageJson) } - val packageJsonRequired = if (project == rootProject) { + if (project == rootProject) { packageManager.hookRootPackage(rootProject, packageJson, npmPackages) - } else false - - return if (packageJsonRequired || !packageJson.empty) NpmPackage(project, packageJson, npmDependencies) else null - } - - private fun visitNpmDependencies( - project: Project, - npmDependencies: MutableSet, - gradleComponents: GradleNodeModules, - packageJson: PackageJson - ) { - val kotlin = project.kotlinExtensionOrNull - - if (kotlin != null) { - when (kotlin) { - is KotlinSingleTargetExtension -> visitTarget(kotlin.target, project, npmDependencies, gradleComponents) - is KotlinMultiplatformExtension -> kotlin.targets.forEach { - visitTarget(it, project, npmDependencies, gradleComponents) - } - } - - npmDependencies.forEach { - packageJson.dependencies[it.key] = chooseVersion(packageJson.dependencies[it.key], it.version) - } } + + return if (!packageJson.empty) NpmPackage(project, packageJson, npmDependencies) else null } private fun chooseVersion(oldVersion: String?, newVersion: String): String = @@ -199,7 +193,8 @@ internal class NpmResolver private constructor(val rootProject: Project) { target: KotlinTarget, project: Project, npmDependencies: MutableSet, - gradleComponents: GradleNodeModules + gradleComponentsSync: GradleNodeModulesSync, + transitiveDependencies: MutableList ) { if (target.platformType == KotlinPlatformType.js) { target.compilations.toList().forEach { compilation -> @@ -211,7 +206,7 @@ internal class NpmResolver private constructor(val rootProject: Project) { } } - gradleComponents.visitCompilation(compilation) + gradleComponentsSync.visitCompilation(compilation, project, transitiveDependencies) } } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/PackageJson.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/PackageJson.kt index 7e4f64781ea..a4b81fdaee2 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/PackageJson.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/PackageJson.kt @@ -10,34 +10,11 @@ class PackageJson( var version: String ) { val empty: Boolean - get() = private == null && - workspaces == null && - dependencies.isEmpty() && - devDependencies.isEmpty() - - val scopedName: ScopedName - get() = scopedName(name) + get() = private == null && workspaces == null && dependencies.isEmpty() var private: Boolean? = null - var main: String? = null - var workspaces: Collection? = null - val devDependencies = mutableMapOf() val dependencies = mutableMapOf() - - companion object { - fun scopedName(name: String): ScopedName = if (name.contains("/")) ScopedName( - scope = name.substringBeforeLast("/").removePrefix("@"), - name = name.substringAfterLast("/") - ) else ScopedName(scope = null, name = name) - - operator fun invoke(scope: String, name: String, version: String) = - PackageJson(ScopedName(scope, name).toString(), version) - } - - data class ScopedName(val scope: String?, val name: String) { - override fun toString() = if (scope == null) name else "@$scope/$name" - } } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/tasks/KotlinJsNodeModulesTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/tasks/KotlinJsNodeModulesTask.kt deleted file mode 100644 index cd2f70fb5ed..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/tasks/KotlinJsNodeModulesTask.kt +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.gradle.targets.js.tasks - -import org.gradle.api.DefaultTask -import org.gradle.api.file.FileCollection -import org.gradle.api.tasks.* -import org.jetbrains.kotlin.gradle.targets.js.internal.RewriteSourceMapFilterReader -import java.io.File - -open class KotlinJsNodeModulesTask : DefaultTask() { - @OutputDirectory - @SkipWhenEmpty - lateinit var nodeModulesDir: File - - @Classpath - @InputFiles - @SkipWhenEmpty - lateinit var classpath: FileCollection - - @TaskAction - fun copyFromRuntimeClasspath() { - project.sync { sync -> - sync.includeEmptyDirs = false - - classpath.forEach { - if (it.isZip) sync.from(project.zipTree(it)) - else sync.from(it) - } - - sync.include { fileTreeElement -> - isKotlinJsRuntimeFile(fileTreeElement.file) - } - - sync.eachFile { - if (it.name.endsWith(".js.map")) { - it.filter( - mapOf( - "srcSourceRoot" to it.file.parentFile, - "targetSourceRoot" to nodeModulesDir - ), - RewriteSourceMapFilterReader::class.java - ) - } - } - - sync.into(nodeModulesDir) - } - } -} - -internal val File.isZip - get() = isFile && (name.endsWith(".jar") || name.endsWith(".zip")) - -internal fun isKotlinJsRuntimeFile(file: File): Boolean { - if (!file.isFile) return false - val name = file.name - return (name.endsWith(".js") && !name.endsWith(".meta.js")) - || name.endsWith(".js.map") -} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/tasks/KotlinNodeJsTestTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/tasks/KotlinNodeJsTestTask.kt index ffa85160559..e2177ac62a0 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/tasks/KotlinNodeJsTestTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/tasks/KotlinNodeJsTestTask.kt @@ -15,6 +15,8 @@ import org.jetbrains.kotlin.gradle.internal.testing.TCServiceMessagesClientSetti import org.jetbrains.kotlin.gradle.internal.testing.TCServiceMessagesTestExecutionSpec import org.jetbrains.kotlin.gradle.targets.js.internal.parseNodeJsStackTrace import org.jetbrains.kotlin.gradle.targets.js.internal.parseNodeJsStackTraceAsJvm +import org.jetbrains.kotlin.gradle.targets.js.npm.NpmProjectLayout +import org.jetbrains.kotlin.gradle.targets.js.npm.NpmResolver import org.jetbrains.kotlin.gradle.tasks.KotlinTestTask import org.jetbrains.kotlin.gradle.testing.IgnoredTestSuites import java.io.File @@ -24,9 +26,6 @@ open class KotlinNodeJsTestTask : KotlinTestTask() { var ignoredTestSuites: IgnoredTestSuites = IgnoredTestSuites.showWithContents - @InputDirectory - lateinit var nodeModulesDir: File - @Input @SkipWhenEmpty var nodeModulesToLoad: Set = setOf() @@ -53,11 +52,19 @@ open class KotlinNodeJsTestTask : KotlinTestTask() { options(nodeJsProcessOptions) } + override fun executeTests() { + NpmResolver.resolve(project) + super.executeTests() + } + override fun createTestExecutionSpec(): TCServiceMessagesTestExecutionSpec { val extendedForkOptions = DefaultProcessForkOptions(fileResolver) nodeJsProcessOptions.copyTo(extendedForkOptions) - extendedForkOptions.environment.addPath("NODE_PATH", nodeModulesDir.canonicalPath) + NpmResolver.resolve(project) + + val npmProjectLayout = NpmProjectLayout[project] + extendedForkOptions.workingDir = npmProjectLayout.nodeWorkDir val nodeJsArgs = mutableListOf() @@ -83,7 +90,7 @@ open class KotlinNodeJsTestTask : KotlinTestTask() { extendedForkOptions, nodeJsArgs + testRuntimeNodeModules - .map { nodeModulesDir.resolve(it) } + .map { npmProjectLayout.nodeModulesDir.resolve(it) } .filter { it.exists() } .map { it.absolutePath } + cliArgs.toList(), @@ -122,10 +129,4 @@ data class KotlinNodeJsTestRunnerCliArgs( enum class IgnoredTestSuitesReporting { skip, reportAsIgnoredTest, reportAllInnerTestsAsIgnored } -} - -private fun MutableMap.addPath(key: String, path: String) { - val prev = get(key) - if (prev == null) set(key, path) - else set(key, prev as String + File.pathSeparator + path) } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/Yarn.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/Yarn.kt index f1fc285e65f..588453b54d1 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/Yarn.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/Yarn.kt @@ -6,121 +6,56 @@ package org.jetbrains.kotlin.gradle.targets.js.yarn import org.gradle.api.Project -import org.gradle.api.internal.project.ProjectInternal -import org.gradle.internal.hash.FileHasher -import org.jetbrains.kotlin.daemon.common.toHexString import org.jetbrains.kotlin.gradle.internal.execWithProgress +import org.jetbrains.kotlin.gradle.internal.operation import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin -import org.jetbrains.kotlin.gradle.targets.js.npm.* -import java.io.File +import org.jetbrains.kotlin.gradle.targets.js.npm.NpmApi +import org.jetbrains.kotlin.gradle.targets.js.npm.NpmProjectLayout +import org.jetbrains.kotlin.gradle.targets.js.npm.NpmResolver +import org.jetbrains.kotlin.gradle.targets.js.npm.PackageJson +import org.slf4j.LoggerFactory object Yarn : NpmApi { - override fun setup(project: Project) { - YarnPlugin.apply(project).executeSetup() - } + private val log = LoggerFactory.getLogger("org.jetbrains.kotlin.gradle.targets.js.yarn.Yarn") - val Project.packageJsonHashFile: File - get() = buildDir.resolve("package.json.hash") + override fun setup(project: Project) { + YarnPlugin[project].executeSetup() + } fun yarnExec( project: Project, description: String, vararg args: String, - npmProject: NpmProject = NpmProject[project] + npmProjectLayout: NpmProjectLayout = NpmProjectLayout[project] ) { - val nodeJsEnv = NodeJsPlugin.apply(project).root.environment - val yarnEnv = YarnPlugin.apply(project).environment - - val packageJsonHashFile = project.packageJsonHashFile - val packageJsonHash = if (packageJsonHashFile.exists()) packageJsonHashFile.readText() else null - - val hasher = (project as ProjectInternal).services.get(FileHasher::class.java) - val hash = hasher.hash(npmProject.packageJsonFile).toByteArray().toHexString() - - if (packageJsonHash == hash) return - - packageJsonHashFile.writeText(hash) - - val nodeWorkDir = npmProject.nodeWorkDir + val nodeJsEnv = NodeJsPlugin[project].buildEnv() + val yarnEnv = YarnPlugin[project].buildEnv() project.execWithProgress(description) { exec -> - exec.executable = nodeJsEnv.nodeExecutable + exec.executable = nodeJsEnv.nodeExec exec.args = listOf(yarnEnv.home.resolve("bin/yarn.js").absolutePath) + args - exec.workingDir = nodeWorkDir - } - } - - private fun yarnLockReadTransitiveDependencies( - nodeWorkDir: File, - srcDependenciesList: Collection - ) { - val yarnLock = nodeWorkDir.resolve("yarn.lock") - if (yarnLock.isFile) { - val byKey = YarnLock.parse(yarnLock).entries.associateBy { it.key } - val visited = mutableSetOf() - - fun resolveRecursively(src: NpmDependency): NpmDependency { - if (src in visited) return src - visited.add(src) - - val key = YarnLock.key(src.key, src.version) - val deps = byKey[key] - if (deps != null) { - src.dependencies.addAll(deps.dependencies.map { dep -> - val scopedName = dep.scopedName - - resolveRecursively( - NpmDependency( - src.project, - scopedName.scope, - scopedName.name, - dep.version ?: "*" - ) - ) - }) - } else { - // todo: [WARN] cannot find $key in yarn.lock - } - - return src - } - - srcDependenciesList.forEach { src -> - resolveRecursively(src) - } + exec.workingDir = npmProjectLayout.nodeWorkDir } } @Suppress("EXPOSED_PARAMETER_TYPE") override fun resolveProject(npmPackage: NpmResolver.NpmPackage) { val project = npmPackage.project - if (!project.yarn.useWorkspaces) { + if (!YarnPlugin[project].checkUseWorkspace()) { yarnExec(project, NpmApi.resolveOperationDescription("yarn for ${project.path}")) - yarnLockReadTransitiveDependencies(NpmProject[project].nodeWorkDir, npmPackage.npmDependencies) } } @Suppress("EXPOSED_PARAMETER_TYPE") - override fun hookRootPackage( - rootProject: Project, - rootPackageJson: PackageJson, - allWorkspaces: Collection - ): Boolean { - if (rootProject.yarn.useWorkspaces) { + override fun hookRootPackage(rootProject: Project, rootPackageJson: PackageJson, allWorkspaces: Collection) { + if (YarnPlugin[rootProject].checkUseWorkspace()) { rootPackageJson.private = true rootPackageJson.workspaces = allWorkspaces .filter { it.project != rootProject } - .map { it.project.projectDir.relativeTo(rootProject.rootDir).path } - - return true + .map { it.project.rootDir.relativeTo(rootProject.rootDir).path } } - - return false } - override fun getHoistGradleNodeModules(project: Project): Boolean = - project.rootProject.yarn.useWorkspaces - @Suppress("EXPOSED_PARAMETER_TYPE") override fun resolveRootProject( rootProject: Project, @@ -128,9 +63,8 @@ object Yarn : NpmApi { ) { check(rootProject == rootProject.rootProject) - if (rootProject.yarn.useWorkspaces) { + if (YarnPlugin[rootProject].checkUseWorkspace()) { yarnExec(rootProject, NpmApi.resolveOperationDescription("yarn")) - yarnLockReadTransitiveDependencies(NpmProject[rootProject].nodeWorkDir, subprojects.flatMap { it.npmDependencies }) } else { if (subprojects.any { it.project != rootProject }) { // todo: proofread message @@ -139,15 +73,11 @@ object Yarn : NpmApi { "It is recommended to enable yarn workspaces to store common NPM dependencies in root project. " + "To enable it add this to your root project: \n" + "nodeJs { manageNodeModules = true } \n" + + "yarn { useWorkspaces = true } \n" + "Note: with `manageNodeModules` enabled, your `node_modules` and `package.json` files will be managed by " + "Gradle, will be overridden during build and should be ignored in VCS." ) } } } - - override fun cleanProject(project: Project) { - super.cleanProject(project) - project.packageJsonHashFile.delete() - } } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnExtension.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnExtension.kt deleted file mode 100644 index b93fb4d4046..00000000000 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnExtension.kt +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.gradle.targets.js.yarn - -import org.gradle.api.GradleException -import org.gradle.api.Project -import org.jetbrains.kotlin.gradle.logging.kotlinInfo - -open class YarnExtension(project: Project) { - private val gradleHome = project.gradle.gradleUserHomeDir.also { - project.logger.kotlinInfo("Storing cached files in $it") - } - - var installationDir = gradleHome.resolve("yarn") - - var distBaseUrl = "https://github.com/yarnpkg/yarn/releases/download" - var version = "1.15.2" - - var command = "yarn" - - var download = true - - internal fun buildEnv() = YarnEnv( - downloadUrl = "$distBaseUrl/v$version/yarn-v$version.tar.gz", - home = installationDir.resolve("yarn-v$version") - ) - - companion object { - const val YARN: String = "yarn" - - operator fun get(project: Project): YarnExtension { - val extension = project.extensions.findByType(YarnExtension::class.java) - if (extension != null) - return extension - - val parentProject = project.parent - if (parentProject != null) - return get(parentProject) - - throw GradleException("YarnExtension is not installed") - } - } -} diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnPlugin.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnPlugin.kt index a36976edea2..dbcadf4c597 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnPlugin.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnPlugin.kt @@ -15,16 +15,14 @@ open class YarnPlugin : Plugin { "YarnPlugin can be applied only to root project" } - val nodeJs = NodeJsPlugin.apply(this) - this.extensions.create(YarnRootExtension.YARN, YarnRootExtension::class.java, this) tasks.create(YarnSetupTask.NAME, YarnSetupTask::class.java) { - it.dependsOn(nodeJs.nodeJsSetupTask) + it.dependsOn(NodeJsPlugin[this].nodeJsSetupTask) } } companion object { - fun apply(project: Project): YarnRootExtension { + operator fun get(project: Project): YarnRootExtension { val rootProject = project.rootProject rootProject.plugins.apply(YarnPlugin::class.java) return rootProject.extensions.getByName(YarnRootExtension.YARN) as YarnRootExtension diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnRootExtension.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnRootExtension.kt index f1b2490be7a..1a836c3854a 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnRootExtension.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnRootExtension.kt @@ -5,10 +5,10 @@ package org.jetbrains.kotlin.gradle.targets.js.yarn +import org.gradle.api.GradleException import org.gradle.api.Project import org.jetbrains.kotlin.gradle.logging.kotlinInfo import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin -import org.jetbrains.kotlin.gradle.targets.js.nodejs.nodeJs open class YarnRootExtension(val project: Project) { private val gradleHome = project.gradle.gradleUserHomeDir.also { @@ -23,36 +23,43 @@ open class YarnRootExtension(val project: Project) { val yarnSetupTask: YarnSetupTask get() = project.tasks.getByName(YarnSetupTask.NAME) as YarnSetupTask - var disableWorkspaces: Boolean = false + var useWorkspaces: Boolean = false - val useWorkspaces: Boolean - get() = project.nodeJs.root.manageNodeModules && !disableWorkspaces + internal fun checkUseWorkspace(): Boolean = if (useWorkspaces) { + check(NodeJsPlugin[project].manageNodeModules) { + "Yarn workspace can be used only with `manageNodeModules`: (add `nodeJs { manageNodeModules = true }` to your root project)" + } + + true + } else false internal fun executeSetup() { - NodeJsPlugin.apply(project).executeSetup() + NodeJsPlugin[project].executeSetup() - val env = environment + val env = buildEnv() if (!env.home.isDirectory) { yarnSetupTask.setup() } } - internal val environment - get() = YarnEnv( - downloadUrl = "$downloadBaseUrl/v$version/yarn-v$version.tar.gz", - home = installationDir.resolve("yarn-v$version") - ) + internal fun buildEnv() = YarnEnv( + downloadUrl = "$downloadBaseUrl/v$version/yarn-v$version.tar.gz", + home = installationDir.resolve("yarn-v$version") + ) companion object { const val YARN: String = "yarn" operator fun get(project: Project): YarnRootExtension { - val rootProject = project.rootProject - rootProject.plugins.apply(YarnPlugin::class.java) - return rootProject.extensions.getByName(YARN) as YarnRootExtension + val extension = project.extensions.findByType(YarnRootExtension::class.java) + if (extension != null) + return extension + + val parentProject = project.parent + if (parentProject != null) + return get(parentProject) + + throw GradleException("YarnRootExtension is not installed") } } } - -val Project.yarn: YarnRootExtension - get() = YarnRootExtension[this] \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnSetupTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnSetupTask.kt index 8d7b989ba98..8840b53ae05 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnSetupTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnSetupTask.kt @@ -14,8 +14,8 @@ import java.io.File import javax.xml.ws.Action open class YarnSetupTask : DefaultTask() { - private val settings = project.yarn - private val env by lazy { settings.environment } + private val settings = YarnRootExtension[project] + private val env by lazy { settings.buildEnv() } @Suppress("MemberVisibilityCanBePrivate") val downloadUrl