Gradle, npm: disable manageNodeModules by default

#KT-30530

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/exec.kt

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.internal
+
+import org.gradle.api.Project
+import org.gradle.api.internal.project.ProjectInternal
+import org.gradle.process.internal.ExecAction
+import org.gradle.process.internal.ExecActionFactory
+import java.io.ByteArrayOutputStream
+import java.io.PipedInputStream
+import java.io.PipedOutputStream
+import kotlin.concurrent.thread
+
+internal fun Project.execWithProgress(description: String, body: (ExecAction) -> Unit) {
+    this as ProjectInternal
+
+    val stderr = ByteArrayOutputStream()
+    val stdout = StringBuilder()
+    val stdInPipe = PipedInputStream()
+    val exec = services.get(ExecActionFactory::class.java).newExecAction()
+    body(exec)
+    project.operation(description) {
+        exec.errorOutput = stderr
+        exec.standardOutput = PipedOutputStream(stdInPipe)
+        val outputReaderThread = thread(name = "output reader for [$description]") {
+            stdInPipe.reader().useLines { lines ->
+                lines.forEach {
+                    stdout.appendln(it)
+                    progress(it)
+                }
+            }
+        }
+        exec.isIgnoreExitValue = true
+        val result = exec.execute()
+        outputReaderThread.join()
+        if (result.exitValue != 0) {
+            error(stderr.toString() + "\n" + stdout)
+        }
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/progress.kt

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.internal
+
+import org.gradle.api.Project
+import org.gradle.api.internal.project.ProjectInternal
+import org.gradle.internal.logging.events.ProgressStartEvent
+import org.gradle.internal.logging.progress.ProgressLogger
+
+fun Project.operation(
+    description: String,
+    initialStatus: String? = null,
+    body: ProgressLogger.() -> Unit
+) {
+    val services = (project as ProjectInternal).services
+    val progressFactory = services.get(org.gradle.internal.logging.progress.ProgressLoggerFactory::class.java)
+    val operation = progressFactory.newOperation(ProgressStartEvent.BUILD_OP_CATEGORY)
+    operation.start(description, initialStatus)
+    try {
+        operation.body()
+    } finally {
+        operation.completed()
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/KotlinJsCompilationTestsConfigurator.kt

@@ -3,10 +3,9 @@ package org.jetbrains.kotlin.gradle.targets.js
 import org.gradle.api.file.FileCollection
 import org.gradle.language.base.plugins.LifecycleBasePlugin
 import org.jetbrains.kotlin.gradle.plugin.KotlinCompilationToRunnableFiles
-import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsExtension
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin
-import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsSetupTask
-import org.jetbrains.kotlin.gradle.targets.js.tasks.*
+import org.jetbrains.kotlin.gradle.targets.js.npm.NpmProjectLayout
+import org.jetbrains.kotlin.gradle.targets.js.tasks.KotlinNodeJsTestTask
 import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile
 import org.jetbrains.kotlin.gradle.tasks.registerTask
 import org.jetbrains.kotlin.gradle.testing.internal.configureConventions
@@ -51,7 +50,7 @@ internal class KotlinJsCompilationTestsConfigurator(
             implementation(kotlin("test-nodejs-runner"))
         }
 
-        val projectWithNodeJsPlugin = NodeJsPlugin.ensureAppliedInHierarchy(target.project)
+        val nodeJs = NodeJsPlugin[target.project]
 
         val testTask = registerTask(project, testTaskName, KotlinNodeJsTestTask::class.java) { testJs ->
             testJs.group = LifecycleBasePlugin.VERIFICATION_GROUP
@@ -68,8 +67,6 @@ internal class KotlinJsCompilationTestsConfigurator(
             }
 
             testJs.nodeJsProcessOptions.workingDir = project.projectDir
-
-            testJs.nodeModulesDir = project.rootDir.resolve("node_modules")
             testJs.testRuntimeNodeModules = listOf(
                 "kotlin-test-nodejs-runner.js",
                 "kotlin-nodejs-source-map-support.js"
@@ -83,11 +80,11 @@ internal class KotlinJsCompilationTestsConfigurator(
         project.afterEvaluate {
             // defer nodeJs executable setup, as nodejs project settings may change during configuration
             testTask.configure {
-                val nodeJsSetupTask = projectWithNodeJsPlugin.tasks.findByName(NodeJsSetupTask.NAME)
+                val nodeJsSetupTask = nodeJs.nodeJsSetupTask
                 it.dependsOn(nodeJsSetupTask)
 
                 if (it.nodeJsProcessOptions.executable == null) {
-                    it.nodeJsProcessOptions.executable = NodeJsExtension[projectWithNodeJsPlugin].buildEnv().nodeExec
+                    it.nodeJsProcessOptions.executable = nodeJs.buildEnv().nodeExec
                 }
             }
         }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsExtension.kt

@@ -9,23 +9,16 @@ import org.gradle.api.Project
 import org.jetbrains.kotlin.gradle.targets.js.npm.PackageJson
 
 open class NodeJsExtension(val project: Project) {
-    val root: NodeJsRootExtension
-        get() = this as? NodeJsRootExtension ?: NodeJsRootExtension[project]
-
     internal val packageJsonHandlers = mutableListOf<PackageJson.() -> Unit>()
 
-    @Suppress("unused")
     fun packageJson(handler: PackageJson.() -> Unit) {
         packageJsonHandlers.add(handler)
     }
 
     companion object {
         operator fun get(project: Project): NodeJsExtension {
-            NodeJsPlugin.apply(project.rootProject)
+            NodeJsPlugin[project.rootProject]
             return project.extensions.getByName(NodeJsRootExtension.NODE_JS) as NodeJsExtension
         }
     }
-}
-
-val Project.nodeJs: NodeJsExtension
-    get() = NodeJsExtension[this]
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsPlatform.kt

@@ -2,7 +2,7 @@ package org.jetbrains.kotlin.gradle.targets.js.nodejs
 
 /**
  * Provides platform and architecture names that is used to download NodeJs.
- * See [NodeJsEnv.ivyDependency] that is filled in [NodeJsRootExtension.environment].
+ * See [NodeJsEnv.ivyDependency] that is filled in [NodeJsRootExtension.buildEnv].
  */
 internal object NodeJsPlatform {
     private val props = System.getProperties()

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsPlugin.kt

@@ -4,7 +4,10 @@ import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.plugins.BasePlugin
 import org.gradle.api.tasks.Delete
+import org.jetbrains.kotlin.backend.common.onlyIf
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension.Companion.NODE_JS
+import org.jetbrains.kotlin.gradle.targets.js.npm.NpmProjectLayout
+import org.jetbrains.kotlin.gradle.targets.js.npm.NpmResolver
 
 open class NodeJsPlugin : Plugin<Project> {
     override fun apply(project: Project): Unit = project.run {
@@ -12,10 +15,10 @@ open class NodeJsPlugin : Plugin<Project> {
             "NodeJsPlugin can be applied only to root project"
         }
 
-        this.extensions.create(NODE_JS, NodeJsRootExtension::class.java, this)
+        val nodeJs = this.extensions.create(NODE_JS, NodeJsRootExtension::class.java, this)
         tasks.create(NodeJsSetupTask.NAME, NodeJsSetupTask::class.java)
 
-        setupCleanNodeModulesTask(project)
+//        setupCleanNodeModulesTask(project, nodeJs)
 
         allprojects {
             if (it != project) {
@@ -24,13 +27,21 @@ open class NodeJsPlugin : Plugin<Project> {
         }
     }
 
-    private fun setupCleanNodeModulesTask(project: Project) {
+    private fun setupCleanNodeModulesTask(
+        project: Project,
+        nodeJs: NodeJsRootExtension
+    ) {
         project.tasks.create("cleanNodeModules", Delete::class.java) {
             it.description = "Deletes node_modules and package.json file"
             it.group = BasePlugin.BUILD_GROUP
 
-            it.doLast {
-                project.nodeJs.root.packageManager.cleanProject(project)
+            val npmProjectLayout = NpmProjectLayout[it.project]
+
+            project.delete(npmProjectLayout.nodeModulesDir)
+            project.afterEvaluate {
+                if (nodeJs.manageNodeModules) {
+                    project.delete(npmProjectLayout.packageJsonFile)
+                }
             }
 
             project.tasks.maybeCreate(BasePlugin.CLEAN_TASK_NAME).dependsOn(it)
@@ -38,7 +49,7 @@ open class NodeJsPlugin : Plugin<Project> {
     }
 
     companion object {
-        fun apply(project: Project): NodeJsRootExtension {
+        operator fun get(project: Project): NodeJsRootExtension {
             val rootProject = project.rootProject
             rootProject.plugins.apply(NodeJsPlugin::class.java)
             return rootProject.extensions.getByName(NODE_JS) as NodeJsRootExtension

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsRootExtension.kt

@@ -19,43 +19,46 @@ open class NodeJsRootExtension(project: Project) : NodeJsExtension(project) {
 
     var nodeCommand = "node"
 
+    var npmCustomVersion: String? = null
+    var npmCommand = "npm"
+
     var manageNodeModules: Boolean = false
     var packageManager: NpmApi = Yarn
 
     val nodeJsSetupTask: NodeJsSetupTask
         get() = project.tasks.getByName(NodeJsSetupTask.NAME) as NodeJsSetupTask
 
-    internal val environment: NodeJsEnv
-        get() {
-            val platform = NodeJsPlatform.name
-            val architecture = NodeJsPlatform.architecture
+    internal fun buildEnv(): NodeJsEnv {
+        val platform = NodeJsPlatform.name
+        val architecture = NodeJsPlatform.architecture
 
-            val nodeDir = installationDir.resolve("node-v$nodeVersion-$platform-$architecture")
-            val isWindows = NodeJsPlatform.name == NodeJsPlatform.WIN
-            val nodeBinDir = if (isWindows) nodeDir else nodeDir.resolve("bin")
+        val nodeDir = installationDir.resolve("node-v$nodeVersion-$platform-$architecture")
+        val isWindows = NodeJsPlatform.name == NodeJsPlatform.WIN
+        val nodeBinDir = if (isWindows) nodeDir else nodeDir.resolve("bin")
 
-            fun getExecutable(command: String, customCommand: String, windowsExtension: String): String {
-                val finalCommand = if (isWindows && customCommand == command) "$command.$windowsExtension" else customCommand
-                return if (download) File(nodeBinDir, finalCommand).absolutePath else finalCommand
-            }
-
-            fun getIvyDependency(): String {
-                val type = if (isWindows) "zip" else "tar.gz"
-                return "org.nodejs:node:$nodeVersion:$platform-$architecture@$type"
-            }
-
-            return NodeJsEnv(
-                nodeDir = nodeDir,
-                nodeBinDir = nodeBinDir,
-                nodeExecutable = getExecutable("node", nodeCommand, "exe"),
-                platformName = platform,
-                architectureName = architecture,
-                ivyDependency = getIvyDependency()
-            )
+        fun getExecutable(command: String, customCommand: String, windowsExtension: String): String {
+            val finalCommand = if (isWindows && customCommand == command) "$command.$windowsExtension" else customCommand
+            return if (download) File(nodeBinDir, finalCommand).absolutePath else finalCommand
         }
 
+        fun getIvyDependency(): String {
+            val type = if (isWindows) "zip" else "tar.gz"
+            return "org.nodejs:node:$nodeVersion:$platform-$architecture@$type"
+        }
+
+        return NodeJsEnv(
+            nodeDir = nodeDir,
+            nodeBinDir = nodeBinDir,
+            nodeExec = getExecutable("node", nodeCommand, "exe"),
+            npmExec = getExecutable("npm", npmCommand, "cmd"),
+            platformName = platform,
+            architectureName = architecture,
+            ivyDependency = getIvyDependency()
+        )
+    }
+
     internal fun executeSetup() {
-        val nodeJsEnv = environment
+        val nodeJsEnv = buildEnv()
         if (download) {
             if (!nodeJsEnv.nodeBinDir.isDirectory) {
                 nodeJsSetupTask.exec()
@@ -66,6 +69,6 @@ open class NodeJsRootExtension(project: Project) : NodeJsExtension(project) {
     companion object {
         const val NODE_JS: String = "nodeJs"
 
-        operator fun get(project: Project) = NodeJsPlugin.apply(project)
+        operator fun get(project: Project): NodeJsRootExtension = NodeJsPlugin[project]
     }
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsSetupTask.kt

@@ -11,8 +11,8 @@ import java.io.File
 import java.net.URI
 
 open class NodeJsSetupTask : DefaultTask() {
-    private val settings = project.nodeJs.root
-    private val env by lazy { settings.environment }
+    private val settings = NodeJsRootExtension[project]
+    private val env by lazy { settings.buildEnv() }
 
     init {
         group = NodeJsRootExtension.NODE_JS
@@ -55,7 +55,7 @@ open class NodeJsSetupTask : DefaultTask() {
             }
         }
 
-        val dep = this.project.dependencies.create(ivyDependency)
+        val dep = this.project.dependencies.create(env.ivyDependency)
         val conf = this.project.configurations.detachedConfiguration(dep)
         conf.isTransitive = false
         val result = conf.resolve().single()
@@ -66,7 +66,7 @@ open class NodeJsSetupTask : DefaultTask() {
         unpackNodeArchive(result, destination.parentFile) // parent because archive contains name already
 
         if (!env.isWindows) {
-            File(env.nodeExecutable).setExecutable(true)
+            File(env.nodeExec).setExecutable(true)
         }
     }
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/utils.kt

@@ -5,15 +5,4 @@
 
 package org.jetbrains.kotlin.gradle.targets.js.nodejs
 
-import org.gradle.api.Project
-
-internal fun nodeJsSetupWithoutTasks(project: Project) {
-    val nodeJsProject = NodeJsPlugin.ensureAppliedInHierarchy(project)
-    val nodeJs = NodeJsExtension[nodeJsProject]
-    val nodeJsEnv = nodeJs.buildEnv()
-    if (nodeJs.download) {
-        if (!nodeJsEnv.nodeBinDir.isDirectory) {
-            (nodeJsProject.tasks.findByName(NodeJsSetupTask.NAME) as NodeJsSetupTask).exec()
-        }
-    }
-}
+import org.gradle.api.Project

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/GradleNodeModulesSync.kt

@@ -0,0 +1,241 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.targets.js.npm
+
+import com.google.gson.GsonBuilder
+import com.google.gson.JsonParseException
+import org.gradle.api.Project
+import org.gradle.api.internal.project.ProjectInternal
+import org.gradle.internal.hash.FileHasher
+import org.jetbrains.kotlin.daemon.common.toHexString
+import org.jetbrains.kotlin.gradle.dsl.KotlinCommonOptions
+import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation
+import org.jetbrains.kotlin.gradle.targets.js.npm.GradleNodeModulesSync.*
+import org.jetbrains.kotlin.gradle.targets.js.npm.NpmProjectLayout.Companion.PACKAGE_JSON
+import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile
+import java.io.File
+
+/**
+ * Incrementally sync gradle JS dependencies into node_modules directory.
+ *
+ * Working by traversing giving classpath.
+ * For each classpath element (called [Element]) `.js` and `.js.map` files are stored into `node_modules`.
+ *
+ * To do it incrementally, state of previously traversed classpath elements are stored in
+ * `node_modules/.from-gradle` file. Unchanged elements will not been visited.
+ *
+ * See [Element] and [getOrCompute] for more details.
+ */
+internal class GradleNodeModulesSync(val project: Project) {
+    companion object {
+        const val STATE_FILE_NAME = ".from-gradle"
+    }
+
+    private val hasher = (project as ProjectInternal).services.get(FileHasher::class.java)
+    private val npmProjectLayout = NpmProjectLayout[project]
+    private val nodeModulesDir = npmProjectLayout.nodeModulesDir
+    private val gson = GsonBuilder().setPrettyPrinting().create()
+
+    private class State {
+        val contents: MutableMap<String, Element> = mutableMapOf()
+
+        operator fun get(elementHash: ByteArray) =
+            contents[elementHash.toHexString()]
+
+        operator fun set(elementHash: ByteArray, element: Element) {
+            contents[elementHash.toHexString()] = element
+        }
+    }
+
+    /**
+     * Traversed element from classpath.
+     * Stored in state file.
+     *
+     * When classpath traversed all elements are in one of this states:
+     * - unchanged: was loaded from previous state and contents not changed.
+     *   only [fileNames] are known and contents of element not visited,
+     *   but visitor are saved in [_fillFiles] to load in next case
+     * - unchanged but missed in node_modules: some of [fileNames] missed in node_modules
+     *   in this case [_fillFiles] called before [files] was getted for copy spec
+     * - unchanged and existed in node_modules: nothing will be done in this case
+     * - new: not existed in previous state. In this case element is visited,
+     *   both [fileNames] and [_files] are filled already
+     * - deleted: was loaded from previous state, but not existed in current state
+     *   only [fileNames] are known for this case, which is used to remove outdated files
+     * - changed: this is just combination of deleted and new state
+     */
+    private class Element {
+        lateinit var source: String
+        val fileNames: MutableList<String> = mutableListOf()
+
+        @Transient
+        @Suppress("PropertyName")
+        var _files: List<File>? = null
+
+        @Transient
+        @Suppress("PropertyName")
+        var _fillFiles: (() -> List<File>)? = null
+
+        val files: List<File>
+            get() {
+                if (_files == null) {
+                    _files = _fillFiles!!()
+                }
+                return _files!!
+            }
+    }
+
+    /**
+     * Get info of classpath element from previous state (with storing visitor for case when files are deleted in node_modules)
+     * or compute it now.
+     *
+     * @param file classpath element
+     * @param buildElement js files list provider
+     */
+    private fun getOrCompute(file: File, buildElement: () -> List<File>) {
+        val hash = hasher.hash(file).toByteArray()
+        val old = old[hash]
+        if (old != null) old._fillFiles = buildElement
+        new[hash] = old ?: Element().also {
+            it.source = file.absolutePath
+            it._files = buildElement()
+            it._files!!.mapTo(it.fileNames) { it.name }
+        }
+    }
+
+    private lateinit var old: State
+    private val new = State()
+
+    private val stateFile = nodeModulesDir.resolve(STATE_FILE_NAME)
+
+    fun loadOldState() {
+        val state: State? = if (stateFile.isFile) try {
+            stateFile.reader().use {
+                gson.fromJson(it, State::class.java)
+            }
+        } catch (e: JsonParseException) {
+            project.logger.warn("Cannot load previous state of Gradle depencies stored in node_modules", e)
+            null
+        } else {
+            if (stateFile.isDirectory) stateFile.deleteRecursively()
+            null
+        }
+
+        old = state ?: State()
+    }
+
+    fun sync() {
+        val children = nodeModulesDir.list()?.toSet() ?: setOf()
+        val deleted = old.contents.keys.toMutableSet()
+        val absented = mutableSetOf<String>()
+
+        new.contents.forEach { (key, value) ->
+            deleted.remove(key)
+            if (key !in old.contents.keys) absented.add(key)
+            else if (value.fileNames.any { it !in children }) absented.add(key)
+        }
+
+        deleted.forEach {
+            old.contents[it]!!.fileNames.forEach { fileName ->
+                nodeModulesDir.resolve(fileName).delete()
+            }
+        }
+
+        if (absented.isNotEmpty()) {
+            project.copy { copy ->
+                absented.forEach { key ->
+                    copy.from(new.contents[key]!!.files)
+                }
+                copy.into(nodeModulesDir)
+            }
+        }
+
+        if (deleted.isNotEmpty() || absented.isNotEmpty()) {
+            saveNewState()
+        }
+    }
+
+    private fun saveNewState() {
+        stateFile.writer().use {
+            gson.toJson(new, it)
+        }
+    }
+
+    class TransitiveNpmDependency(
+        val key: String,
+        val version: String
+    )
+
+    fun visitCompilation(
+        compilation: KotlinCompilation<KotlinCommonOptions>,
+        project: Project,
+        transitiveDependencies: MutableList<TransitiveNpmDependency>
+    ): List<TransitiveNpmDependency> {
+        val kotlin2JsCompile = compilation.compileKotlinTask as Kotlin2JsCompile
+
+        // classpath
+        kotlin2JsCompile.classpath.forEach { srcFile ->
+            when {
+                srcFile.name == PACKAGE_JSON -> visitPackageJson(srcFile, transitiveDependencies)
+                isKotlinJsRuntimeFile(srcFile) -> getOrCompute(srcFile) {
+                    listOf(srcFile)
+                }
+                srcFile.isZip -> getOrCompute(srcFile) {
+                    mutableListOf<File>().also { files ->
+                        this.project.zipTree(srcFile).forEach { innerFile ->
+                            when {
+                                innerFile.name == PACKAGE_JSON -> visitPackageJson(innerFile, transitiveDependencies)
+                                isKotlinJsRuntimeFile(innerFile) -> files.add(innerFile)
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        // output
+        if (kotlin2JsCompile.state.executed) {
+            copyCompileOutput(project, kotlin2JsCompile)
+        } else {
+            kotlin2JsCompile.doLast {
+                copyCompileOutput(project, kotlin2JsCompile)
+            }
+        }
+
+        return transitiveDependencies
+    }
+
+    private fun copyCompileOutput(project: Project, kotlin2JsCompile: Kotlin2JsCompile) {
+        project.copy { copy ->
+            copy.from(kotlin2JsCompile.outputFile)
+            copy.from(kotlin2JsCompile.outputFile.path + ".map")
+            copy.into(nodeModulesDir)
+        }
+    }
+
+    private fun visitPackageJson(
+        innerFile: File,
+        transitiveDependencies: MutableList<TransitiveNpmDependency>
+    ) {
+        val packageJson = innerFile.reader().use {
+            gson.fromJson(it, PackageJson::class.java)
+        }
+
+        packageJson.dependencies.forEach { (key, version) ->
+            transitiveDependencies.add(TransitiveNpmDependency(key, version))
+        }
+    }
+
+    private val File.isZip
+        get() = isFile && (name.endsWith(".jar") || name.endsWith(".zip"))
+
+    private fun isKotlinJsRuntimeFile(file: File): Boolean {
+        if (!file.isFile) return false
+        val name = file.name
+        return (name.endsWith(".js") && !name.endsWith(".meta.js"))
+                || name.endsWith(".js.map")
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NodeModulesGradleComponents.kt

@@ -1,148 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.gradle.targets.js.npm
-
-import com.google.gson.GsonBuilder
-import com.google.gson.JsonIOException
-import com.google.gson.JsonSyntaxException
-import org.gradle.api.Project
-import org.gradle.api.internal.project.ProjectInternal
-import org.gradle.internal.hash.FileHasher
-import org.jetbrains.kotlin.gradle.dsl.KotlinCommonOptions
-import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation
-import org.jetbrains.kotlin.gradle.targets.js.tasks.isKotlinJsRuntimeFile
-import org.jetbrains.kotlin.gradle.targets.js.tasks.isZip
-import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile
-import java.io.File
-
-class NodeModulesGradleComponents(val rootProject: Project) {
-    val hasher = (rootProject as ProjectInternal).services.get(FileHasher::class.java)
-    val nodeModulesDir = rootProject.rootDir.resolve("node_modules")
-    val gson = GsonBuilder().setPrettyPrinting().create()
-
-    private class State {
-        val contents: MutableMap<ByteArray, Element> = mutableMapOf()
-    }
-
-    private class Element {
-        val fileNames: MutableList<String> = mutableListOf()
-
-        @Transient
-        val files = mutableListOf<File>()
-    }
-
-    class TransitiveNpmDependency(
-        val project: String,
-        val from: String,
-        val key: String,
-        val version: String
-    )
-
-    private lateinit var old: State
-    private val new = State()
-
-    val stateFile = rootProject.buildDir.resolve("nodeModulesGradleComponents.json")
-
-    fun loadOldState() {
-        val state: State? = if (stateFile.exists()) try {
-            stateFile.reader().use {
-                gson.fromJson(it, State::class.java)
-            }
-        } catch (e: JsonSyntaxException) {
-            // todo: warn
-            null
-        } catch (e: JsonIOException) {
-            // todo: warn
-            null
-        } else null
-
-        old = state ?: State()
-    }
-
-    fun sync() {
-        val children = nodeModulesDir.list()?.toSet() ?: setOf()
-        val deleted = old.contents.keys.toMutableSet()
-        val absense = mutableSetOf<ByteArray>()
-
-        new.contents.forEach { (key, value) ->
-            deleted.remove(key)
-            if (key !in old.contents.keys) absense.add(key)
-            else if (value.fileNames.any { it !in children }) absense.add(key)
-        }
-
-        deleted.forEach {
-            old.contents[it]!!.fileNames.forEach { fileName ->
-                nodeModulesDir.resolve(fileName).delete()
-            }
-        }
-
-        rootProject.copy { copy ->
-            absense.forEach { key ->
-                copy.from(new.contents[key]!!.files)
-                copy.into(nodeModulesDir)
-            }
-        }
-
-        saveNewState()
-    }
-
-    private fun saveNewState() {
-        stateFile.writer().use {
-            gson.toJson(new, it)
-        }
-    }
-
-    private fun getOrCompute(file: File, buildElement: (Element) -> Unit) {
-        val hash = hasher.hash(file).toByteArray()
-        new.contents[hash] = old.contents[hash] ?: Element().also {
-            buildElement(it)
-            it.files.mapTo(it.fileNames) { it.name }
-        }
-    }
-
-    fun visitCompilation(
-        compilation: KotlinCompilation<KotlinCommonOptions>,
-        project: Project,
-        transitiveDependencies: MutableList<TransitiveNpmDependency>
-    ): List<TransitiveNpmDependency> {
-        val kotlin2JsCompile = compilation.compileKotlinTask as Kotlin2JsCompile
-
-        // classpath
-        kotlin2JsCompile.classpath.forEach { srcFile ->
-            when {
-                isKotlinJsRuntimeFile(srcFile) -> getOrCompute(srcFile) { element ->
-                    element.files.add(srcFile)
-                }
-                srcFile.isZip -> getOrCompute(srcFile) { element ->
-                    rootProject.zipTree(srcFile).forEach { innerFile ->
-                        if (innerFile.name == "package.json") {
-                            val packageJson = innerFile.reader().use {
-                                gson.fromJson(it, PackageJson::class.java)
-                            }
-
-                            packageJson.dependencies.forEach { (key, version) ->
-                                transitiveDependencies.add(TransitiveNpmDependency(project.path, srcFile.path, key, version))
-                            }
-                        } else if (isKotlinJsRuntimeFile(innerFile)) {
-                            element.files.add(innerFile)
-                        }
-                    }
-                }
-            }
-        }
-
-        // output
-        kotlin2JsCompile.doLast {
-            project.copy { copy ->
-                copy.from(kotlin2JsCompile.outputFile)
-                copy.from(kotlin2JsCompile.outputFile.path + ".map")
-                copy.into(nodeModulesDir)
-            }
-        }
-
-        return transitiveDependencies
-    }
-}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/Npm.kt

@@ -1,19 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.gradle.targets.js.npm
-
-import org.gradle.api.Project
-import org.jetbrains.kotlin.gradle.targets.js.nodejs.nodeJsSetupWithoutTasks
-
-object Npm: NpmApi {
-    override fun setup(project: Project) {
-        nodeJsSetupWithoutTasks(project)
-    }
-
-    override fun resolveRootProject(project: Project) {
-        TODO("not implemented") //To change body of created functions use File | Settings | File Templates.
-    }
-}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmApi.kt

@@ -27,13 +27,6 @@ interface NpmApi {
     open val hoistGradleNodeModules
         get() = false
 
-    fun cleanProject(project: Project) {
-        val npmProject = project.npmProject
-
-        npmProject.nodeModulesDir.deleteRecursively()
-        npmProject.packageJsonFile.delete()
-    }
-
     companion object {
         fun resolveOperationDescription(packageManagerTitle: String): String =
             "Resolving NPM dependencies using $packageManagerTitle"

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmDependency.kt

@@ -10,74 +10,50 @@ import org.gradle.api.artifacts.Dependency
 import org.gradle.api.artifacts.FileCollectionDependency
 import org.gradle.api.artifacts.SelfResolvingDependency
 import org.gradle.api.file.FileCollection
-import org.gradle.api.internal.artifacts.DependencyResolveContext
-import org.gradle.api.internal.artifacts.ResolvableDependency
 import org.gradle.api.internal.artifacts.dependencies.SelfResolvingDependencyInternal
 import org.gradle.api.tasks.TaskDependency
+import java.io.File
 import org.gradle.internal.component.local.model.DefaultLibraryBinaryIdentifier
 import org.jetbrains.kotlin.gradle.targets.js.npm.NpmResolver.ResolutionCallResult.*
-import java.io.File
 
 data class NpmDependency(
-    internal val project: Project,
+    private val project: Project,
     private val org: String?,
     private val name: String,
     private val version: String
 ) : SelfResolvingDependency,
     SelfResolvingDependencyInternal,
-    ResolvableDependency,
     FileCollectionDependency {
     override fun getGroup(): String? = org
 
-    internal val dependencies = mutableSetOf<NpmDependency>()
-
     override fun resolve(): MutableSet<File> {
-        resolveProject() ?: return mutableSetOf()
-
-        val all = mutableSetOf<File>()
-        val visited = mutableSetOf<NpmDependency>()
-
-        fun visit(item: NpmDependency) {
-            if (item in visited) return
-            visited.add(item)
-            item.tryFindNodeModule()?.let {
-                all.add(it)
-            }
-        }
-
-        visit(this)
-
-        return all
-    }
-
-    internal fun resolveProject(): NpmResolver.ResolvedProject? {
         val result = NpmResolver.resolve(project)
 
         return when (result) {
-            is AlreadyInProgress -> null
+            is AlreadyInProgress -> mutableSetOf()
             is AlreadyResolved -> {
                 check(this in result.resolution.dependencies) {
                     "Project hierarchy is already resolved in NPM without $this"
                 }
 
-                result.resolution
+                tryFindNodeModule()
             }
             is ResolvedNow -> {
                 check(this in result.resolution.dependencies) {
                     "$this was not visited during resolution"
                 }
 
-                result.resolution
+                tryFindNodeModule()
             }
         }
     }
 
-    private fun tryFindNodeModule(): File? {
+    private fun tryFindNodeModule(): MutableSet<File> {
         var p = project
         do {
-            val result = NpmProject[p].nodeModulesDir.resolve(key)
-            if (result.exists()) return result
-            p = project.parent ?: return null
+            val result = p.rootDir.resolve("node_modules/$key")
+            if (result.exists()) return mutableSetOf(result)
+            p = project.parent ?: return mutableSetOf()
         } while (true)
     }
 
@@ -85,13 +61,6 @@ data class NpmDependency(
 
     override fun toString() = "$key: $version"
 
-    override fun resolve(context: DependencyResolveContext) {
-        resolve()
-        dependencies.forEach {
-            context.add(it)
-        }
-    }
-
     override fun resolve(transitive: Boolean): MutableSet<File> = resolve()
 
     override fun getFiles(): FileCollection = project.files(resolve())

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmProjectLayout.kt

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.targets.js.npm
+
+import org.gradle.api.Project
+import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin
+import java.io.File
+
+class NpmProjectLayout(val nodeWorkDir: File) {
+    val nodeModulesDir
+        get() = nodeWorkDir.resolve(NODE_MODULES)
+
+    val packageJsonFile: File
+        get() {
+            return nodeWorkDir.resolve(PACKAGE_JSON)
+        }
+
+    companion object {
+        const val PACKAGE_JSON = "package.json"
+        const val NODE_MODULES = "node_modules"
+
+        operator fun get(project: Project): NpmProjectLayout {
+            val manageNodeModules = NodeJsPlugin[project].manageNodeModules
+
+            val nodeWorkDir =
+                if (manageNodeModules) project.rootDir
+                else project.buildDir
+
+            return NpmProjectLayout(nodeWorkDir)
+        }
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmResolver.kt

@@ -10,21 +10,23 @@ import com.google.gson.GsonBuilder
 import org.gradle.api.Project
 import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension
 import org.jetbrains.kotlin.gradle.dsl.KotlinSingleTargetExtension
-import org.jetbrains.kotlin.gradle.dsl.kotlinExtensionOrNull
+import org.jetbrains.kotlin.gradle.dsl.kotlinExtension
 import org.jetbrains.kotlin.gradle.plugin.KotlinPlatformType
 import org.jetbrains.kotlin.gradle.plugin.KotlinTarget
+import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsExtension
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension
-import org.jetbrains.kotlin.gradle.targets.js.nodejs.nodeJs
 import org.jetbrains.kotlin.gradle.targets.js.npm.NpmResolver.ResolutionCallResult.*
 
 /**
- * Generates `package.json` file for projects with npm or js dependencies and
- * runs selected [NodeJsRootExtension.packageManager] to download and install it.
+ * [NpmResolver] runs selected [NodeJsRootExtension.packageManager] on root project
+ * with configured `package.json` files in all projects.
  *
- * All [NpmDependency] for configurations related to kotlin/js will be added to `package.json`.
- * For external gradle modules, fake npm packages will be created and added to `package.json`
- * as path to directory.
+ * `package.json`
+ *
+ * - compile(npm(...)) dependencies
+ * - compile(npm(...)) dependencies from external modules
+ * - compile(non npm(...)) dependencies transitively
  */
 internal class NpmResolver private constructor(val rootProject: Project) {
     companion object {
@@ -38,11 +40,7 @@ internal class NpmResolver private constructor(val rootProject: Project) {
             val resolved = process?.resolved
 
             return if (resolved != null) AlreadyResolved(resolved)
-            else {
-                val resolver = NpmResolver(rootProject)
-                check(resolver.resolve(rootProject, null))
-                ResolvedNow(ResolvedProject(resolver.npmPackages))
-            }
+            else ResolvedNow(NpmResolver(rootProject).resolve(rootProject, null)!!)
         }
     }
 
@@ -52,10 +50,9 @@ internal class NpmResolver private constructor(val rootProject: Project) {
         class ResolvedNow(val resolution: ResolvedProject) : ResolutionCallResult()
     }
 
-    private val nodeJs = NodeJsPlugin.apply(rootProject).root
-    private val npmProject = rootProject.npmProject
+    private val nodeJs = NodeJsPlugin[rootProject]
     private val packageManager = nodeJs.packageManager
-    private val hoistGradleNodeModules = npmProject.hoistGradleNodeModules
+    private val hoistGradleNodeModules = packageManager.hoistGradleNodeModules
     private val npmPackages = mutableListOf<NpmPackage>()
     private val gson = GsonBuilder()
         .setPrettyPrinting()
@@ -69,11 +66,7 @@ internal class NpmResolver private constructor(val rootProject: Project) {
         }
     }
 
-    class ResolvedProject(val npmPackages: Collection<NpmPackage>) {
-        val dependencies by lazy {
-            npmPackages.flatMapTo(mutableSetOf()) { it.npmDependencies }
-        }
-    }
+    class ResolvedProject(val dependencies: Set<NpmDependency>)
 
     class NpmPackage(
         val project: Project,
@@ -81,7 +74,7 @@ internal class NpmResolver private constructor(val rootProject: Project) {
         val npmDependencies: Set<NpmDependency>
     ) {
         fun savePackageJson(gson: Gson) {
-            NpmProject[project].packageJsonFile.writer().use {
+            NpmProjectLayout[project].packageJsonFile.writer().use {
                 gson.toJson(packageJson, it)
             }
         }
@@ -97,29 +90,29 @@ internal class NpmResolver private constructor(val rootProject: Project) {
 
     private fun resolve(
         project: Project,
-        parentGradleModules: GradleNodeModules?
-    ): Boolean {
+        parentGradleNodeModules: GradleNodeModulesSync?
+    ): ResolvedProject? {
         val existedProcess = ProjectData[project]
-        if (existedProcess != null) {
+        val process = if (existedProcess != null) {
             if (existedProcess.resolved != null) error("yarn dependencies for $project already resolved")
             else {
                 // called inside resolution of classpath (from visitTarget)
                 // we should return as we are already in progress
-                return false
+                return null
             }
         } else ProjectData().also {
             ProjectData[project] = it
         }
 
-        val gradleModules =
-            if (hoistGradleNodeModules && parentGradleModules != null) parentGradleModules
-            else GradleNodeModules(project)
+        val gradleNodeModules =
+            if (hoistGradleNodeModules && parentGradleNodeModules != null) parentGradleNodeModules
+            else newGradleNodeModulesSync(project)
 
         project.subprojects.forEach {
-            resolve(it, gradleModules)
+            resolve(it, gradleNodeModules)
         }
 
-        val npmPackage = extractNpmPackage(project, gradleModules)
+        val npmPackage = extractNpmPackage(project, gradleNodeModules)
         if (npmPackage != null) {
             npmPackage.savePackageJson(gson)
 
@@ -136,60 +129,61 @@ internal class NpmResolver private constructor(val rootProject: Project) {
             }
         }
 
-        if (gradleModules != parentGradleModules) {
-            gradleModules.close()
+        if (gradleNodeModules != parentGradleNodeModules) {
+            gradleNodeModules.sync()
         }
 
-        return true
+        val npmDependencies = npmPackage?.npmDependencies ?: setOf()
+        return ResolvedProject(npmDependencies).also {
+            process.resolved = it
+        }
+    }
+
+    private fun newGradleNodeModulesSync(project: Project): GradleNodeModulesSync {
+        return GradleNodeModulesSync(project).also {
+            it.loadOldState()
+        }
     }
 
     private fun extractNpmPackage(
         project: Project,
-        gradleComponents: GradleNodeModules
+        gradleComponentsSync: GradleNodeModulesSync
     ): NpmPackage? {
         val packageJson = PackageJson(project.name, project.version.toString())
+        val transitiveDependencies = mutableListOf<GradleNodeModulesSync.TransitiveNpmDependency>()
+
+        val kotlin = project.kotlinExtension
         val npmDependencies = mutableSetOf<NpmDependency>()
-
-        visitNpmDependencies(project, npmDependencies, gradleComponents, packageJson)
-
-        if (!hoistGradleNodeModules || project == project.rootProject) {
-            gradleComponents.modules.forEach {
-                val relativePath = it.path.relativeTo(NpmProject[project].nodeWorkDir)
-                packageJson.dependencies[it.name] = "file:$relativePath"
+        when (kotlin) {
+            is KotlinSingleTargetExtension -> visitTarget(
+                kotlin.target,
+                project,
+                npmDependencies,
+                gradleComponentsSync,
+                transitiveDependencies
+            )
+            is KotlinMultiplatformExtension -> kotlin.targets.forEach {
+                visitTarget(it, project, npmDependencies, gradleComponentsSync, transitiveDependencies)
             }
         }
 
-        project.nodeJs.packageJsonHandlers.forEach {
+        npmDependencies.forEach {
+            packageJson.dependencies[it.key] = chooseVersion(packageJson.dependencies[it.key], it.version)
+        }
+
+        transitiveDependencies.forEach {
+            packageJson.dependencies[it.key] = chooseVersion(packageJson.dependencies[it.key], it.version)
+        }
+
+        NodeJsExtension[project].packageJsonHandlers.forEach {
             it(packageJson)
         }
 
-        val packageJsonRequired = if (project == rootProject) {
+        if (project == rootProject) {
             packageManager.hookRootPackage(rootProject, packageJson, npmPackages)
-        } else false
-
-        return if (packageJsonRequired || !packageJson.empty) NpmPackage(project, packageJson, npmDependencies) else null
-    }
-
-    private fun visitNpmDependencies(
-        project: Project,
-        npmDependencies: MutableSet<NpmDependency>,
-        gradleComponents: GradleNodeModules,
-        packageJson: PackageJson
-    ) {
-        val kotlin = project.kotlinExtensionOrNull
-
-        if (kotlin != null) {
-            when (kotlin) {
-                is KotlinSingleTargetExtension -> visitTarget(kotlin.target, project, npmDependencies, gradleComponents)
-                is KotlinMultiplatformExtension -> kotlin.targets.forEach {
-                    visitTarget(it, project, npmDependencies, gradleComponents)
-                }
-            }
-
-            npmDependencies.forEach {
-                packageJson.dependencies[it.key] = chooseVersion(packageJson.dependencies[it.key], it.version)
-            }
         }
+
+        return if (!packageJson.empty) NpmPackage(project, packageJson, npmDependencies) else null
     }
 
     private fun chooseVersion(oldVersion: String?, newVersion: String): String =
@@ -199,7 +193,8 @@ internal class NpmResolver private constructor(val rootProject: Project) {
         target: KotlinTarget,
         project: Project,
         npmDependencies: MutableSet<NpmDependency>,
-        gradleComponents: GradleNodeModules
+        gradleComponentsSync: GradleNodeModulesSync,
+        transitiveDependencies: MutableList<GradleNodeModulesSync.TransitiveNpmDependency>
     ) {
         if (target.platformType == KotlinPlatformType.js) {
             target.compilations.toList().forEach { compilation ->
@@ -211,7 +206,7 @@ internal class NpmResolver private constructor(val rootProject: Project) {
                     }
                 }
 
-                gradleComponents.visitCompilation(compilation)
+                gradleComponentsSync.visitCompilation(compilation, project, transitiveDependencies)
             }
         }
     }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/PackageJson.kt

@@ -10,34 +10,11 @@ class PackageJson(
     var version: String
 ) {
     val empty: Boolean
-        get() = private == null &&
-                workspaces == null &&
-                dependencies.isEmpty() &&
-                devDependencies.isEmpty()
-
-    val scopedName: ScopedName
-        get() = scopedName(name)
+        get() = private == null && workspaces == null && dependencies.isEmpty()
 
     var private: Boolean? = null
 
-    var main: String? = null
-
     var workspaces: Collection<String>? = null
 
-    val devDependencies = mutableMapOf<String, String>()
     val dependencies = mutableMapOf<String, String>()
-
-    companion object {
-        fun scopedName(name: String): ScopedName = if (name.contains("/")) ScopedName(
-            scope = name.substringBeforeLast("/").removePrefix("@"),
-            name = name.substringAfterLast("/")
-        ) else ScopedName(scope = null, name = name)
-
-        operator fun invoke(scope: String, name: String, version: String) =
-            PackageJson(ScopedName(scope, name).toString(), version)
-    }
-
-    data class ScopedName(val scope: String?, val name: String) {
-        override fun toString() = if (scope == null) name else "@$scope/$name"
-    }
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/tasks/KotlinJsNodeModulesTask.kt

@@ -1,63 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.gradle.targets.js.tasks
-
-import org.gradle.api.DefaultTask
-import org.gradle.api.file.FileCollection
-import org.gradle.api.tasks.*
-import org.jetbrains.kotlin.gradle.targets.js.internal.RewriteSourceMapFilterReader
-import java.io.File
-
-open class KotlinJsNodeModulesTask : DefaultTask() {
-    @OutputDirectory
-    @SkipWhenEmpty
-    lateinit var nodeModulesDir: File
-
-    @Classpath
-    @InputFiles
-    @SkipWhenEmpty
-    lateinit var classpath: FileCollection
-
-    @TaskAction
-    fun copyFromRuntimeClasspath() {
-        project.sync { sync ->
-            sync.includeEmptyDirs = false
-
-            classpath.forEach {
-                if (it.isZip) sync.from(project.zipTree(it))
-                else sync.from(it)
-            }
-
-            sync.include { fileTreeElement ->
-                isKotlinJsRuntimeFile(fileTreeElement.file)
-            }
-
-            sync.eachFile {
-                if (it.name.endsWith(".js.map")) {
-                    it.filter(
-                        mapOf(
-                            "srcSourceRoot" to it.file.parentFile,
-                            "targetSourceRoot" to nodeModulesDir
-                        ),
-                        RewriteSourceMapFilterReader::class.java
-                    )
-                }
-            }
-
-            sync.into(nodeModulesDir)
-        }
-    }
-}
-
-internal val File.isZip
-    get() = isFile && (name.endsWith(".jar") || name.endsWith(".zip"))
-
-internal fun isKotlinJsRuntimeFile(file: File): Boolean {
-    if (!file.isFile) return false
-    val name = file.name
-    return (name.endsWith(".js") && !name.endsWith(".meta.js"))
-            || name.endsWith(".js.map")
-}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/tasks/KotlinNodeJsTestTask.kt

@@ -15,6 +15,8 @@ import org.jetbrains.kotlin.gradle.internal.testing.TCServiceMessagesClientSetti
 import org.jetbrains.kotlin.gradle.internal.testing.TCServiceMessagesTestExecutionSpec
 import org.jetbrains.kotlin.gradle.targets.js.internal.parseNodeJsStackTrace
 import org.jetbrains.kotlin.gradle.targets.js.internal.parseNodeJsStackTraceAsJvm
+import org.jetbrains.kotlin.gradle.targets.js.npm.NpmProjectLayout
+import org.jetbrains.kotlin.gradle.targets.js.npm.NpmResolver
 import org.jetbrains.kotlin.gradle.tasks.KotlinTestTask
 import org.jetbrains.kotlin.gradle.testing.IgnoredTestSuites
 import java.io.File
@@ -24,9 +26,6 @@ open class KotlinNodeJsTestTask : KotlinTestTask() {
     var ignoredTestSuites: IgnoredTestSuites =
         IgnoredTestSuites.showWithContents
 
-    @InputDirectory
-    lateinit var nodeModulesDir: File
-
     @Input
     @SkipWhenEmpty
     var nodeModulesToLoad: Set<String> = setOf()
@@ -53,11 +52,19 @@ open class KotlinNodeJsTestTask : KotlinTestTask() {
         options(nodeJsProcessOptions)
     }
 
+    override fun executeTests() {
+        NpmResolver.resolve(project)
+        super.executeTests()
+    }
+
     override fun createTestExecutionSpec(): TCServiceMessagesTestExecutionSpec {
         val extendedForkOptions = DefaultProcessForkOptions(fileResolver)
         nodeJsProcessOptions.copyTo(extendedForkOptions)
 
-        extendedForkOptions.environment.addPath("NODE_PATH", nodeModulesDir.canonicalPath)
+        NpmResolver.resolve(project)
+
+        val npmProjectLayout = NpmProjectLayout[project]
+        extendedForkOptions.workingDir = npmProjectLayout.nodeWorkDir
 
         val nodeJsArgs = mutableListOf<String>()
 
@@ -83,7 +90,7 @@ open class KotlinNodeJsTestTask : KotlinTestTask() {
             extendedForkOptions,
             nodeJsArgs +
                     testRuntimeNodeModules
-                        .map { nodeModulesDir.resolve(it) }
+                        .map { npmProjectLayout.nodeModulesDir.resolve(it) }
                         .filter { it.exists() }
                         .map { it.absolutePath } +
                     cliArgs.toList(),
@@ -122,10 +129,4 @@ data class KotlinNodeJsTestRunnerCliArgs(
     enum class IgnoredTestSuitesReporting {
         skip, reportAsIgnoredTest, reportAllInnerTestsAsIgnored
     }
-}
-
-private fun MutableMap<String, Any>.addPath(key: String, path: String) {
-    val prev = get(key)
-    if (prev == null) set(key, path)
-    else set(key, prev as String + File.pathSeparator + path)
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/Yarn.kt

@@ -6,121 +6,56 @@
 package org.jetbrains.kotlin.gradle.targets.js.yarn
 
 import org.gradle.api.Project
-import org.gradle.api.internal.project.ProjectInternal
-import org.gradle.internal.hash.FileHasher
-import org.jetbrains.kotlin.daemon.common.toHexString
 import org.jetbrains.kotlin.gradle.internal.execWithProgress
+import org.jetbrains.kotlin.gradle.internal.operation
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin
-import org.jetbrains.kotlin.gradle.targets.js.npm.*
-import java.io.File
+import org.jetbrains.kotlin.gradle.targets.js.npm.NpmApi
+import org.jetbrains.kotlin.gradle.targets.js.npm.NpmProjectLayout
+import org.jetbrains.kotlin.gradle.targets.js.npm.NpmResolver
+import org.jetbrains.kotlin.gradle.targets.js.npm.PackageJson
+import org.slf4j.LoggerFactory
 
 object Yarn : NpmApi {
-    override fun setup(project: Project) {
-        YarnPlugin.apply(project).executeSetup()
-    }
+    private val log = LoggerFactory.getLogger("org.jetbrains.kotlin.gradle.targets.js.yarn.Yarn")
 
-    val Project.packageJsonHashFile: File
-        get() = buildDir.resolve("package.json.hash")
+    override fun setup(project: Project) {
+        YarnPlugin[project].executeSetup()
+    }
 
     fun yarnExec(
         project: Project,
         description: String,
         vararg args: String,
-        npmProject: NpmProject = NpmProject[project]
+        npmProjectLayout: NpmProjectLayout = NpmProjectLayout[project]
     ) {
-        val nodeJsEnv = NodeJsPlugin.apply(project).root.environment
-        val yarnEnv = YarnPlugin.apply(project).environment
-
-        val packageJsonHashFile = project.packageJsonHashFile
-        val packageJsonHash = if (packageJsonHashFile.exists()) packageJsonHashFile.readText() else null
-
-        val hasher = (project as ProjectInternal).services.get(FileHasher::class.java)
-        val hash = hasher.hash(npmProject.packageJsonFile).toByteArray().toHexString()
-
-        if (packageJsonHash == hash) return
-
-        packageJsonHashFile.writeText(hash)
-
-        val nodeWorkDir = npmProject.nodeWorkDir
+        val nodeJsEnv = NodeJsPlugin[project].buildEnv()
+        val yarnEnv = YarnPlugin[project].buildEnv()
 
         project.execWithProgress(description) { exec ->
-            exec.executable = nodeJsEnv.nodeExecutable
+            exec.executable = nodeJsEnv.nodeExec
             exec.args = listOf(yarnEnv.home.resolve("bin/yarn.js").absolutePath) + args
-            exec.workingDir = nodeWorkDir
-        }
-    }
-
-    private fun yarnLockReadTransitiveDependencies(
-        nodeWorkDir: File,
-        srcDependenciesList: Collection<NpmDependency>
-    ) {
-        val yarnLock = nodeWorkDir.resolve("yarn.lock")
-        if (yarnLock.isFile) {
-            val byKey = YarnLock.parse(yarnLock).entries.associateBy { it.key }
-            val visited = mutableSetOf<NpmDependency>()
-
-            fun resolveRecursively(src: NpmDependency): NpmDependency {
-                if (src in visited) return src
-                visited.add(src)
-
-                val key = YarnLock.key(src.key, src.version)
-                val deps = byKey[key]
-                if (deps != null) {
-                    src.dependencies.addAll(deps.dependencies.map { dep ->
-                        val scopedName = dep.scopedName
-
-                        resolveRecursively(
-                            NpmDependency(
-                                src.project,
-                                scopedName.scope,
-                                scopedName.name,
-                                dep.version ?: "*"
-                            )
-                        )
-                    })
-                } else {
-                    // todo: [WARN] cannot find $key in yarn.lock
-                }
-
-                return src
-            }
-
-            srcDependenciesList.forEach { src ->
-                resolveRecursively(src)
-            }
+            exec.workingDir = npmProjectLayout.nodeWorkDir
         }
     }
 
     @Suppress("EXPOSED_PARAMETER_TYPE")
     override fun resolveProject(npmPackage: NpmResolver.NpmPackage) {
         val project = npmPackage.project
-        if (!project.yarn.useWorkspaces) {
+        if (!YarnPlugin[project].checkUseWorkspace()) {
             yarnExec(project, NpmApi.resolveOperationDescription("yarn for ${project.path}"))
-            yarnLockReadTransitiveDependencies(NpmProject[project].nodeWorkDir, npmPackage.npmDependencies)
         }
     }
 
     @Suppress("EXPOSED_PARAMETER_TYPE")
-    override fun hookRootPackage(
-        rootProject: Project,
-        rootPackageJson: PackageJson,
-        allWorkspaces: Collection<NpmResolver.NpmPackage>
-    ): Boolean {
-        if (rootProject.yarn.useWorkspaces) {
+    override fun hookRootPackage(rootProject: Project, rootPackageJson: PackageJson, allWorkspaces: Collection<NpmResolver.NpmPackage>) {
+        if (YarnPlugin[rootProject].checkUseWorkspace()) {
             rootPackageJson.private = true
             rootPackageJson.workspaces = allWorkspaces
                 .filter { it.project != rootProject }
-                .map { it.project.projectDir.relativeTo(rootProject.rootDir).path }
-
-            return true
+                .map { it.project.rootDir.relativeTo(rootProject.rootDir).path }
         }
-
-        return false
     }
 
-    override fun getHoistGradleNodeModules(project: Project): Boolean =
-        project.rootProject.yarn.useWorkspaces
-
     @Suppress("EXPOSED_PARAMETER_TYPE")
     override fun resolveRootProject(
         rootProject: Project,
@@ -128,9 +63,8 @@ object Yarn : NpmApi {
     ) {
         check(rootProject == rootProject.rootProject)
 
-        if (rootProject.yarn.useWorkspaces) {
+        if (YarnPlugin[rootProject].checkUseWorkspace()) {
             yarnExec(rootProject, NpmApi.resolveOperationDescription("yarn"))
-            yarnLockReadTransitiveDependencies(NpmProject[rootProject].nodeWorkDir, subprojects.flatMap { it.npmDependencies })
         } else {
             if (subprojects.any { it.project != rootProject }) {
                 // todo: proofread message
@@ -139,15 +73,11 @@ object Yarn : NpmApi {
                             "It is recommended to enable yarn workspaces to store common NPM dependencies in root project. " +
                             "To enable it add this to your root project: \n" +
                             "nodeJs { manageNodeModules = true } \n" +
+                            "yarn { useWorkspaces = true } \n" +
                             "Note: with `manageNodeModules` enabled, your `node_modules` and `package.json` files will be managed by " +
                             "Gradle, will be overridden during build and should be ignored in VCS."
                 )
             }
         }
     }
-
-    override fun cleanProject(project: Project) {
-        super.cleanProject(project)
-        project.packageJsonHashFile.delete()
-    }
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnExtension.kt

@@ -1,46 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.gradle.targets.js.yarn
-
-import org.gradle.api.GradleException
-import org.gradle.api.Project
-import org.jetbrains.kotlin.gradle.logging.kotlinInfo
-
-open class YarnExtension(project: Project) {
-    private val gradleHome = project.gradle.gradleUserHomeDir.also {
-        project.logger.kotlinInfo("Storing cached files in $it")
-    }
-
-    var installationDir = gradleHome.resolve("yarn")
-
-    var distBaseUrl = "https://github.com/yarnpkg/yarn/releases/download"
-    var version = "1.15.2"
-
-    var command = "yarn"
-
-    var download = true
-
-    internal fun buildEnv() = YarnEnv(
-        downloadUrl = "$distBaseUrl/v$version/yarn-v$version.tar.gz",
-        home = installationDir.resolve("yarn-v$version")
-    )
-
-    companion object {
-        const val YARN: String = "yarn"
-
-        operator fun get(project: Project): YarnExtension {
-            val extension = project.extensions.findByType(YarnExtension::class.java)
-            if (extension != null)
-                return extension
-
-            val parentProject = project.parent
-            if (parentProject != null)
-                return get(parentProject)
-
-            throw GradleException("YarnExtension is not installed")
-        }
-    }
-}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnPlugin.kt

@@ -15,16 +15,14 @@ open class YarnPlugin : Plugin<Project> {
             "YarnPlugin can be applied only to root project"
         }
 
-        val nodeJs = NodeJsPlugin.apply(this)
-
         this.extensions.create(YarnRootExtension.YARN, YarnRootExtension::class.java, this)
         tasks.create(YarnSetupTask.NAME, YarnSetupTask::class.java) {
-            it.dependsOn(nodeJs.nodeJsSetupTask)
+            it.dependsOn(NodeJsPlugin[this].nodeJsSetupTask)
         }
     }
 
     companion object {
-        fun apply(project: Project): YarnRootExtension {
+        operator fun get(project: Project): YarnRootExtension {
             val rootProject = project.rootProject
             rootProject.plugins.apply(YarnPlugin::class.java)
             return rootProject.extensions.getByName(YarnRootExtension.YARN) as YarnRootExtension

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnRootExtension.kt

@@ -5,10 +5,10 @@
 
 package org.jetbrains.kotlin.gradle.targets.js.yarn
 
+import org.gradle.api.GradleException
 import org.gradle.api.Project
 import org.jetbrains.kotlin.gradle.logging.kotlinInfo
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin
-import org.jetbrains.kotlin.gradle.targets.js.nodejs.nodeJs
 
 open class YarnRootExtension(val project: Project) {
     private val gradleHome = project.gradle.gradleUserHomeDir.also {
@@ -23,36 +23,43 @@ open class YarnRootExtension(val project: Project) {
     val yarnSetupTask: YarnSetupTask
         get() = project.tasks.getByName(YarnSetupTask.NAME) as YarnSetupTask
 
-    var disableWorkspaces: Boolean = false
+    var useWorkspaces: Boolean = false
 
-    val useWorkspaces: Boolean
-        get() = project.nodeJs.root.manageNodeModules && !disableWorkspaces
+    internal fun checkUseWorkspace(): Boolean = if (useWorkspaces) {
+        check(NodeJsPlugin[project].manageNodeModules) {
+            "Yarn workspace can be used only with `manageNodeModules`: (add `nodeJs { manageNodeModules = true }` to your root project)"
+        }
+
+        true
+    } else false
 
     internal fun executeSetup() {
-        NodeJsPlugin.apply(project).executeSetup()
+        NodeJsPlugin[project].executeSetup()
 
-        val env = environment
+        val env = buildEnv()
         if (!env.home.isDirectory) {
             yarnSetupTask.setup()
         }
     }
 
-    internal val environment
-        get() = YarnEnv(
-            downloadUrl = "$downloadBaseUrl/v$version/yarn-v$version.tar.gz",
-            home = installationDir.resolve("yarn-v$version")
-        )
+    internal fun buildEnv() = YarnEnv(
+        downloadUrl = "$downloadBaseUrl/v$version/yarn-v$version.tar.gz",
+        home = installationDir.resolve("yarn-v$version")
+    )
 
     companion object {
         const val YARN: String = "yarn"
 
         operator fun get(project: Project): YarnRootExtension {
-            val rootProject = project.rootProject
-            rootProject.plugins.apply(YarnPlugin::class.java)
-            return rootProject.extensions.getByName(YARN) as YarnRootExtension
+            val extension = project.extensions.findByType(YarnRootExtension::class.java)
+            if (extension != null)
+                return extension
+
+            val parentProject = project.parent
+            if (parentProject != null)
+                return get(parentProject)
+
+            throw GradleException("YarnRootExtension is not installed")
         }
     }
 }
-
-val Project.yarn: YarnRootExtension
-    get() = YarnRootExtension[this]

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnSetupTask.kt

@@ -14,8 +14,8 @@ import java.io.File
 import javax.xml.ws.Action
 
 open class YarnSetupTask : DefaultTask() {
-    private val settings = project.yarn
-    private val env by lazy { settings.environment }
+    private val settings = YarnRootExtension[project]
+    private val env by lazy { settings.buildEnv() }
 
     @Suppress("MemberVisibilityCanBePrivate")
     val downloadUrl