Add flag to enable intra-project parallel tasks

To enable parallel tasks execution within a project,
add 'kotlin.parallel.tasks.in.project=true'
to gradle.properties or local.properties file .

    #KT-28155 fixed

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BaseGradleIT.kt

@@ -178,7 +178,8 @@ abstract class BaseGradleIT {
         val kotlinDaemonDebugPort: Int? = null,
         val usePreciseJavaTracking: Boolean? = null,
         val withBuildCache: Boolean = false,
-        val kaptOptions: KaptOptions? = null
+        val kaptOptions: KaptOptions? = null,
+        val parallelTasksInProject: Boolean? = null
     )
 
     data class KaptOptions(val verbose: Boolean, val useWorkers: Boolean)
@@ -446,6 +447,18 @@ abstract class BaseGradleIT {
         assertTasksUpToDate(tasks.toList())
     }
 
+    fun CompiledProject.assertTasksSubmittedWork(vararg tasks: String) {
+        for (task in tasks) {
+            assertContains("Starting Kotlin compiler work from task '$task'")
+        }
+    }
+
+    fun CompiledProject.assertTasksDidNotSubmitWork(vararg tasks: String) {
+        for (task in tasks) {
+            assertNotContains("Starting Kotlin compiler work from task '$task'")
+        }
+    }
+
     fun CompiledProject.getOutputForTask(taskName: String): String {
         val taskOutputRegex = ("\\[LIFECYCLE] \\[class org\\.gradle(?:\\.internal\\.buildevents)?\\.TaskExecutionLogger] :$taskName" +
                 "([\\s\\S]+?)" +
@@ -567,6 +580,10 @@ abstract class BaseGradleIT {
                 add("-Pkapt.use.worker.api=${kaptOptions.useWorkers}")
             }
 
+            options.parallelTasksInProject?.let {
+                add("-Pkotlin.parallel.tasks.in.project=$it")
+            }
+
             // Workaround: override a console type set in the user machine gradle.properties (since Gradle 4.3):
             add("--console=plain")
 

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/NewMultiplatformIT.kt

@@ -249,6 +249,35 @@ class NewMultiplatformIT : BaseGradleIT() {
         }
     }
 
+    @Test
+    fun testParallelTasks() {
+        parallelTasksImpl(isParallel = true)
+    }
+
+    @Test
+    fun testNoParallelTasks() {
+        parallelTasksImpl(isParallel = false)
+    }
+
+    private fun parallelTasksImpl(isParallel: Boolean) = with(Project("new-mpp-parallel", gradleVersion)) {
+        val options = defaultBuildOptions().copy(parallelTasksInProject = isParallel)
+        build("assemble", options = options) {
+            assertSuccessful()
+            val tasks = arrayOf(":compileKotlinMetadata", ":compileKotlinJvm", ":compileKotlinJs")
+            if (isParallel) {
+                assertTasksSubmittedWork(*tasks)
+            } else {
+                assertTasksDidNotSubmitWork(*tasks)
+            }
+            val expectedKotlinOutputFiles = listOf(
+                kotlinClassesDir(sourceSet = "metadata/main") + "common/A.kotlin_metadata",
+                kotlinClassesDir(sourceSet = "jvm/main") + "common/A.class",
+                kotlinClassesDir(sourceSet = "js/main") + "new-mpp-parallel.js"
+            )
+            expectedKotlinOutputFiles.forEach { assertFileExists(it) }
+        }
+    }
+
     @Test
     fun testLibWithTests() = with(Project("new-mpp-lib-with-tests", gradleVersion)) {
         build("check") {

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-parallel/build.gradle

@@ -0,0 +1,37 @@
+buildscript {
+	repositories {
+        mavenLocal()
+        jcenter()
+	}
+	dependencies {
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+	}
+}
+
+apply plugin: 'kotlin-multiplatform'
+
+repositories {
+    mavenLocal()
+    jcenter()
+}
+
+kotlin {
+    sourceSets {
+        commonMain {}
+        jvmMain {
+            dependencies {
+                implementation 'org.jetbrains.kotlin:kotlin-stdlib'
+            }
+        }
+        jsMain {
+            dependencies {
+                implementation 'org.jetbrains.kotlin:kotlin-stdlib-js'
+            }
+        }
+    }
+
+    targets {
+        fromPreset(presets.jvm, 'jvm')
+        fromPreset(presets.js, 'js')
+    }
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-parallel/src/commonMain/kotlin/A.kt

@@ -0,0 +1,9 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package common
+
+class A(val x: Int)
+

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerRunnerWithWorkers.kt

@@ -5,16 +5,19 @@
 
 package org.jetbrains.kotlin.compilerRunner
 
-import org.gradle.api.Project
+import org.gradle.api.Task
 import org.gradle.workers.ForkMode
 import org.gradle.workers.IsolationMode
 import org.gradle.workers.WorkerExecutor
+import org.jetbrains.kotlin.gradle.plugin.kotlinDebug
 
 internal class GradleCompilerRunnerWithWorkers(
-    project: Project,
+    task: Task,
     private val workersExecutor: WorkerExecutor
-) : GradleCompilerRunner(project) {
+) : GradleCompilerRunner(task) {
     override fun runCompilerAsync(workArgs: GradleKotlinCompilerWorkArguments) {
+        project.logger.kotlinDebug { "Starting Kotlin compiler work from task '${task.path}'" }
+        // todo: write tests with Workers enabled;
         workersExecutor.submit(GradleKotlinCompilerWork::class.java) { config ->
             config.isolationMode = IsolationMode.NONE
             config.forkMode = ForkMode.NEVER

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerRunner.kt

@@ -17,6 +17,7 @@
 package org.jetbrains.kotlin.compilerRunner
 
 import org.gradle.api.Project
+import org.gradle.api.Task
 import org.gradle.api.invocation.Gradle
 import org.gradle.api.plugins.JavaPluginConvention
 import org.gradle.jvm.tasks.Jar
@@ -50,7 +51,10 @@ const val COULD_NOT_CONNECT_TO_DAEMON_MESSAGE = "Could not connect to Kotlin com
 internal fun kotlinCompilerExecutionStrategy(): String =
     System.getProperty(KOTLIN_COMPILER_EXECUTION_STRATEGY_PROPERTY) ?: DAEMON_EXECUTION_STRATEGY
 
-internal open class GradleCompilerRunner(protected val project: Project) {
+internal open class GradleCompilerRunner(protected val task: Task) {
+    protected val project: Project
+        get() = task.project
+
     /**
      * Compiler might be executed asynchronuosly. Do not do anything requiring end of compilation after this function is called.
      * @see [GradleKotlinCompilerWork]

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/Kapt3KotlinGradleSubplugin.kt

@@ -419,7 +419,7 @@ class Kapt3KotlinGradleSubplugin : KotlinGradleSubplugin<KotlinCompile> {
         kaptTask.destinationDir = getKaptIncrementalDataDir()
         kaptTask.mapClasspath { kotlinCompile.classpath }
         kaptTask.generatedSourcesDir = sourcesOutputDir
-        mapKotlinTaskProperties(project, kaptTask)
+        PropertiesProvider(project).mapKotlinTaskProperties(kaptTask)
 
         kaptTask.kaptClasspathConfigurations = kaptClasspathConfigurations
         buildAndAddOptionsTo(kaptTask, kaptTask.pluginOptions, aptMode = "stubs")

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/KaptWithKotlincTask.kt

@@ -67,7 +67,7 @@ open class KaptWithKotlincTask : KaptTask(), CompilerArgumentAwareWithInput<K2JV
             throw GradleException("Could not find tools.jar in system classpath, which is required for kapt to work")
         }
 
-        val compilerRunner = GradleCompilerRunner(project)
+        val compilerRunner = GradleCompilerRunner(this)
         compilerRunner.runJvmCompilerAsync(
             sourcesToCompile = emptyList(),
             commonSources = emptyList(),

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinProperties.kt

@@ -23,21 +23,19 @@ import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 import java.util.*
 
-fun mapKotlinTaskProperties(project: Project, task: AbstractKotlinCompile<*>) {
-    PropertiesProvider(project).apply {
-        coroutines?.let { task.coroutinesFromGradleProperties = it }
-        useFallbackCompilerSearch?.let { task.useFallbackCompilerSearch = it }
+internal fun PropertiesProvider.mapKotlinTaskProperties(task: AbstractKotlinCompile<*>) {
+    coroutines?.let { task.coroutinesFromGradleProperties = it }
+    useFallbackCompilerSearch?.let { task.useFallbackCompilerSearch = it }
 
-        if (task is KotlinCompile) {
-            incrementalJvm?.let { task.incremental = it }
-            usePreciseJavaTracking?.let {
-                task.usePreciseJavaTracking = it
-            }
+    if (task is KotlinCompile) {
+        incrementalJvm?.let { task.incremental = it }
+        usePreciseJavaTracking?.let {
+            task.usePreciseJavaTracking = it
         }
+    }
 
-        if (task is Kotlin2JsCompile) {
-            incrementalJs?.let { task.incremental = it }
-        }
+    if (task is Kotlin2JsCompile) {
+        incrementalJs?.let { task.incremental = it }
     }
 }
 
@@ -71,6 +69,14 @@ internal class PropertiesProvider(private val project: Project) {
     val useFallbackCompilerSearch: Boolean?
         get() = booleanProperty("kotlin.useFallbackCompilerSearch")
 
+    /**
+     * Enables parallel tasks execution within a project with Workers API.
+     * Does not enable using actual worker proccesses
+     * (Kotlin Daemon can be shared which uses less memory)
+     */
+    val parallelTasksInProject: Boolean?
+        get() = booleanProperty("kotlin.parallel.tasks.in.project")
+
     private fun booleanProperty(propName: String): Boolean? =
         property(propName)?.toBoolean()
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/mpp/kotlinTargetPresets.kt

@@ -83,7 +83,7 @@ class KotlinMetadataTargetPreset(
         get() = KotlinPlatformType.common
 
     override fun buildCompilationProcessor(compilation: KotlinCommonCompilation): KotlinSourceSetProcessor<*> {
-        val tasksProvider = KotlinTasksProvider(compilation.target.targetName, useWorkersForCompilation = true)
+        val tasksProvider = KotlinTasksProvider(compilation.target.targetName)
         return KotlinCommonSourceSetProcessor(project, compilation, tasksProvider, kotlinPluginVersion)
     }
 
@@ -129,7 +129,7 @@ class KotlinJvmTargetPreset(
         get() = KotlinPlatformType.jvm
 
     override fun buildCompilationProcessor(compilation: KotlinJvmCompilation): KotlinSourceSetProcessor<*> {
-        val tasksProvider = KotlinTasksProvider(compilation.target.targetName, useWorkersForCompilation = true)
+        val tasksProvider = KotlinTasksProvider(compilation.target.targetName)
         return Kotlin2JvmSourceSetProcessor(project, tasksProvider, compilation, kotlinPluginVersion)
     }
 
@@ -158,7 +158,7 @@ class KotlinJsTargetPreset(
         get() = KotlinPlatformType.js
 
     override fun buildCompilationProcessor(compilation: KotlinJsCompilation): KotlinSourceSetProcessor<*> {
-        val tasksProvider = KotlinTasksProvider(compilation.target.targetName, useWorkersForCompilation = true)
+        val tasksProvider = KotlinTasksProvider(compilation.target.targetName)
         return Kotlin2JsSourceSetProcessor(project, tasksProvider, compilation, kotlinPluginVersion)
     }
 
@@ -180,7 +180,7 @@ class KotlinAndroidTargetPreset(
         }
 
         KotlinAndroidPlugin.applyToTarget(
-            project, result, AndroidTasksProvider(name, useWorkersForCompilation = true),
+            project, result, AndroidTasksProvider(name),
             kotlinPluginVersion
         )
 
@@ -207,8 +207,7 @@ class KotlinJvmWithJavaTargetPreset(
         }
 
         AbstractKotlinPlugin.configureTarget(target) { compilation ->
-            val tasksProvider = KotlinTasksProvider(name, useWorkersForCompilation = true)
-            Kotlin2JvmSourceSetProcessor(project, tasksProvider, compilation, kotlinPluginVersion)
+            Kotlin2JvmSourceSetProcessor(project, KotlinTasksProvider(name), compilation, kotlinPluginVersion)
         }
 
         target.compilations.all { compilation ->

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt

@@ -250,7 +250,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments>() : AbstractKo
     }
 
     internal open fun compilerRunner(): GradleCompilerRunner =
-        GradleCompilerRunner(project)
+        GradleCompilerRunner(this)
 
     override fun compile() {
         assert(false, { "unexpected call to compile()" })
@@ -456,21 +456,21 @@ open class KotlinCompile : AbstractKotlinCompile<K2JVMCompilerArguments>(), Kotl
 internal open class KotlinCompileWithWorkers @Inject constructor(
     @Suppress("UnstableApiUsage") private val workerExecutor: WorkerExecutor
 ) : KotlinCompile() {
-    override fun compilerRunner() = GradleCompilerRunnerWithWorkers(project, workerExecutor)
+    override fun compilerRunner() = GradleCompilerRunnerWithWorkers(this, workerExecutor)
 }
 
 @CacheableTask
 internal open class Kotlin2JsCompileWithWorkers @Inject constructor(
     @Suppress("UnstableApiUsage") private val workerExecutor: WorkerExecutor
 ) : Kotlin2JsCompile() {
-    override fun compilerRunner() = GradleCompilerRunnerWithWorkers(project, workerExecutor)
+    override fun compilerRunner() = GradleCompilerRunnerWithWorkers(this, workerExecutor)
 }
 
 @CacheableTask
 internal open class KotlinCompileCommonWithWorkers @Inject constructor(
     @Suppress("UnstableApiUsage") private val workerExecutor: WorkerExecutor
 ) : KotlinCompileCommon() {
-    override fun compilerRunner() = GradleCompilerRunnerWithWorkers(project, workerExecutor)
+    override fun compilerRunner() = GradleCompilerRunnerWithWorkers(this, workerExecutor)
 }
 
 @CacheableTask

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/TasksProvider.kt

@@ -22,44 +22,45 @@ import org.jetbrains.kotlin.gradle.plugin.*
 import org.jetbrains.kotlin.gradle.plugin.mpp.defaultSourceSetName
 import org.jetbrains.kotlin.gradle.plugin.sources.applyLanguageSettingsToKotlinTask
 
-internal open class KotlinTasksProvider(
-    val targetName: String,
-    private val useWorkersForCompilation: Boolean = false
-) {
+internal open class KotlinTasksProvider(val targetName: String) {
     open fun createKotlinJVMTask(
         project: Project,
         name: String,
         compilation: KotlinCompilation
     ): KotlinCompile {
-        val taskClass = if (useWorkersForCompilation) KotlinCompileWithWorkers::class.java else KotlinCompile::class.java
+        val properties = PropertiesProvider(project)
+        val taskClass = taskOrWorkersTask<KotlinCompile, KotlinCompileWithWorkers>(properties)
         return project.tasks.create(name, taskClass).apply {
-            configure(this, project, compilation)
+            configure(this, project, properties, compilation)
         }
     }
 
     fun createKotlinJSTask(project: Project, name: String, compilation: KotlinCompilation): Kotlin2JsCompile {
-        val taskClass = if (useWorkersForCompilation) Kotlin2JsCompileWithWorkers::class.java else Kotlin2JsCompile::class.java
+        val properties = PropertiesProvider(project)
+        val taskClass = taskOrWorkersTask<Kotlin2JsCompile, Kotlin2JsCompileWithWorkers>(properties)
         return project.tasks.create(name, taskClass).apply {
-            configure(this, project, compilation)
+            configure(this, project, properties, compilation)
         }
     }
 
 
     fun createKotlinCommonTask(project: Project, name: String, compilation: KotlinCompilation): KotlinCompileCommon {
-        val taskClass = if (useWorkersForCompilation) KotlinCompileCommonWithWorkers::class.java else KotlinCompileCommon::class.java
+        val properties = PropertiesProvider(project)
+        val taskClass = taskOrWorkersTask<KotlinCompileCommon, KotlinCompileCommonWithWorkers>(properties)
         return project.tasks.create(name, taskClass).apply {
-            configure(this, project, compilation)
+            configure(this, project, properties, compilation)
         }
     }
 
     open fun configure(
         kotlinTask: AbstractKotlinCompile<*>,
         project: Project,
+        propertiesProvider: PropertiesProvider,
         compilation: KotlinCompilation
     ) {
         kotlinTask.sourceSetName = compilation.name
         kotlinTask.friendTaskName = taskToFriendTaskMapper[kotlinTask]
-        mapKotlinTaskProperties(project, kotlinTask)
+        propertiesProvider.mapKotlinTaskProperties(kotlinTask)
 
         project.whenEvaluated {
             val languageSettings = project.kotlinExtension.sourceSets.findByName(compilation.defaultSourceSetName)?.languageSettings
@@ -72,17 +73,22 @@ internal open class KotlinTasksProvider(
 
     protected open val taskToFriendTaskMapper: TaskToFriendTaskMapper =
         RegexTaskToFriendTaskMapper.Default(targetName)
+
+    private inline fun <reified Task, reified WorkersTask : Task> taskOrWorkersTask(properties: PropertiesProvider): Class<out Task> =
+        if (properties.parallelTasksInProject != true) Task::class.java else WorkersTask::class.java
 }
 
-internal class AndroidTasksProvider(
-    targetName: String,
-    useWorkersForCompilation: Boolean = false
-) : KotlinTasksProvider(targetName, useWorkersForCompilation = useWorkersForCompilation) {
+internal class AndroidTasksProvider(targetName: String) : KotlinTasksProvider(targetName) {
     override val taskToFriendTaskMapper: TaskToFriendTaskMapper =
         RegexTaskToFriendTaskMapper.Android(targetName)
 
-    override fun configure(kotlinTask: AbstractKotlinCompile<*>, project: Project, compilation: KotlinCompilation) {
-        super.configure(kotlinTask, project, compilation)
+    override fun configure(
+        kotlinTask: AbstractKotlinCompile<*>,
+        project: Project,
+        propertiesProvider: PropertiesProvider,
+        compilation: KotlinCompilation
+    ) {
+        super.configure(kotlinTask, project, propertiesProvider, compilation)
         kotlinTask.useModuleDetection = true
     }
 }