Show user-friendly message on OOM compilation error

This should provide a hint to user how to mitigate out-of-memory error
on compilation inside Kotlin daemon or in-process.

^KT-47522 Fixed

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/ExitCode.java

@@ -20,7 +20,9 @@ public enum ExitCode {
     OK(0),
     COMPILATION_ERROR(1),
     INTERNAL_ERROR(2),
-    SCRIPT_EXECUTION_ERROR(3);
+    SCRIPT_EXECUTION_ERROR(3),
+    OOM_ERROR(137), // 137 is commonly used to indicate OOM error
+    ;
 
     private final int code;
 

compiler/cli/src/org/jetbrains/kotlin/cli/common/CLICompiler.kt

@@ -20,8 +20,7 @@ import com.intellij.openapi.Disposable
 import com.intellij.openapi.util.Disposer
 import org.jetbrains.kotlin.analyzer.CompilationErrorException
 import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY
-import org.jetbrains.kotlin.cli.common.ExitCode.COMPILATION_ERROR
-import org.jetbrains.kotlin.cli.common.ExitCode.INTERNAL_ERROR
+import org.jetbrains.kotlin.cli.common.ExitCode.*
 import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
 import org.jetbrains.kotlin.cli.common.environment.setIdeaIoUseFallback
 import org.jetbrains.kotlin.cli.common.messages.*
@@ -122,7 +121,7 @@ abstract class CLICompiler<A : CommonCompilerArguments> : CLITool<A>() {
             return COMPILATION_ERROR
         } catch (t: Throwable) {
             MessageCollectorUtil.reportException(collector, t)
-            return INTERNAL_ERROR
+            return if (t is OutOfMemoryError) OOM_ERROR else INTERNAL_ERROR
         } finally {
             collector.flush()
         }

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinDaemonIT.kt

@@ -100,6 +100,25 @@ class KotlinDaemonIT : KGPDaemonsBaseTest() {
         }
     }
 
+    @DisplayName("On Kotlin daemon OOM helpful message is displayed")
+    @GradleTest
+    fun displaySpecialMessageOnOOM(gradleVersion: GradleVersion) {
+        project(
+            "kotlinProject",
+            gradleVersion,
+            buildOptions = defaultBuildOptions.copy(logLevel = LogLevel.INFO)
+        ) {
+            gradleProperties.append(
+                "\nkotlin.daemon.jvmargs=-Xmx16m"
+            )
+
+            buildAndFail("assemble") {
+                assertOutputContains("Not enough memory to run compilation. Try to increase it via 'gradle.properties':")
+                assertOutputContains("kotlin.daemon.jvmargs=-Xmx<size>")
+            }
+        }
+    }
+
     private fun BuildResult.assertGradleClasspathNotLeaked() {
         assertOutputContains("Kotlin compiler classpath:")
         val daemonClasspath = output

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt

@@ -127,12 +127,12 @@ internal class GradleKotlinCompilerWork @Inject constructor(
     override fun run() {
         try {
             val messageCollector = GradlePrintingMessageCollector(log, allWarningsAsErrors)
-            val exitCode = compileWithDaemonOrFallbackImpl(messageCollector)
+            val (exitCode, executionStrategy) = compileWithDaemonOrFallbackImpl(messageCollector)
             if (incrementalCompilationEnvironment?.disableMultiModuleIC == true) {
                 incrementalCompilationEnvironment.multiModuleICSettings.buildHistoryFile.delete()
             }
 
-            throwGradleExceptionIfError(exitCode)
+            throwGradleExceptionIfError(exitCode, executionStrategy)
         } finally {
             val properties = ArrayList<TaskExecutionProperties>()
             COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value.toBooleanLenient()?.let {
@@ -151,7 +151,7 @@ internal class GradleKotlinCompilerWork @Inject constructor(
         }
     }
 
-    private fun compileWithDaemonOrFallbackImpl(messageCollector: MessageCollector): ExitCode {
+    private fun compileWithDaemonOrFallbackImpl(messageCollector: MessageCollector): Pair<ExitCode, KotlinCompilerExecutionStrategy> {
         with(log) {
             kotlinDebug { "Kotlin compiler class: ${compilerClassName}" }
             kotlinDebug { "Kotlin compiler classpath: ${compilerFullClasspath.joinToString { it.canonicalPath }}" }
@@ -162,7 +162,7 @@ internal class GradleKotlinCompilerWork @Inject constructor(
             val daemonExitCode = compileWithDaemon(messageCollector)
 
             if (daemonExitCode != null) {
-                return daemonExitCode
+                return daemonExitCode to KotlinCompilerExecutionStrategy.DAEMON
             } else {
                 log.warn("Could not connect to kotlin daemon. Using fallback strategy.")
             }
@@ -170,9 +170,9 @@ internal class GradleKotlinCompilerWork @Inject constructor(
 
         val isGradleDaemonUsed = System.getProperty("org.gradle.daemon")?.let(String::toBoolean)
         return if (compilerExecutionStrategy == KotlinCompilerExecutionStrategy.IN_PROCESS || isGradleDaemonUsed == false) {
-            compileInProcess(messageCollector)
+            compileInProcess(messageCollector) to KotlinCompilerExecutionStrategy.IN_PROCESS
         } else {
-            compileOutOfProcess()
+            compileOutOfProcess() to KotlinCompilerExecutionStrategy.OUT_OF_PROCESS
         }
     }
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinJsDce.kt

@@ -132,7 +132,7 @@ abstract class KotlinJsDce : AbstractKotlinCompileTool<K2JSDceArguments>(), Kotl
             buildDir,
             jvmArgs
         )
-        throwGradleExceptionIfError(exitCode)
+        throwGradleExceptionIfError(exitCode, KotlinCompilerExecutionStrategy.OUT_OF_PROCESS)
     }
 
     private fun isDceCandidate(file: File): Boolean {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/tasksUtils.kt

@@ -14,13 +14,26 @@ import org.jetbrains.kotlin.incremental.cleanDirectoryContents
 import org.jetbrains.kotlin.incremental.forceDeleteRecursively
 import java.io.File
 
-fun throwGradleExceptionIfError(exitCode: ExitCode) {
+fun throwGradleExceptionIfError(
+    exitCode: ExitCode,
+    executionStrategy: KotlinCompilerExecutionStrategy
+) {
     when (exitCode) {
         ExitCode.COMPILATION_ERROR -> throw GradleException("Compilation error. See log for more details")
         ExitCode.INTERNAL_ERROR -> throw GradleException("Internal compiler error. See log for more details")
         ExitCode.SCRIPT_EXECUTION_ERROR -> throw GradleException("Script execution error. See log for more details")
-        ExitCode.OK -> {
+        ExitCode.OOM_ERROR -> {
+            var exceptionMessage = "Not enough memory to run compilation."
+            when (executionStrategy) {
+                KotlinCompilerExecutionStrategy.DAEMON ->
+                    exceptionMessage += " Try to increase it via 'gradle.properties':\nkotlin.daemon.jvmargs=-Xmx<size>"
+                KotlinCompilerExecutionStrategy.IN_PROCESS ->
+                    exceptionMessage += " Try to increase it via 'gradle.properties':\norg.gradle.jvmargs=-Xmx<size>"
+                KotlinCompilerExecutionStrategy.OUT_OF_PROCESS -> Unit
+            }
+            throw GradleException(exceptionMessage)
         }
+        ExitCode.OK -> Unit
         else -> throw IllegalStateException("Unexpected exit code: $exitCode")
     }
 }