On OOM in Kotlin daemon also show message how to increase heap size

^KT-58320 Fixed

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinDaemonIT.kt

@@ -114,10 +114,8 @@ class KotlinDaemonIT : KGPDaemonsBaseTest() {
             )
 
             buildAndFail("assemble") {
-                // 'assertOutputContains' is not suitable here as test produces OOM which TC will false-positively treat as
+                assertOutputContains("Not enough memory to run compilation. Try to increase it via 'gradle.properties':")
-                // whole configuration error. So we should not output test logs into build logs.
+                assertOutputContains("kotlin.daemon.jvmargs=-Xmx<size>")
-                assert(output.contains("Not enough memory to run compilation. Try to increase it via 'gradle.properties':"))
-                assert(output.contains("kotlin.daemon.jvmargs=-Xmx<size>"))
             }
         }
     }

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/compilerRunner/GradleKotlinCompilerWork.kt

@@ -17,9 +17,10 @@ import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskExecutionResults
 import org.jetbrains.kotlin.gradle.plugin.internal.state.TaskLoggers
 import org.jetbrains.kotlin.build.report.statistics.StatTag
 import org.jetbrains.kotlin.gradle.report.*
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompilerExecutionStrategy
+import org.jetbrains.kotlin.gradle.tasks.*
+import org.jetbrains.kotlin.gradle.tasks.OOMErrorException
 import org.jetbrains.kotlin.gradle.tasks.cleanOutputsAndLocalState
-import org.jetbrains.kotlin.gradle.tasks.throwExceptionIfCompilationFailed
+import org.jetbrains.kotlin.gradle.tasks.kotlinDaemonOOMHelperMessage
 import org.jetbrains.kotlin.gradle.utils.stackTraceAsString
 import org.jetbrains.kotlin.incremental.ChangedFiles
 import org.jetbrains.kotlin.incremental.ClasspathChanges
@@ -246,7 +247,11 @@ internal class GradleKotlinCompilerWork @Inject constructor(
             exitCodeFromProcessExitCode(log, res.get())
         } catch (e: Throwable) {
             bufferingMessageCollector.flush(messageCollector)
-            throw e
+            if (e is OutOfMemoryError || e.hasOOMCause()) {
+                throw OOMErrorException(kotlinDaemonOOMHelperMessage)
+            } else {
+                throw e
+            }
         } finally {
             val memoryUsageAfterBuild = daemon.getUsedMemory(withGC = false).takeIf { it.isGood }?.get()
 

libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/tasks/tasksUtils.kt

@@ -23,13 +23,10 @@ fun throwExceptionIfCompilationFailed(
         ExitCode.INTERNAL_ERROR -> throw FailedCompilationException("Internal compiler error. See log for more details")
         ExitCode.SCRIPT_EXECUTION_ERROR -> throw FailedCompilationException("Script execution error. See log for more details")
         ExitCode.OOM_ERROR -> {
-            var exceptionMessage = "Not enough memory to run compilation."
+            val exceptionMessage = when (executionStrategy) {
-            when (executionStrategy) {
+                KotlinCompilerExecutionStrategy.DAEMON -> kotlinDaemonOOMHelperMessage
-                KotlinCompilerExecutionStrategy.DAEMON ->
+                KotlinCompilerExecutionStrategy.IN_PROCESS -> kotlinInProcessOOMHelperMessage
-                    exceptionMessage += " Try to increase it via 'gradle.properties':\nkotlin.daemon.jvmargs=-Xmx<size>"
+                KotlinCompilerExecutionStrategy.OUT_OF_PROCESS -> kotlinOutOfProcessOOMHelperMessage
-                KotlinCompilerExecutionStrategy.IN_PROCESS ->
-                    exceptionMessage += " Try to increase it via 'gradle.properties':\norg.gradle.jvmargs=-Xmx<size>"
-                KotlinCompilerExecutionStrategy.OUT_OF_PROCESS -> Unit
             }
             throw OOMErrorException(exceptionMessage)
         }
@@ -38,6 +35,19 @@ fun throwExceptionIfCompilationFailed(
     }
 }
 
+internal const val kotlinDaemonOOMHelperMessage = "Not enough memory to run compilation. " +
+        "Try to increase it via 'gradle.properties':\nkotlin.daemon.jvmargs=-Xmx<size>"
+
+internal const val kotlinInProcessOOMHelperMessage = "Not enough memory to run compilation. " +
+        " Try to increase it via 'gradle.properties':\norg.gradle.jvmargs=-Xmx<size>"
+
+internal const val kotlinOutOfProcessOOMHelperMessage = "Not enough memory to run compilation."
+
+internal fun Throwable.hasOOMCause(): Boolean = when (cause) {
+    is OutOfMemoryError -> true
+    else -> cause?.hasOOMCause() ?: false
+}
+
 /** Exception thrown when [ExitCode] != [ExitCode.OK]. */
 internal open class FailedCompilationException(message: String) : RuntimeException(message)