From ea49a9a2921eb5113d88319306cb3cbeb0268044 Mon Sep 17 00:00:00 2001 From: Yahor Berdnikau Date: Thu, 9 Nov 2023 17:36:34 +0100 Subject: [PATCH] [Gradle] Limit max heap size for Kotlin daemon in integration tests This change should further reduce memory pressure on running these tests on CI agents. Kapt4 tests require more Kotlin daemon memory than default 1g. It is a known issue and eventually should be addressed by the compiler team. --- .../kotlin/gradle/ExecutionStrategyIT.kt | 3 +++ .../org/jetbrains/kotlin/gradle/Kapt3IT.kt | 25 +++++++++++++------ .../jetbrains/kotlin/gradle/KotlinDaemonIT.kt | 5 ++-- .../kotlin/gradle/KotlinDaemonJvmArgsTest.kt | 18 +++++++------ .../kotlin/gradle/testbase/testDsl.kt | 19 ++++++++++++++ 5 files changed, 52 insertions(+), 18 deletions(-) diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ExecutionStrategyIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ExecutionStrategyIT.kt index d736fb02e28..92d7e59bc0b 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ExecutionStrategyIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ExecutionStrategyIT.kt @@ -101,6 +101,7 @@ abstract class ExecutionStrategyIT : KGPDaemonsBaseTest() { projectName = "kotlinBuiltins", gradleVersion = gradleVersion, addHeapDumpOptions = false, + enableKotlinDaemonMemoryLimitInMb = null, buildOptions = defaultBuildOptions.copy( useDaemonFallbackStrategy = false, compilerExecutionStrategy = KotlinCompilerExecutionStrategy.DAEMON, @@ -123,6 +124,7 @@ abstract class ExecutionStrategyIT : KGPDaemonsBaseTest() { project( projectName = "kotlinBuiltins", gradleVersion = gradleVersion, + enableKotlinDaemonMemoryLimitInMb = null, addHeapDumpOptions = false ) { setupProject(this) @@ -203,6 +205,7 @@ abstract class ExecutionStrategyIT : KGPDaemonsBaseTest() { projectName = "kotlinBuiltins", gradleVersion = gradleVersion, addHeapDumpOptions = addHeapDumpOptions, + enableKotlinDaemonMemoryLimitInMb = if (shouldConfigureStrategyViaGradleProperty) null else 1024, buildOptions = defaultBuildOptions.copy( useDaemonFallbackStrategy = testFallbackStrategy, compilerExecutionStrategy = if (shouldConfigureStrategyViaGradleProperty) { diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/Kapt3IT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/Kapt3IT.kt index d443de2604f..1085f767037 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/Kapt3IT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/Kapt3IT.kt @@ -152,7 +152,7 @@ open class Kapt3IT : Kapt3BaseIT() { @DisplayName("Kapt is skipped when no annotation processors are added") @GradleTest fun testKaptSkipped(gradleVersion: GradleVersion) { - project("kaptSkipped".withPrefix, gradleVersion) { + project("kaptSkipped".withPrefix, gradleVersion, enableKotlinDaemonMemoryLimitInMb = 2048) { build("build") { assertTasksSkipped(":kaptGenerateStubsKotlin", ":kaptKotlin") } @@ -162,7 +162,7 @@ open class Kapt3IT : Kapt3BaseIT() { @DisplayName("Kapt is not skipped when all annotation processors are declared as indirect dependencies") @GradleTest fun testKaptNotSkippedWithIndirectDependencies(gradleVersion: GradleVersion) { - project("indirectDependencies".withPrefix, gradleVersion) { + project("indirectDependencies".withPrefix, gradleVersion, enableKotlinDaemonMemoryLimitInMb = 2048) { build("assemble") { assertTasksExecuted(":kaptGenerateStubsKotlin", ":kaptKotlin") } @@ -179,6 +179,7 @@ open class Kapt3IT : Kapt3BaseIT() { project( "simple".withPrefix, gradleVersion, + enableKotlinDaemonMemoryLimitInMb = 2048, ) { //language=Groovy buildGradle.appendText( @@ -208,7 +209,8 @@ open class Kapt3IT : Kapt3BaseIT() { ) { project( "simple".withPrefix, - gradleVersion + gradleVersion, + enableKotlinDaemonMemoryLimitInMb = 2048 ) { //language=Groovy buildGradle.appendText( @@ -337,7 +339,7 @@ open class Kapt3IT : Kapt3BaseIT() { @DisplayName("Kapt tasks is up-to-date on the second run") @GradleTest fun testSimple(gradleVersion: GradleVersion) { - project("simple".withPrefix, gradleVersion) { + project("simple".withPrefix, gradleVersion, enableKotlinDaemonMemoryLimitInMb = 2048) { build("build") { assertKaptSuccessful() @@ -551,6 +553,7 @@ open class Kapt3IT : Kapt3BaseIT() { project( "incrementalRebuild".withPrefix, gradleVersion, + enableKotlinDaemonMemoryLimitInMb = 2048, buildOptions = defaultBuildOptions.copy(incremental = true) ) { val generatedSrc = "build/generated/source/kapt/main" @@ -586,6 +589,7 @@ open class Kapt3IT : Kapt3BaseIT() { project( "simple".withPrefix, gradleVersion, + enableKotlinDaemonMemoryLimitInMb = 2048, buildOptions = defaultBuildOptions.copy(incremental = true) ) { val internalDummyKt = javaSourcesDir().resolve("foo/InternalDummy.kt") @@ -634,6 +638,7 @@ open class Kapt3IT : Kapt3BaseIT() { project( "simple".withPrefix, gradleVersion, + enableKotlinDaemonMemoryLimitInMb = 2048, buildOptions = defaultBuildOptions.copy(logLevel = LogLevel.DEBUG) ) { val arg = "-Xsuppress-version-warnings" @@ -658,7 +663,7 @@ open class Kapt3IT : Kapt3BaseIT() { @DisplayName("generates Kotlin code") @GradleTest fun testOutputKotlinCode(gradleVersion: GradleVersion) { - project("kaptOutputKotlinCode".withPrefix, gradleVersion) { + project("kaptOutputKotlinCode".withPrefix, gradleVersion, enableKotlinDaemonMemoryLimitInMb = 2048) { build("build") { assertKaptSuccessful() assertFileExists(projectPath.resolve("build/generated/source/kapt/main/example/TestClassCustomized.java")) @@ -672,7 +677,7 @@ open class Kapt3IT : Kapt3BaseIT() { @DisplayName("location mapping is working as expected") @GradleTest fun testLocationMapping(gradleVersion: GradleVersion) { - project("locationMapping".withPrefix, gradleVersion) { + project("locationMapping".withPrefix, gradleVersion, enableKotlinDaemonMemoryLimitInMb = 2048) { val regex = "((Test\\.java)|(test\\.kt)):(\\d+): error: GenError element".toRegex() fun BuildResult.getErrorMessages(): String = @@ -804,7 +809,7 @@ open class Kapt3IT : Kapt3BaseIT() { @DisplayName("up-to-date checks are working") @GradleTest fun testKaptAvoidance(gradleVersion: GradleVersion) { - project("kaptAvoidance".withPrefix, gradleVersion) { + project("kaptAvoidance".withPrefix, gradleVersion, enableKotlinDaemonMemoryLimitInMb = 2048) { subProject("app").buildGradle.modify { "$it\n\nkapt.includeCompileClasspath = true" @@ -1059,6 +1064,7 @@ open class Kapt3IT : Kapt3BaseIT() { project( "jpms-module".withPrefix, gradleVersion, + enableKotlinDaemonMemoryLimitInMb = 2048, ) { build("assemble") { assertTasksExecuted(":kaptKotlin", ":kaptGenerateStubsKotlin", ":compileKotlin", ":compileJava") @@ -1089,6 +1095,7 @@ open class Kapt3IT : Kapt3BaseIT() { project( "incrementalRebuild".withPrefix, gradleVersion, + enableKotlinDaemonMemoryLimitInMb = 2048, buildOptions = defaultBuildOptions.withConfigurationCache ) { build("assemble") @@ -1158,7 +1165,7 @@ open class Kapt3IT : Kapt3BaseIT() { @DisplayName("Generated sources attached to KotlinSourceSet are also used by generate stubs task") @GradleTest fun testGeneratedSourcesUsedInGenerateStubsTask(gradleVersion: GradleVersion) { - project("generatedSources".withPrefix, gradleVersion) { + project("generatedSources".withPrefix, gradleVersion, enableKotlinDaemonMemoryLimitInMb = 2048) { build("assemble") } } @@ -1191,6 +1198,7 @@ open class Kapt3IT : Kapt3BaseIT() { project( "simple".withPrefix, gradleVersion, + enableKotlinDaemonMemoryLimitInMb = 2048, buildOptions = defaultBuildOptions.copy(logLevel = LogLevel.DEBUG) ) { buildGradle.appendText( @@ -1379,6 +1387,7 @@ open class Kapt3IT : Kapt3BaseIT() { project( "simple".withPrefix, gradleVersion, + enableKotlinDaemonMemoryLimitInMb = 2048, buildOptions = defaultBuildOptions.copy(logLevel = LogLevel.DEBUG) ) { diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinDaemonIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinDaemonIT.kt index 60ddf20894c..12d163d3e15 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinDaemonIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinDaemonIT.kt @@ -23,7 +23,7 @@ import org.gradle.tooling.internal.consumer.ConnectorServices import org.gradle.util.GradleVersion import org.jetbrains.kotlin.compilerRunner.* import org.jetbrains.kotlin.gradle.testbase.* -import org.junit.jupiter.api.Assumptions.assumeFalse +import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.DisplayName import java.nio.file.Paths import kotlin.test.assertTrue @@ -108,6 +108,7 @@ class KotlinDaemonIT : KGPDaemonsBaseTest() { project( "kotlinProject", gradleVersion, + enableKotlinDaemonMemoryLimitInMb = null, buildOptions = defaultBuildOptions.copy(logLevel = LogLevel.INFO) ) { gradleProperties.append( @@ -156,11 +157,11 @@ class KotlinDaemonIT : KGPDaemonsBaseTest() { } @DisplayName("KT-57154: Compiler should use specified toolchain regardless of Gradle Runtime JDK") + @Disabled("KT-58894: re-enable once fixed") @JdkVersions(versions = [JavaVersion.VERSION_1_8, JavaVersion.VERSION_11, JavaVersion.VERSION_17]) @GradleWithJdkTest @GradleTestVersions(minVersion = TestVersions.Gradle.MAX_SUPPORTED) internal fun testCompilerRuntimeJdkToolchainIndependence(gradleVersion: GradleVersion, jdkVersion: JdkVersions.ProvidedJdk) { - assumeFalse(defaultBuildOptions.isK2ByDefault, "KT-58894") project( projectName = "kotlin-java-toolchain/onlyJdk11Compatible", gradleVersion = gradleVersion, diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinDaemonJvmArgsTest.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinDaemonJvmArgsTest.kt index 2073560c184..46cb3d18212 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinDaemonJvmArgsTest.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinDaemonJvmArgsTest.kt @@ -9,6 +9,7 @@ import org.gradle.api.logging.LogLevel import org.gradle.util.GradleVersion import org.jetbrains.kotlin.gradle.testbase.* import org.junit.jupiter.api.DisplayName +import kotlin.io.path.writeText @DisplayName("Kotlin daemon JVM args") class KotlinDaemonJvmArgsTest : KGPDaemonsBaseTest() { @@ -18,7 +19,7 @@ class KotlinDaemonJvmArgsTest : KGPDaemonsBaseTest() { @GradleTest @DisplayName("Kotlin daemon by default should inherit Gradle daemon max jvm heap size") internal fun shouldInheritGradleDaemonArgsByDefault(gradleVersion: GradleVersion) { - project("simpleProject", gradleVersion) { + project("simpleProject", gradleVersion, enableKotlinDaemonMemoryLimitInMb = null) { gradleProperties.append( """ org.gradle.jvmargs = -Xmx758m @@ -36,7 +37,7 @@ class KotlinDaemonJvmArgsTest : KGPDaemonsBaseTest() { @DisplayName("Kotlin daemon should allow to define own jvm options via gradle daemon jvm args system property") @GradleTest internal fun shouldAllowToRedefineViaDGradleOption(gradleVersion: GradleVersion) { - project("simpleProject", gradleVersion) { + project("simpleProject", gradleVersion, enableKotlinDaemonMemoryLimitInMb = null) { gradleProperties.append( """ org.gradle.jvmargs =-Xmx758m -Dkotlin.daemon.jvm.options=Xmx1g,Xms128m @@ -54,8 +55,8 @@ class KotlinDaemonJvmArgsTest : KGPDaemonsBaseTest() { @DisplayName("Jvm args defined in gradle.properties should override Gradle daemon jvm arguments inheritance") @GradleTest internal fun shouldUseArgumentsFromGradleProperties(gradleVersion: GradleVersion) { - project("simpleProject", gradleVersion) { - gradleProperties.append( + project("simpleProject", gradleVersion, enableKotlinDaemonMemoryLimitInMb = null) { + gradleProperties.writeText( """ org.gradle.jvmargs =-Xmx758m -Xms128m kotlin.daemon.jvmargs = -Xmx486m -Xms256m @@ -73,7 +74,7 @@ class KotlinDaemonJvmArgsTest : KGPDaemonsBaseTest() { @DisplayName("Should use arguments from extension DSL") @GradleTest internal fun shouldUseDslArguments(gradleVersion: GradleVersion) { - project("simpleProject", gradleVersion) { + project("simpleProject", gradleVersion, enableKotlinDaemonMemoryLimitInMb = null) { gradleProperties.append( """ org.gradle.jvmargs =-Xmx758m -Xms128m @@ -104,9 +105,10 @@ class KotlinDaemonJvmArgsTest : KGPDaemonsBaseTest() { internal fun allowOverrideArgsForSpecificTask(gradleVersion: GradleVersion) { project( projectName = "simpleProject", - gradleVersion = gradleVersion + gradleVersion = gradleVersion, + enableKotlinDaemonMemoryLimitInMb = null ) { - gradleProperties.append( + gradleProperties.writeText( """ org.gradle.jvmargs = -Xmx758m -Xms128m kotlin.daemon.jvmargs = -Xmx486m -Xms256m @@ -143,7 +145,7 @@ class KotlinDaemonJvmArgsTest : KGPDaemonsBaseTest() { @DisplayName("Should inherit Gradle memory settings if it is not set in Kotlin daemon jvm args") @GradleTest fun inheritGradleMemorySettingsIfKotlinArgsNotContain(gradleVersion: GradleVersion) { - project("simpleProject", gradleVersion) { + project("simpleProject", gradleVersion, enableKotlinDaemonMemoryLimitInMb = null) { gradleProperties.append( """ org.gradle.jvmargs =-Xmx758m -Xms128m diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/testDsl.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/testDsl.kt index 0b67d5d48a2..51f9d6deb8b 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/testDsl.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/testDsl.kt @@ -32,6 +32,8 @@ import kotlin.test.assertTrue * @param [projectName] test project name in `src/test/resources/testProject` directory. * @param [buildOptions] common Gradle build options * @param [buildJdk] path to JDK build should run with. *Note* Only append to 'gradle.properties'! + * @param [enableKotlinDaemonMemoryLimitInMb] limit max heap size for Kotlin Daemon. + * `null` enables the default limit inherited from the Gradle process. */ @OptIn(EnvironmentalVariablesOverride::class) fun KGPBaseTest.project( @@ -42,6 +44,7 @@ fun KGPBaseTest.project( enableBuildScan: Boolean = false, addHeapDumpOptions: Boolean = true, enableGradleDebug: Boolean = false, + enableKotlinDaemonMemoryLimitInMb: Int? = 1024, projectPathAdditionalSuffix: String = "", buildJdk: File? = null, localRepoDir: Path? = null, @@ -76,6 +79,7 @@ fun KGPBaseTest.project( forceOutput = forceOutput, enableBuildScan = enableBuildScan, enableGradleDebug = enableGradleDebug, + enableKotlinDaemonMemoryLimitInMb = enableKotlinDaemonMemoryLimitInMb, environmentVariables = environmentVariables ) addHeapDumpOptions.ifTrue { testProject.addHeapDumpOptions() } @@ -98,6 +102,8 @@ fun KGPBaseTest.project( * @param [projectName] test project name in 'src/test/resources/testProject` directory. * @param [buildOptions] common Gradle build options * @param [buildJdk] path to JDK build should run with. *Note* Only append to 'gradle.properties'! + * @param [enableKotlinDaemonMemoryLimitInMb] limit max heap size for Kotlin Daemon. + * `null` enables the default limit inherited from the Gradle process. */ @OptIn(EnvironmentalVariablesOverride::class) fun KGPBaseTest.nativeProject( @@ -108,6 +114,7 @@ fun KGPBaseTest.nativeProject( enableBuildScan: Boolean = false, addHeapDumpOptions: Boolean = true, enableGradleDebug: Boolean = false, + enableKotlinDaemonMemoryLimitInMb: Int? = 1024, projectPathAdditionalSuffix: String = "", buildJdk: File? = null, localRepoDir: Path? = null, @@ -123,6 +130,7 @@ fun KGPBaseTest.nativeProject( enableBuildScan = enableBuildScan, addHeapDumpOptions = addHeapDumpOptions, enableGradleDebug = enableGradleDebug, + enableKotlinDaemonMemoryLimitInMb = enableKotlinDaemonMemoryLimitInMb, projectPathAdditionalSuffix = projectPathAdditionalSuffix, buildJdk = buildJdk, localRepoDir = localRepoDir, @@ -143,6 +151,7 @@ fun TestProject.build( kotlinDaemonDebugPort: Int? = this.kotlinDaemonDebugPort, enableBuildCacheDebug: Boolean = false, enableBuildScan: Boolean = this.enableBuildScan, + enableKotlinDaemonMemoryLimitInMb: Int? = this.enableKotlinDaemonMemoryLimitInMb, buildOptions: BuildOptions = this.buildOptions, environmentVariables: EnvironmentalVariables = this.environmentVariables, assertions: BuildResult.() -> Unit = {}, @@ -155,6 +164,7 @@ fun TestProject.build( buildOptions, enableBuildCacheDebug, enableBuildScan, + enableKotlinDaemonMemoryLimitInMb, gradleVersion, kotlinDaemonDebugPort ) @@ -182,6 +192,7 @@ fun TestProject.buildAndFail( enableBuildCacheDebug: Boolean = false, enableBuildScan: Boolean = this.enableBuildScan, buildOptions: BuildOptions = this.buildOptions, + enableKotlinDaemonMemoryLimitInMb: Int? = this.enableKotlinDaemonMemoryLimitInMb, environmentVariables: EnvironmentalVariables = this.environmentVariables, assertions: BuildResult.() -> Unit = {}, ) { @@ -193,6 +204,7 @@ fun TestProject.buildAndFail( buildOptions, enableBuildCacheDebug, enableBuildScan, + enableKotlinDaemonMemoryLimitInMb, gradleVersion, kotlinDaemonDebugPort ) @@ -233,6 +245,7 @@ internal inline fun TestProject.getModels( buildOptions, false, enableBuildScan, + enableKotlinDaemonMemoryLimitInMb, gradleVersion ) @@ -350,6 +363,7 @@ class TestProject( val gradleVersion: GradleVersion, val forceOutput: Boolean, val enableBuildScan: Boolean, + val enableKotlinDaemonMemoryLimitInMb: Int?, /** * Whether the test and the Gradle build launched by the test should be executed in the same process so that we can use the same * debugger for both (see https://docs.gradle.org/current/javadoc/org/gradle/testkit/runner/GradleRunner.html#isDebug--). @@ -413,6 +427,7 @@ private fun commonBuildSetup( buildOptions: BuildOptions, enableBuildCacheDebug: Boolean, enableBuildScan: Boolean, + enableKotlinDaemonMemoryLimitInMb: Int?, gradleVersion: GradleVersion, kotlinDaemonDebugPort: Int? = null, ): List { @@ -429,6 +444,10 @@ private fun commonBuildSetup( // Decreasing Gradle daemon idle timeout to 1 min from default 3 hours. // This should help with OOM on CI when agents do not have enough free memory available. "-Dorg.gradle.daemon.idletimeout=60000", + if (enableKotlinDaemonMemoryLimitInMb != null) { + // Limiting Kotlin daemon heap size to reduce memory pressure on CI agents + "-Pkotlin.daemon.jvmargs=-Xmx${enableKotlinDaemonMemoryLimitInMb}m" + } else null, if (enableBuildCacheDebug) "-Dorg.gradle.caching.debug=true" else null, if (enableBuildScan) "--scan" else null, kotlinDaemonDebugPort?.let {