From e1533579f9170680a0840fb40862bd2746cb3229 Mon Sep 17 00:00:00 2001 From: Yahor Berdnikau Date: Thu, 8 Feb 2024 23:13:21 +0100 Subject: [PATCH] [Gradle] Limit Gradle daemon memory to 1g Limiting spawned by our Gradle integrations tests Gradle daemons maximum heap size to 1g helps to reduce memory pressure on the system when running these tests in parallel. Especially it is actual for our CI agents. ^KT-65701 Fixed --- .../kotlin/gradle/ExecutionStrategyIT.kt | 6 ++- .../kotlin/gradle/KotlinDaemonJvmArgsTest.kt | 38 ++++++++++++++++--- .../kotlin/gradle/native/CommonNativeIT.kt | 3 -- .../kotlin/gradle/testbase/testDsl.kt | 15 ++++++++ .../kotlin/gradle/testbase/testHelpers.kt | 13 ------- 5 files changed, 51 insertions(+), 24 deletions(-) diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ExecutionStrategyIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ExecutionStrategyIT.kt index 0da0c41b114..cbc56a99263 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ExecutionStrategyIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/ExecutionStrategyIT.kt @@ -213,6 +213,7 @@ abstract class ExecutionStrategyIT : KGPDaemonsBaseTest() { gradleVersion = gradleVersion, addHeapDumpOptions = addHeapDumpOptions, enableKotlinDaemonMemoryLimitInMb = if (shouldConfigureStrategyViaGradleProperty) null else 1024, + enableGradleDaemonMemoryLimitInMb = null, // We need to make an assertion based on default Gradle Daemon JDK configuration buildOptions = defaultBuildOptions.copy( useDaemonFallbackStrategy = testFallbackStrategy, compilerExecutionStrategy = if (shouldConfigureStrategyViaGradleProperty) { @@ -248,9 +249,10 @@ abstract class ExecutionStrategyIT : KGPDaemonsBaseTest() { assertOutputContains("Using fallback strategy: Compile without Kotlin daemon") } else if (executionStrategy == KotlinCompilerExecutionStrategy.DAEMON) { // 256m is the default value for Gradle 5.0+ - val defauldJvmSettingsForGivenGradleVersion = if (gradleVersion < GradleVersion.version(TestVersions.Gradle.G_8_0)) "256" else "384" + val defaultJvmSettingsForGivenGradleVersion = + if (gradleVersion < GradleVersion.version(TestVersions.Gradle.G_8_0)) "256" else "384" assertKotlinDaemonJvmOptions( - listOf("-XX:MaxMetaspaceSize=${defauldJvmSettingsForGivenGradleVersion}m", "-ea") + listOf("-XX:MaxMetaspaceSize=${defaultJvmSettingsForGivenGradleVersion}m", "-ea") ) } } diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinDaemonJvmArgsTest.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinDaemonJvmArgsTest.kt index 46cb3d18212..537373731fa 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinDaemonJvmArgsTest.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/KotlinDaemonJvmArgsTest.kt @@ -19,7 +19,12 @@ class KotlinDaemonJvmArgsTest : KGPDaemonsBaseTest() { @GradleTest @DisplayName("Kotlin daemon by default should inherit Gradle daemon max jvm heap size") internal fun shouldInheritGradleDaemonArgsByDefault(gradleVersion: GradleVersion) { - project("simpleProject", gradleVersion, enableKotlinDaemonMemoryLimitInMb = null) { + project( + projectName = "simpleProject", + gradleVersion = gradleVersion, + enableKotlinDaemonMemoryLimitInMb = null, + enableGradleDaemonMemoryLimitInMb = null, + ) { gradleProperties.append( """ org.gradle.jvmargs = -Xmx758m @@ -37,7 +42,12 @@ class KotlinDaemonJvmArgsTest : KGPDaemonsBaseTest() { @DisplayName("Kotlin daemon should allow to define own jvm options via gradle daemon jvm args system property") @GradleTest internal fun shouldAllowToRedefineViaDGradleOption(gradleVersion: GradleVersion) { - project("simpleProject", gradleVersion, enableKotlinDaemonMemoryLimitInMb = null) { + project( + projectName = "simpleProject", + gradleVersion = gradleVersion, + enableKotlinDaemonMemoryLimitInMb = null, + enableGradleDaemonMemoryLimitInMb = null, + ) { gradleProperties.append( """ org.gradle.jvmargs =-Xmx758m -Dkotlin.daemon.jvm.options=Xmx1g,Xms128m @@ -55,7 +65,12 @@ class KotlinDaemonJvmArgsTest : KGPDaemonsBaseTest() { @DisplayName("Jvm args defined in gradle.properties should override Gradle daemon jvm arguments inheritance") @GradleTest internal fun shouldUseArgumentsFromGradleProperties(gradleVersion: GradleVersion) { - project("simpleProject", gradleVersion, enableKotlinDaemonMemoryLimitInMb = null) { + project( + projectName = "simpleProject", + gradleVersion = gradleVersion, + enableKotlinDaemonMemoryLimitInMb = null, + enableGradleDaemonMemoryLimitInMb = null, + ) { gradleProperties.writeText( """ org.gradle.jvmargs =-Xmx758m -Xms128m @@ -74,7 +89,12 @@ class KotlinDaemonJvmArgsTest : KGPDaemonsBaseTest() { @DisplayName("Should use arguments from extension DSL") @GradleTest internal fun shouldUseDslArguments(gradleVersion: GradleVersion) { - project("simpleProject", gradleVersion, enableKotlinDaemonMemoryLimitInMb = null) { + project( + projectName = "simpleProject", + gradleVersion = gradleVersion, + enableKotlinDaemonMemoryLimitInMb = null, + enableGradleDaemonMemoryLimitInMb = null, + ) { gradleProperties.append( """ org.gradle.jvmargs =-Xmx758m -Xms128m @@ -106,7 +126,8 @@ class KotlinDaemonJvmArgsTest : KGPDaemonsBaseTest() { project( projectName = "simpleProject", gradleVersion = gradleVersion, - enableKotlinDaemonMemoryLimitInMb = null + enableKotlinDaemonMemoryLimitInMb = null, + enableGradleDaemonMemoryLimitInMb = null, ) { gradleProperties.writeText( """ @@ -145,7 +166,12 @@ class KotlinDaemonJvmArgsTest : KGPDaemonsBaseTest() { @DisplayName("Should inherit Gradle memory settings if it is not set in Kotlin daemon jvm args") @GradleTest fun inheritGradleMemorySettingsIfKotlinArgsNotContain(gradleVersion: GradleVersion) { - project("simpleProject", gradleVersion, enableKotlinDaemonMemoryLimitInMb = null) { + project( + projectName = "simpleProject", + gradleVersion = gradleVersion, + enableKotlinDaemonMemoryLimitInMb = null, + enableGradleDaemonMemoryLimitInMb = null, + ) { gradleProperties.append( """ org.gradle.jvmargs =-Xmx758m -Xms128m diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/native/CommonNativeIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/native/CommonNativeIT.kt index 7fad28996bd..1989df46279 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/native/CommonNativeIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/native/CommonNativeIT.kt @@ -63,9 +63,6 @@ class CommonNativeIT : KGPBaseTest() { gradleVersion, localRepoDir = defaultLocalRepo(gradleVersion) ) { - - configureJvmMemory() - val libCompileTasks = libTargets.map { ":lib:compileKotlin${it.capitalize()}" } val appCompileTasks = appTargets.map { ":app:compileKotlin${it.capitalize()}" } val appLinkFrameworkTasks = appTargets.map { ":app:linkDebugFramework${it.capitalize()}" } diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/testDsl.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/testDsl.kt index 122882b3228..5e083f3fc2a 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/testDsl.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/testDsl.kt @@ -46,6 +46,7 @@ fun KGPBaseTest.project( enableBuildScan: Boolean = false, addHeapDumpOptions: Boolean = true, enableGradleDebug: Boolean = false, + enableGradleDaemonMemoryLimitInMb: Int? = 1024, enableKotlinDaemonMemoryLimitInMb: Int? = 1024, projectPathAdditionalSuffix: String = "", buildJdk: File? = null, @@ -82,6 +83,7 @@ fun KGPBaseTest.project( forceOutput = forceOutput, enableBuildScan = enableBuildScan, enableGradleDebug = enableGradleDebug, + enableGradleDaemonMemoryLimitInMb = enableGradleDaemonMemoryLimitInMb, enableKotlinDaemonMemoryLimitInMb = enableKotlinDaemonMemoryLimitInMb, environmentVariables = environmentVariables ) @@ -117,6 +119,7 @@ fun KGPBaseTest.nativeProject( enableBuildScan: Boolean = false, addHeapDumpOptions: Boolean = true, enableGradleDebug: Boolean = false, + enableGradleDaemonMemoryLimitInMb: Int? = 1024, enableKotlinDaemonMemoryLimitInMb: Int? = 1024, projectPathAdditionalSuffix: String = "", buildJdk: File? = null, @@ -135,6 +138,7 @@ fun KGPBaseTest.nativeProject( dependencyManagement = dependencyManagement, addHeapDumpOptions = addHeapDumpOptions, enableGradleDebug = enableGradleDebug, + enableGradleDaemonMemoryLimitInMb = enableGradleDaemonMemoryLimitInMb, enableKotlinDaemonMemoryLimitInMb = enableKotlinDaemonMemoryLimitInMb, projectPathAdditionalSuffix = projectPathAdditionalSuffix, buildJdk = buildJdk, @@ -156,6 +160,7 @@ fun TestProject.build( kotlinDaemonDebugPort: Int? = this.kotlinDaemonDebugPort, enableBuildCacheDebug: Boolean = false, enableBuildScan: Boolean = this.enableBuildScan, + enableGradleDaemonMemoryLimitInMb: Int? = this.enableGradleDaemonMemoryLimitInMb, enableKotlinDaemonMemoryLimitInMb: Int? = this.enableKotlinDaemonMemoryLimitInMb, buildOptions: BuildOptions = this.buildOptions, environmentVariables: EnvironmentalVariables = this.environmentVariables, @@ -169,6 +174,7 @@ fun TestProject.build( buildOptions, enableBuildCacheDebug, enableBuildScan, + enableGradleDaemonMemoryLimitInMb, enableKotlinDaemonMemoryLimitInMb, gradleVersion, kotlinDaemonDebugPort @@ -197,6 +203,7 @@ fun TestProject.buildAndFail( enableBuildCacheDebug: Boolean = false, enableBuildScan: Boolean = this.enableBuildScan, buildOptions: BuildOptions = this.buildOptions, + enableGradleDaemonMemoryLimitInMb: Int? = this.enableGradleDaemonMemoryLimitInMb, enableKotlinDaemonMemoryLimitInMb: Int? = this.enableKotlinDaemonMemoryLimitInMb, environmentVariables: EnvironmentalVariables = this.environmentVariables, assertions: BuildResult.() -> Unit = {}, @@ -209,6 +216,7 @@ fun TestProject.buildAndFail( buildOptions, enableBuildCacheDebug, enableBuildScan, + enableGradleDaemonMemoryLimitInMb, enableKotlinDaemonMemoryLimitInMb, gradleVersion, kotlinDaemonDebugPort @@ -254,6 +262,7 @@ internal inline fun TestProject.getModels( buildOptions, false, enableBuildScan, + enableGradleDaemonMemoryLimitInMb, enableKotlinDaemonMemoryLimitInMb, gradleVersion ) @@ -373,6 +382,7 @@ class TestProject( val gradleVersion: GradleVersion, val forceOutput: Boolean, val enableBuildScan: Boolean, + val enableGradleDaemonMemoryLimitInMb: Int?, val enableKotlinDaemonMemoryLimitInMb: Int?, /** * Whether the test and the Gradle build launched by the test should be executed in the same process so that we can use the same @@ -444,6 +454,7 @@ private fun commonBuildSetup( buildOptions: BuildOptions, enableBuildCacheDebug: Boolean, enableBuildScan: Boolean, + enableGradleDaemonMemoryLimitInMb: Int?, enableKotlinDaemonMemoryLimitInMb: Int?, gradleVersion: GradleVersion, kotlinDaemonDebugPort: Int? = null, @@ -461,6 +472,10 @@ private fun commonBuildSetup( // Decreasing Gradle daemon idle timeout to 1 min from default 3 hours. // This should help with OOM on CI when agents do not have enough free memory available. "-Dorg.gradle.daemon.idletimeout=60000", + if (enableGradleDaemonMemoryLimitInMb != null) { + // Limiting Gradle daemon heap size to reduce memory pressure on CI agents + "-Dorg.gradle.jvmargs=-Xmx${enableGradleDaemonMemoryLimitInMb}m" + } else null, if (enableKotlinDaemonMemoryLimitInMb != null) { // Limiting Kotlin daemon heap size to reduce memory pressure on CI agents "-Pkotlin.daemon.jvmargs=-Xmx${enableKotlinDaemonMemoryLimitInMb}m" diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/testHelpers.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/testHelpers.kt index dae869bf0b8..05c06deb34b 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/testHelpers.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/testHelpers.kt @@ -89,19 +89,6 @@ private fun TestProject.formatEnvironmentForScript(envCommand: String): String { } } -/** - * - * Configures the JVM memory settings for the Gradle project. - * @param memorySizeInGb The amount of memory to allocate to the JVM, in gigabytes. - * Defaults to 1 gigabyte. - */ -fun GradleProject.configureJvmMemory(memorySizeInGb: Number = 1) { - addPropertyToGradleProperties( - propertyName = "org.gradle.jvmargs", - mapOf("-Xmx" to "-Xmx${memorySizeInGb}g") - ) -} - /** * Adds the given options to a Gradle property specified by name, in the project's Gradle properties file. * If the property does not exist, it is created.