diff --git a/gradle/jps.gradle.kts b/gradle/jps.gradle.kts
index 07e2080fa2e..376c7249df0 100644
--- a/gradle/jps.gradle.kts
+++ b/gradle/jps.gradle.kts
@@ -41,6 +41,24 @@ fun JUnit.configureForKotlin(xmx: String = "1600m") {
     workingDirectory = rootDir.toString()
 }
 
+// Needed because of idea.ext plugin can't pass \n symbol
+fun setupGenerateAllTestsRunConfiguration() {
+    rootDir.resolve(".idea/runConfigurations/JPS__Generate_All_Tests.xml").writeText(
+        """
+        |<component name="ProjectRunConfigurationManager">
+        | <configuration default="false" name="[JPS] Generate All Tests" type="Application" factoryName="Application">
+        |    <option name="MAIN_CLASS_NAME" value="org.jetbrains.kotlin.pill.generateAllTests.Main" />
+        |    <module name="kotlin.pill.generate-all-tests.test" />
+        |    <option name="VM_PARAMETERS" value="&quot;-Dline.separator=&#xA;&quot;" />
+        |    <shortenClasspath name="CLASSPATH_FILE" />
+        |    <method v="2">
+        |      <option name="Make" enabled="true" />
+        |    </method>
+        |  </configuration>
+        |</component>
+    """.trimMargin())
+}
+
 // Needed because of idea.ext plugin doesn't allow to set TEST_SEARCH_SCOPE = moduleWithDependencies
 fun setupFirRunConfiguration() {
 
@@ -107,6 +125,7 @@ if (kotlinBuildProperties.isInJpsBuildIdeaSync) {
     rootProject.afterEvaluate {
 
         setupFirRunConfiguration()
+        setupGenerateAllTestsRunConfiguration()
 
         rootProject.allprojects {
             idea {
@@ -196,12 +215,6 @@ if (kotlinBuildProperties.isInJpsBuildIdeaSync) {
                             idea("[JPS] IDEA Ultimate", ideaUltimateSandboxDir, ideaPluginDir)
                         }
 
-                        application("[JPS] Generate All Tests") {
-                            moduleName = "kotlin.pill.generate-all-tests.test"
-                            workingDirectory = rootDir.toString()
-                            mainClass = "org.jetbrains.kotlin.pill.generateAllTests.Main"
-                        }
-
                         defaults<JUnit> {
                             configureForKotlin()
                         }