[FIR] Forking runner for modularized test

compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractModularizedTest.kt

@@ -53,22 +53,27 @@ private val ROOT_PATH_PREFIX = System.getProperty("fir.bench.prefix", "/")
 
 abstract class AbstractModularizedTest : KtUsefulTestCase() {
     private val folderDateFormat = SimpleDateFormat("yyyy-MM-dd")
-    private lateinit var startDate: Date
+    private lateinit var reportDate: Date
 
-    protected fun reportDir() = File(FIR_LOGS_PATH, folderDateFormat.format(startDate))
+    protected fun reportDir() = File(FIR_LOGS_PATH, folderDateFormat.format(reportDate))
         .also {
             it.mkdirs()
         }
 
-    protected val reportDateStr by lazy {
+    protected val reportDateStr: String by lazy {
         val reportDateFormat = SimpleDateFormat("yyyy-MM-dd__HH-mm")
-        reportDateFormat.format(startDate)
+        reportDateFormat.format(reportDate)
+    }
+
+    private fun detectReportDate(): Date {
+        val provided = System.getProperty("fir.bench.report.timestamp") ?: return Date()
+        return Date(provided.toLong())
     }
 
     override fun setUp() {
         super.setUp()
         AbstractTypeChecker.RUN_SLOW_ASSERTIONS = false
-        startDate = Date()
+        reportDate = detectReportDate()
     }
 
     override fun tearDown() {

compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FirMetaModularizedTest.kt

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.fir
+
+import org.junit.Test
+import java.io.File
+import java.lang.management.ManagementFactory
+import kotlin.test.assertEquals
+
+class FirMetaModularizedTest {
+
+    private fun List<String>.filterArguments() = filterNot { it.startsWith("-Djava.security.manager") }
+
+    @Test
+    fun doTest() {
+        val runtimeBean = ManagementFactory.getRuntimeMXBean()
+        val jvmCommand = System.getProperty("java.home") + "/bin/java"
+
+        val runCount = System.getProperty("fir.bench.multirun.count").toInt()
+
+
+        val startTimestamp = System.currentTimeMillis()
+
+        for (i in 0 until runCount) {
+            val pb = ProcessBuilder()
+                .inheritIO()
+                .command(
+                    jvmCommand, "-cp", runtimeBean.classPath,
+                    *runtimeBean.inputArguments.filterArguments().toTypedArray(),
+                    "-Dfir.bench.report.timestamp=$startTimestamp",
+                    StandaloneModularizedTestRunner::class.java.canonicalName
+                )
+                .directory(File("").absoluteFile)
+
+            val process = pb.start()
+
+            assertEquals(0, process.waitFor(), "Forked test should complete normally")
+        }
+    }
+}

compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FirResolveModularizedTotalKotlinTest.kt

@@ -11,6 +11,7 @@ import com.intellij.psi.PsiElementFinder
 import com.intellij.psi.search.GlobalSearchScope
 import com.intellij.psi.search.ProjectScope
 import org.jetbrains.kotlin.asJava.finder.JavaElementFinder
+import org.jetbrains.kotlin.cli.common.toBooleanLenient
 import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.cli.jvm.compiler.TopDownAnalyzerFacadeForJVM
@@ -22,9 +23,6 @@ import org.jetbrains.kotlin.fir.resolve.firProvider
 import org.jetbrains.kotlin.fir.resolve.impl.FirProviderImpl
 import org.jetbrains.kotlin.fir.resolve.transformers.FirTotalResolveTransformer
 import org.jetbrains.kotlin.fir.scopes.ProcessorAction
-import org.jetbrains.kotlin.test.ConfigurationKind
-import org.jetbrains.kotlin.test.KotlinTestUtils
-import org.jetbrains.kotlin.test.TestJdkKind
 import java.io.File
 import java.io.FileOutputStream
 import java.io.PrintStream
@@ -36,9 +34,10 @@ private const val FIR_DUMP_PATH = "tmp/firDump"
 private const val FIR_HTML_DUMP_PATH = "tmp/firDump-html"
 const val FIR_LOGS_PATH = "tmp/fir-logs"
 
-private val DUMP_FIR = System.getProperty("fir.bench.dump", "true") == "true"
+private val DUMP_FIR = System.getProperty("fir.bench.dump", "true").toBooleanLenient()!!
 internal val PASSES = System.getProperty("fir.bench.passes")?.toInt() ?: 3
-internal val SEPARATE_PASS_DUMP = System.getProperty("fir.bench.dump.separate_pass", "false") == "true"
+internal val SEPARATE_PASS_DUMP = System.getProperty("fir.bench.dump.separate_pass", "false").toBooleanLenient()!!
+private val APPEND_ERROR_REPORTS = System.getProperty("fir.bench.report.errors.append", "false").toBooleanLenient()!!
 
 class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() {
 
@@ -157,7 +156,12 @@ class FirResolveModularizedTotalKotlinTest : AbstractModularizedTest() {
     }
 
     private fun printErrors(statistics: FirResolveBench.TotalStatistics) {
-        PrintStream(FileOutputStream(reportDir().resolve("errors-$reportDateStr.log"), true)).use(statistics::reportErrors)
+        PrintStream(
+            FileOutputStream(
+                reportDir().resolve("errors-$reportDateStr.log"),
+                APPEND_ERROR_REPORTS
+            )
+        ).use(statistics::reportErrors)
     }
 
     private fun printStatistics(statistics: FirResolveBench.TotalStatistics, header: String) {

compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/StandaloneModularizedTestRunner.kt

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.fir
+
+import org.junit.internal.runners.JUnit38ClassRunner
+import org.junit.runner.Description
+import org.junit.runner.notification.Failure
+import org.junit.runner.notification.RunListener
+import org.junit.runner.notification.RunNotifier
+import kotlin.system.exitProcess
+
+object StandaloneModularizedTestRunner {
+    @JvmStatic
+    fun main(args: Array<String>) {
+        val runner = JUnit38ClassRunner(FirResolveModularizedTotalKotlinTest::class.java)
+        var ok = true
+        runner.run(
+            RunNotifier().apply {
+                addListener(
+                    object : RunListener() {
+                        override fun testAssumptionFailure(failure: Failure?) {
+                            ok = false
+                            println("assertion failure: $failure")
+                            System.out.flush()
+                            failure?.exception?.printStackTrace()
+                            System.err.flush()
+                        }
+
+                        override fun testFailure(failure: Failure?) {
+                            ok = false
+                            println("test failure: $failure")
+                            System.out.flush()
+                            failure?.exception?.printStackTrace()
+                            System.err.flush()
+                        }
+
+                        override fun testIgnored(description: Description?) {
+                            println("test ignored: $description")
+                        }
+
+                        override fun testStarted(description: Description?) {
+                            println("test started: $description")
+                        }
+
+                        override fun testFinished(description: Description?) {
+                            println("test finished: $description")
+                        }
+                    },
+                )
+            },
+        )
+
+        println("runner exit")
+        exitProcess(if (ok) 0 else 1)
+
+    }
+}