[Test] Add ability to run box test in separate jvm for common codegen tests

compiler/test-infrastructure-utils/tests/org/jetbrains/kotlin/test/TestJdkKind.kt

@@ -15,7 +15,7 @@
  */
 package org.jetbrains.kotlin.test
 
-enum class TestJdkKind {
+enum class TestJdkKind(val requiresSeparateProcess: Boolean = false) {
     MOCK_JDK,
 
     // Differs from common mock JDK only by one additional 'nonExistingMethod' in Collection and constructor from Double in Throwable
@@ -24,16 +24,16 @@ enum class TestJdkKind {
     MODIFIED_MOCK_JDK,
 
     // JDK found at $JDK_16
-    FULL_JDK_6,
+    FULL_JDK_6(requiresSeparateProcess = true),
 
     // JDK found at $JDK_9
-    FULL_JDK_9,
+    FULL_JDK_9(requiresSeparateProcess = true),
 
     // JDK found at $JDK_15
-    FULL_JDK_15,
+    FULL_JDK_15(requiresSeparateProcess = true),
 
     // JDK found at $JDK_15
-    FULL_JDK_17,
+    FULL_JDK_17(requiresSeparateProcess = true),
 
     // JDK found at java.home
     FULL_JDK,

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/model/Modules.kt

@@ -61,6 +61,9 @@ class TestFile(
     val name: String = relativePath.split("/").last()
 }
 
+val TestFile.nameWithoutExtension: String
+    get() = name.substringBeforeLast(".")
+
 enum class DependencyRelation {
     RegularDependency,
     FriendDependency,

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/JvmBoxRunner.kt

@@ -11,20 +11,30 @@ import org.jetbrains.kotlin.codegen.*
 import org.jetbrains.kotlin.codegen.forTestCompile.ForTestCompileRuntime
 import org.jetbrains.kotlin.fileClasses.JvmFileClassUtil.getFileClassInfoNoResolve
 import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.test.TestJdkKind
 import org.jetbrains.kotlin.test.clientserver.TestProxy
 import org.jetbrains.kotlin.test.directives.CodegenTestDirectives
+import org.jetbrains.kotlin.test.directives.JvmEnvironmentConfigurationDirectives.JDK_KIND
+import org.jetbrains.kotlin.test.directives.model.singleOrZeroValue
 import org.jetbrains.kotlin.test.model.BinaryArtifacts
 import org.jetbrains.kotlin.test.model.DependencyKind
 import org.jetbrains.kotlin.test.model.TestModule
+import org.jetbrains.kotlin.test.model.nameWithoutExtension
 import org.jetbrains.kotlin.test.services.*
 import org.jetbrains.kotlin.test.services.configuration.JvmEnvironmentConfigurator.Companion.TEST_CONFIGURATION_KIND_KEY
 import org.jetbrains.kotlin.test.services.jvm.compiledClassesManager
+import org.jetbrains.kotlin.test.services.sourceProviders.MainFunctionForBlackBoxTestsSourceProvider
+import org.jetbrains.kotlin.test.services.sourceProviders.MainFunctionForBlackBoxTestsSourceProvider.Companion.containsBoxMethod
+import org.jetbrains.kotlin.test.util.KtTestUtil
 import org.jetbrains.kotlin.utils.addIfNotNull
 import org.jetbrains.kotlin.utils.addToStdlib.runIf
 import java.io.File
 import java.lang.reflect.Method
+import java.net.URI
 import java.net.URL
 import java.net.URLClassLoader
+import java.nio.file.Paths
+import java.util.concurrent.TimeUnit
 
 class JvmBoxRunner(testServices: TestServices) : JvmBinaryArtifactHandler(testServices) {
     companion object {
@@ -45,24 +55,18 @@ class JvmBoxRunner(testServices: TestServices) : JvmBinaryArtifactHandler(testSe
         val classLoader = createAndVerifyClassLoader(module, info.classFileFactory, reportProblems)
         try {
             for (ktFile in ktFiles) {
-                val className = ktFile.getFacadeFqName() ?: continue
+                if (containsBoxMethod(ktFile.text)) {
-                val clazz = try {
+                    boxMethodFound = true
-                    classLoader.getGeneratedClass(className)
+                    callBoxMethodAndCheckResultWithCleanup(
-                } catch (e: LinkageError) {
+                        ktFile,
-                    throw AssertionError("Failed to load class '$className':\n${info.classFileFactory.createText()}", e)
+                        module,
+                        info.classFileFactory,
+                        classLoader,
+                        unexpectedBehaviour = false,
+                        reportProblems = reportProblems
+                    )
+                    return
                 }
-                val method = clazz.getBoxMethodOrNull() ?: continue
-                boxMethodFound = true
-                callBoxMethodAndCheckResultWithCleanup(
-                    module,
-                    info.classFileFactory,
-                    classLoader,
-                    clazz,
-                    method,
-                    unexpectedBehaviour = false,
-                    reportProblems = reportProblems
-                )
-                return
             }
         } finally {
             classLoader.dispose()
@@ -70,16 +74,15 @@ class JvmBoxRunner(testServices: TestServices) : JvmBinaryArtifactHandler(testSe
     }
 
     private fun callBoxMethodAndCheckResultWithCleanup(
+        ktFile: KtFile,
         module: TestModule,
         classFileFactory: ClassFileFactory,
         classLoader: URLClassLoader,
-        clazz: Class<*>,
-        method: Method,
         unexpectedBehaviour: Boolean,
         reportProblems: Boolean
     ) {
         try {
-            callBoxMethodAndCheckResult(module, classFileFactory, classLoader, clazz, method, unexpectedBehaviour)
+            callBoxMethodAndCheckResult(ktFile, module, classFileFactory, classLoader, unexpectedBehaviour)
         } catch (e: Throwable) {
             if (reportProblems) {
                 try {
@@ -97,28 +100,44 @@ class JvmBoxRunner(testServices: TestServices) : JvmBinaryArtifactHandler(testSe
         }
     }
 
+    private fun findClassAndMethodToExecute(
+        ktFile: KtFile,
+        classLoader: URLClassLoader,
+        classFileFactory: ClassFileFactory
+    ): Pair<Class<*>, Method> {
+        val className = ktFile.getFacadeFqName() ?: error("Class<*> for ${ktFile.name} not found")
+        val clazz = try {
+            classLoader.getGeneratedClass(className)
+        } catch (e: LinkageError) {
+            throw AssertionError("Failed to load class '$className':\n${classFileFactory.createText()}", e)
+        }
+        val method = clazz.getBoxMethodOrNull() ?: error("box method not found")
+        return clazz to method
+    }
+
     private fun callBoxMethodAndCheckResult(
+        ktFile: KtFile,
         module: TestModule,
         classFileFactory: ClassFileFactory,
         classLoader: URLClassLoader,
-        clazz: Class<*>,
-        method: Method,
         unexpectedBehaviour: Boolean
     ) {
         val result = if (BOX_IN_SEPARATE_PROCESS_PORT != null) {
-            invokeBoxInSeparateProcess(module, classFileFactory, classLoader, clazz)
+            invokeBoxInSeparateProcess(
+                module,
+                classFileFactory,
+                classLoader,
+                findClassAndMethodToExecute(ktFile, classLoader, classFileFactory).first
+            )
         } else {
-            val savedClassLoader = Thread.currentThread().contextClassLoader
+            val jdkKind = module.directives.singleOrZeroValue(JDK_KIND)
-            if (savedClassLoader !== classLoader) {
+            if (jdkKind?.requiresSeparateProcess == true) {
-                // otherwise the test infrastructure used in the test may conflict with the one from the context classloader
+                runSeparateJvmInstance(module, jdkKind, classLoader, classFileFactory)
-                Thread.currentThread().contextClassLoader = classLoader
+            } else {
-            }
+                runBoxInCurrentProcess(
-            try {
+                    classLoader,
-                method.invoke(null) as String
+                    findClassAndMethodToExecute(ktFile, classLoader, classFileFactory).second
-            } finally {
+                )
-                if (savedClassLoader !== classLoader) {
-                    Thread.currentThread().contextClassLoader = savedClassLoader
-                }
             }
         }
         if (unexpectedBehaviour) {
@@ -128,24 +147,103 @@ class JvmBoxRunner(testServices: TestServices) : JvmBinaryArtifactHandler(testSe
         }
     }
 
+    private fun runBoxInCurrentProcess(
+        classLoader: URLClassLoader,
+        method: Method,
+    ): String {
+        val savedClassLoader = Thread.currentThread().contextClassLoader
+        if (savedClassLoader !== classLoader) {
+            // otherwise the test infrastructure used in the test may conflict with the one from the context classloader
+            Thread.currentThread().contextClassLoader = classLoader
+        }
+        return try {
+            method.invoke(null) as String
+        } finally {
+            if (savedClassLoader !== classLoader) {
+                Thread.currentThread().contextClassLoader = savedClassLoader
+            }
+        }
+    }
+
+    /*
+     * TODO:
+     * Running separate jvm for each test may be very expensive in case
+     *   there will be a lot of tests which use this feature, so we should
+     *   consider to run single jvm as proxy (see [invokeBoxInSeparateProcess])
+     */
+    private fun runSeparateJvmInstance(
+        module: TestModule,
+        jdkKind: TestJdkKind,
+        classLoader: URLClassLoader,
+        classFileFactory: ClassFileFactory
+    ): String {
+        val jdkHome = when (jdkKind) {
+            TestJdkKind.FULL_JDK_9 -> KtTestUtil.getJdk9Home()
+            TestJdkKind.FULL_JDK_15 -> KtTestUtil.getJdk15Home()
+            TestJdkKind.FULL_JDK_17 -> KtTestUtil.getJdk17Home()
+            else -> error("Unsupported JDK kind: $jdkKind")
+        }
+
+        val javaExe = File(jdkHome, "bin/java.exe").takeIf(File::exists)
+            ?: File(jdkHome, "bin/java").takeIf(File::exists)
+            ?: error("Can't find 'java' executable in $jdkHome")
+
+        val classPath = extractClassPath(module, classLoader, classFileFactory)
+
+        val mainFile = module.files.firstOrNull {
+            it.name == MainFunctionForBlackBoxTestsSourceProvider.BOX_MAIN_FILE_NAME && it.isAdditional
+        } ?: error("No file with main function was generated. Please check TODO source provider")
+
+        val mainFqName = listOfNotNull(
+            MainFunctionForBlackBoxTestsSourceProvider.detectPackage(mainFile),
+            "${mainFile.nameWithoutExtension}Kt"
+        ).joinToString(".")
+
+        val command = arrayOf(
+            javaExe.absolutePath,
+            "-ea",
+            "-classpath",
+            classPath.joinToString(File.pathSeparator, transform = { File(it.toURI()).absolutePath }),
+            mainFqName,
+        )
+        val process = ProcessBuilder(*command).inheritIO().start()
+        process.waitFor(1, TimeUnit.MINUTES)
+        process.outputStream.flush()
+        return when (process.exitValue()) {
+            0 -> "OK"
+            else -> "External process completed with error. Check the build log"
+        }
+    }
+
     private fun invokeBoxInSeparateProcess(
         module: TestModule,
         classFileFactory: ClassFileFactory,
         classLoader: URLClassLoader,
         clazz: Class<*>
     ): String {
-        val classPath = classLoader.extractUrls().toMutableList()
+        val classPath = extractClassPath(module, classLoader, classFileFactory)
-        if (classLoader is GeneratedClassLoader) {
-            val javaPath = testServices.compiledClassesManager.getCompiledJavaDirForModule(module)?.url
-            if (javaPath != null) {
-                classPath.add(0, javaPath)
-            }
-            classPath.add(0, testServices.compiledClassesManager.getCompiledKotlinDirForModule(module, classFileFactory).url)
-        }
         val proxy = TestProxy(Integer.valueOf(BOX_IN_SEPARATE_PROCESS_PORT), clazz.canonicalName, classPath)
         return proxy.runTest()
     }
 
+    @OptIn(ExperimentalStdlibApi::class)
+    private fun extractClassPath(
+        module: TestModule,
+        classLoader: URLClassLoader,
+        classFileFactory: ClassFileFactory
+    ): List<URL> {
+        return buildList {
+            addAll(classLoader.extractUrls())
+            if (classLoader is GeneratedClassLoader) {
+                val javaPath = testServices.compiledClassesManager.getCompiledJavaDirForModule(module)?.url
+                if (javaPath != null) {
+                    add(0, javaPath)
+                }
+                add(0, testServices.compiledClassesManager.getCompiledKotlinDirForModule(module, classFileFactory).url)
+            }
+        }
+    }
+
     private val File.url: URL
         get() = toURI().toURL()
 
@@ -155,9 +253,11 @@ class JvmBoxRunner(testServices: TestServices) : JvmBinaryArtifactHandler(testSe
         reportProblems: Boolean
     ): GeneratedClassLoader {
         val classLoader = createClassLoader(module, classFileFactory)
-        val verificationSucceeded = CodegenTestUtil.verifyAllFilesWithAsm(classFileFactory, classLoader, reportProblems)
+        if (module.directives.singleOrZeroValue(JDK_KIND)?.requiresSeparateProcess != true) {
-        if (!verificationSucceeded) {
+            val verificationSucceeded = CodegenTestUtil.verifyAllFilesWithAsm(classFileFactory, classLoader, reportProblems)
-            assertions.fail { "Verification failed: see exceptions above" }
+            if (!verificationSucceeded) {
+                assertions.fail { "Verification failed: see exceptions above" }
+            }
         }
         return classLoader
     }

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/runners/codegen/BaseCodegenConfiguration.kt

@@ -18,6 +18,7 @@ import org.jetbrains.kotlin.test.services.configuration.ScriptingEnvironmentConf
 import org.jetbrains.kotlin.test.services.sourceProviders.AdditionalDiagnosticsSourceFilesProvider
 import org.jetbrains.kotlin.test.services.sourceProviders.CodegenHelpersSourceFilesProvider
 import org.jetbrains.kotlin.test.services.sourceProviders.CoroutineHelpersSourceFilesProvider
+import org.jetbrains.kotlin.test.services.sourceProviders.MainFunctionForBlackBoxTestsSourceProvider
 
 fun <R : ResultingArtifact.FrontendOutput<R>> TestConfigurationBuilder.commonConfigurationForCodegenTest(
     targetFrontend: FrontendKind<*>,
@@ -45,6 +46,7 @@ fun <R : ResultingArtifact.FrontendOutput<R>> TestConfigurationBuilder.commonCon
         ::AdditionalDiagnosticsSourceFilesProvider,
         ::CoroutineHelpersSourceFilesProvider,
         ::CodegenHelpersSourceFilesProvider,
+        ::MainFunctionForBlackBoxTestsSourceProvider,
     )
 
     useFrontendFacades(frontendFacade)

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/services/sourceProviders/MainFunctionForBlackBoxTestsSourceProvider.kt

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.services.sourceProviders
+
+import org.jetbrains.kotlin.test.directives.JvmEnvironmentConfigurationDirectives.JDK_KIND
+import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
+import org.jetbrains.kotlin.test.directives.model.singleOrZeroValue
+import org.jetbrains.kotlin.test.model.TestFile
+import org.jetbrains.kotlin.test.model.TestModule
+import org.jetbrains.kotlin.test.services.AdditionalSourceProvider
+import org.jetbrains.kotlin.test.services.TestServices
+import org.jetbrains.kotlin.test.services.temporaryDirectoryManager
+
+class MainFunctionForBlackBoxTestsSourceProvider(testServices: TestServices) : AdditionalSourceProvider(testServices) {
+    companion object {
+        private val PACKAGE_REGEXP = """package ([\w.]+)""".toRegex()
+        private val START_BOX_METHOD_REGEX = """^fun box\(\)""".toRegex()
+        private val MIDDLE_BOX_METHOD_REGEX = """\nfun box\(\)""".toRegex()
+
+        const val BOX_MAIN_FILE_NAME = "Generated_Box_Main.kt"
+
+        fun detectPackage(file: TestFile): String? {
+            return PACKAGE_REGEXP.find(file.originalContent)?.groups?.get(1)?.value
+        }
+
+        fun containsBoxMethod(file: TestFile): Boolean {
+            return containsBoxMethod(file.originalContent)
+        }
+
+        fun containsBoxMethod(fileContent: String): Boolean {
+            return START_BOX_METHOD_REGEX.containsMatchIn(fileContent) || MIDDLE_BOX_METHOD_REGEX.containsMatchIn(fileContent)
+        }
+    }
+
+    override fun produceAdditionalFiles(globalDirectives: RegisteredDirectives, module: TestModule): List<TestFile> {
+        val jdkKind = module.directives.singleOrZeroValue(JDK_KIND) ?: return emptyList()
+        if (!jdkKind.requiresSeparateProcess) return emptyList()
+
+        val fileWithBox = module.files.firstOrNull { containsBoxMethod(it) } ?: return emptyList()
+
+        val code = buildString {
+            detectPackage(fileWithBox)?.let {
+                appendLine("package $it")
+            }
+            appendLine(
+                """
+                    fun main() {
+                        val res = box()
+                        if (res != "OK") throw AssertionError(res)
+                    }
+                """.trimIndent()
+            )
+        }
+        val file = testServices.temporaryDirectoryManager.getOrCreateTempDirectory("src").resolve(BOX_MAIN_FILE_NAME)
+        file.writeText(code)
+
+        return listOf(file.toTestFile())
+    }
+}