Fix compilation against JRE 9 on JPS

Write the modular JDK (9+) path to the module.xml file passed to the
compiler from the JPS plugin. This path is then recorded in the compiler
configuration in KotlinToJVMBytecodeCompiler.configureSourceRoots. This
is needed because in JPS plugin, we pass "-no-jdk" and thus no JDK home
path was recorded in the compiler configuration in
K2JVMCompiler.setupJdkClasspathRoots. Presence of JDK home path in the
configuration is crucial for JDK 9 support (see
KotlinCoreEnvironment.Companion.createApplicationEnvironment), because
classes there can only be loaded with the special "jrt" file system, not
as .class files in .jar files

 #KT-17801 Fixed

build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt

@@ -21,7 +21,10 @@ package org.jetbrains.kotlin.incremental
 
 import com.intellij.openapi.util.io.FileUtil
 import com.intellij.openapi.util.text.StringUtil
-import org.jetbrains.kotlin.build.*
+import org.jetbrains.kotlin.build.GeneratedFile
+import org.jetbrains.kotlin.build.GeneratedJvmClass
+import org.jetbrains.kotlin.build.JvmSourceRoot
+import org.jetbrains.kotlin.build.isModuleMappingFile
 import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl
 import org.jetbrains.kotlin.config.IncrementalCompilation
 import org.jetbrains.kotlin.config.Services
@@ -50,6 +53,7 @@ fun makeModuleFile(name: String, isTest: Boolean, outputDir: File, sourcesToComp
             sourcesToCompile,
             javaSourceRoots.map { JvmSourceRoot(it) },
             classpath,
+            null,
             "java-production",
             isTest,
             // this excludes the output directories from the class path, to be removed for true incremental compilation

build-common/src/org/jetbrains/kotlin/modules/KotlinModuleXmlBuilder.kt

@@ -39,6 +39,7 @@ class KotlinModuleXmlBuilder {
             sourceFiles: Iterable<File>,
             javaSourceRoots: Iterable<JvmSourceRoot>,
             classpathRoots: Iterable<File>,
+            modularJdkRoot: File?,
             targetTypeId: String,
             isTests: Boolean,
             directoriesToFilterOut: Set<File>,
@@ -64,6 +65,10 @@ class KotlinModuleXmlBuilder {
         processJavaSourceRoots(javaSourceRoots)
         processClasspath(classpathRoots, directoriesToFilterOut)
 
+        if (modularJdkRoot != null) {
+            p.println("<", MODULAR_JDK_ROOT, " ", PATH, "=\"", getEscapedPath(modularJdkRoot), "\"/>")
+        }
+
         closeTag(p, MODULE)
         return this
     }

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/modules/ModuleBuilder.kt

@@ -29,13 +29,14 @@ class ModuleBuilder(
     private val classpathRoots = ArrayList<String>()
     private val javaSourceRoots = ArrayList<JavaRootPath>()
     private val friendDirs = ArrayList<String>()
+    override var modularJdkRoot: String? = null
 
-    fun addSourceFiles(pattern: String) {
+    fun addSourceFiles(path: String) {
-        sourceFiles.add(pattern)
+        sourceFiles.add(path)
     }
 
-    fun addClasspathEntry(name: String) {
+    fun addClasspathEntry(path: String) {
-        classpathRoots.add(name)
+        classpathRoots.add(path)
     }
 
     fun addJavaSourceRoot(rootPath: JavaRootPath) {

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/modules/ModuleXmlParser.java

@@ -52,6 +52,7 @@ public class ModuleXmlParser {
     public static final String JAVA_SOURCE_PACKAGE_PREFIX = "packagePrefix";
     public static final String PATH = "path";
     public static final String CLASSPATH = "classpath";
+    public static final String MODULAR_JDK_ROOT = "modularJdkRoot";
 
     @NotNull
     public static ModuleScriptData parseModuleScript(
@@ -172,6 +173,10 @@ public class ModuleXmlParser {
                 String packagePrefix = getNullableAttribute(attributes, JAVA_SOURCE_PACKAGE_PREFIX);
                 moduleBuilder.addJavaSourceRoot(new JavaRootPath(path, packagePrefix));
             }
+            else if (MODULAR_JDK_ROOT.equalsIgnoreCase(qName)) {
+                String path = getAttribute(attributes, PATH, qName);
+                moduleBuilder.setModularJdkRoot(path);
+            }
             else {
                 throw createError(qName);
             }

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinToJVMBytecodeCompiler.kt

@@ -204,6 +204,16 @@ object KotlinToJVMBytecodeCompiler {
             }
         }
 
+        for (module in chunk) {
+            val modularJdkRoot = module.modularJdkRoot
+            if (modularJdkRoot != null) {
+                // We use the SDK of the first module in the chunk, which is not always correct because some other module in the chunk
+                // might depend on a different SDK
+                configuration.put(JVMConfigurationKeys.JDK_HOME, File(modularJdkRoot))
+                break
+            }
+        }
+
         configuration.addAll(JVMConfigurationKeys.MODULES, chunk)
     }
 

compiler/tests-common/org/jetbrains/kotlin/test/KotlinTestUtils.java

@@ -497,12 +497,9 @@ public class KotlinTestUtils {
             JvmContentRootsKt.addJvmClasspathRoots(configuration, PathUtil.getJdkClassesRootsFromJre(getJreHome(jdk6)));
         }
         else if (jdkKind == TestJdkKind.FULL_JDK_9) {
-            String jdk9 = System.getenv("JDK_19");
+            File home = getJre9HomeIfPossible();
-            if (jdk9 != null) {
+            if (home != null) {
-                configuration.put(JVMConfigurationKeys.JDK_HOME, new File(jdk9));
+                configuration.put(JVMConfigurationKeys.JDK_HOME, home);
-            }
-            else {
-                System.err.println("Environment variable JDK_19 is not set, the test will be skipped");
             }
         }
         else {
@@ -527,6 +524,17 @@ public class KotlinTestUtils {
         return configuration;
     }
 
+    @Nullable
+    public static File getJre9HomeIfPossible() {
+        String jdk9 = System.getenv("JDK_19");
+        if (jdk9 == null) {
+            // TODO: replace this with a failure as soon as Java 9 is installed on all TeamCity agents
+            System.err.println("Environment variable JDK_19 is not set, the test will be skipped");
+            return null;
+        }
+        return new File(jdk9);
+    }
+
     @NotNull
     private static String getJreHome(@NotNull String jdkHome) {
         File jre = new File(jdkHome, "jre");

compiler/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsWithJdk9Test.java

@@ -18,6 +18,7 @@ package org.jetbrains.kotlin.checkers;
 
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.kotlin.test.ConfigurationKind;
+import org.jetbrains.kotlin.test.KotlinTestUtils;
 import org.jetbrains.kotlin.test.TestJdkKind;
 
 import java.io.File;
@@ -43,8 +44,8 @@ public abstract class AbstractDiagnosticsWithJdk9Test extends AbstractDiagnostic
             @NotNull Map<String, ModuleAndDependencies> modules,
             @NotNull List<TestFile> testFiles
     ) {
-        if (System.getenv("JDK_19") == null) {
+        if (KotlinTestUtils.getJre9HomeIfPossible() == null) {
-            // Skip this test if no environment variable JDK_19 is set up
+            // Skip this test if no Java 9 is found
             return;
         }
         super.doMultiFileTest(file, modules, testFiles);

compiler/util/src/org/jetbrains/kotlin/modules/Module.kt

@@ -30,6 +30,8 @@ interface Module {
     fun getClasspathRoots(): List<String>
 
     fun getJavaSourceRoots(): List<JavaRootPath>
+
+    val modularJdkRoot: String?
 }
 
 data class JavaRootPath(val path: String, val packagePrefix: String? = null)

idea/testData/modules.xml/modularJdkRoot.xml

@@ -0,0 +1,8 @@
+<modules>
+    <!-- Module script for production -->
+    <module name="name" type="java-production" outputDir="output">
+        <!-- Java source roots -->
+        <!-- Classpath -->
+        <modularJdkRoot path="/path/to/modular/jdk"/>
+    </module>
+</modules>

jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/KotlinJpsBuildTest.kt

@@ -21,10 +21,12 @@ import com.intellij.openapi.util.Condition
 import com.intellij.openapi.util.io.FileUtil
 import com.intellij.openapi.util.io.FileUtil.toSystemIndependentName
 import com.intellij.openapi.util.io.FileUtilRt
+import com.intellij.openapi.vfs.StandardFileSystems
 import com.intellij.testFramework.LightVirtualFile
 import com.intellij.testFramework.UsefulTestCase
 import com.intellij.util.ArrayUtil
 import com.intellij.util.containers.ContainerUtil
+import com.intellij.util.io.URLUtil
 import com.intellij.util.io.ZipUtil
 import org.jetbrains.jps.ModuleChunk
 import org.jetbrains.jps.api.CanceledStatus
@@ -45,6 +47,8 @@ import org.jetbrains.jps.model.JpsModuleRootModificationUtil
 import org.jetbrains.jps.model.java.JavaSourceRootType
 import org.jetbrains.jps.model.java.JpsJavaDependencyScope
 import org.jetbrains.jps.model.java.JpsJavaExtensionService
+import org.jetbrains.jps.model.java.JpsJavaSdkType
+import org.jetbrains.jps.model.library.JpsOrderRootType
 import org.jetbrains.jps.model.module.JpsModule
 import org.jetbrains.jps.util.JpsPathUtil
 import org.jetbrains.kotlin.codegen.AsmUtil
@@ -899,6 +903,18 @@ class KotlinJpsBuildTest : AbstractKotlinJpsBuildTestCase() {
         KotlinTestUtils.assertEqualsToFile(expectedFile, actual.toString())
     }
 
+    fun testJre9() {
+        val path = KotlinTestUtils.getJre9HomeIfPossible()?.absolutePath ?: return
+
+        val jdk = myModel.global.addSdk(JDK_NAME, path, "9", JpsJavaSdkType.INSTANCE)
+        jdk.addRoot(StandardFileSystems.JRT_PROTOCOL_PREFIX + path + URLUtil.JAR_SEPARATOR + "java.base", JpsOrderRootType.COMPILED)
+
+        loadProject(workDir.absolutePath + File.separator + PROJECT_NAME + ".ipr")
+        addKotlinRuntimeDependency()
+
+        buildAllModules().assertSuccessful()
+    }
+
     private fun BuildResult.checkErrors() {
         val actualErrors = getMessages(BuildMessage.Kind.ERROR)
                 .map { it as CompilerMessage }

jps-plugin/jps-tests/test/org/jetbrains/kotlin/jvm/compiler/ClasspathOrderTest.kt

@@ -46,6 +46,7 @@ class ClasspathOrderTest : TestCaseWithTmpdir() {
                 listOf(sourceDir),
                 listOf(JvmSourceRoot(sourceDir)),
                 listOf(PathUtil.getKotlinPathsForDistDirectory().runtimePath),
+                null,
                 JavaModuleBuildTargetType.PRODUCTION.typeId,
                 JavaModuleBuildTargetType.PRODUCTION.isTests,
                 setOf(),

jps-plugin/jps-tests/test/org/jetbrains/kotlin/modules/KotlinModuleXmlGeneratorTest.java

@@ -33,10 +33,11 @@ public class KotlinModuleXmlGeneratorTest extends TestCase {
                 Arrays.asList(new File("s1"), new File("s2")),
                 Collections.singletonList(new JvmSourceRoot(new File("java"), null)),
                 Arrays.asList(new File("cp1"), new File("cp2")),
+                null,
                 JavaModuleBuildTargetType.PRODUCTION.getTypeId(),
                 JavaModuleBuildTargetType.PRODUCTION.isTests(),
-                Collections.<File>emptySet(),
+                Collections.emptySet(),
-                Collections.<File>emptyList()
+                Collections.emptyList()
         ).asText().toString();
         KotlinTestUtils.assertEqualsToFile(new File("idea/testData/modules.xml/basic.xml"), actual);
     }
@@ -46,12 +47,13 @@ public class KotlinModuleXmlGeneratorTest extends TestCase {
                 "name",
                 "output",
                 Arrays.asList(new File("s1"), new File("s2")),
-                Collections.<JvmSourceRoot>emptyList(),
+                Collections.emptyList(),
                 Arrays.asList(new File("cp1"), new File("cp2")),
+                null,
                 JavaModuleBuildTargetType.PRODUCTION.getTypeId(),
                 JavaModuleBuildTargetType.PRODUCTION.isTests(),
                 Collections.singleton(new File("cp1")),
-                Collections.<File>emptyList()
+                Collections.emptyList()
         ).asText().toString();
         KotlinTestUtils.assertEqualsToFile(new File("idea/testData/modules.xml/filtered.xml"), actual);
     }
@@ -62,25 +64,43 @@ public class KotlinModuleXmlGeneratorTest extends TestCase {
                 "name",
                 "output",
                 Arrays.asList(new File("s1"), new File("s2")),
-                Collections.<JvmSourceRoot>emptyList(),
+                Collections.emptyList(),
                 Arrays.asList(new File("cp1"), new File("cp2")),
+                null,
                 JavaModuleBuildTargetType.PRODUCTION.getTypeId(),
                 JavaModuleBuildTargetType.PRODUCTION.isTests(),
                 Collections.singleton(new File("cp1")),
-                Collections.<File>emptyList()
+                Collections.emptyList()
         );
         builder.addModule(
                 "name2",
                 "output2",
                 Arrays.asList(new File("s12"), new File("s22")),
-                Collections.<JvmSourceRoot>emptyList(),
+                Collections.emptyList(),
                 Arrays.asList(new File("cp12"), new File("cp22")),
+                null,
                 JavaModuleBuildTargetType.TEST.getTypeId(),
                 JavaModuleBuildTargetType.TEST.isTests(),
                 Collections.singleton(new File("cp12")),
-                Collections.<File>emptyList()
+                Collections.emptyList()
         );
         String actual = builder.asText().toString();
         KotlinTestUtils.assertEqualsToFile(new File("idea/testData/modules.xml/multiple.xml"), actual);
     }
+
+    public void testModularJdkRoot() throws Exception {
+        String actual = new KotlinModuleXmlBuilder().addModule(
+                "name",
+                "output",
+                Collections.emptyList(),
+                Collections.emptyList(),
+                Collections.emptyList(),
+                new File("/path/to/modular/jdk"),
+                JavaModuleBuildTargetType.PRODUCTION.getTypeId(),
+                JavaModuleBuildTargetType.PRODUCTION.isTests(),
+                Collections.emptySet(),
+                Collections.emptyList()
+        ).asText().toString();
+        KotlinTestUtils.assertEqualsToFile(new File("idea/testData/modules.xml/modularJdkRoot.xml"), actual);
+    }
 }

jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilderModuleScriptGenerator.kt

@@ -18,9 +18,11 @@ package org.jetbrains.kotlin.jps.build
 
 import com.intellij.openapi.util.io.FileUtil
 import com.intellij.openapi.util.text.StringUtil
+import com.intellij.openapi.vfs.StandardFileSystems
 import com.intellij.util.SmartList
 import com.intellij.util.containers.ContainerUtil
 import com.intellij.util.containers.MultiMap
+import com.intellij.util.io.URLUtil
 import org.jetbrains.jps.ModuleChunk
 import org.jetbrains.jps.builders.java.JavaModuleBuildTargetType
 import org.jetbrains.jps.incremental.CompileContext
@@ -28,6 +30,7 @@ import org.jetbrains.jps.incremental.ModuleBuildTarget
 import org.jetbrains.jps.incremental.ProjectBuildException
 import org.jetbrains.jps.model.java.JpsJavaExtensionService
 import org.jetbrains.jps.model.module.JpsModule
+import org.jetbrains.jps.model.module.JpsSdkDependency
 import org.jetbrains.kotlin.build.JvmSourceRoot
 import org.jetbrains.kotlin.config.IncrementalCompilation
 import org.jetbrains.kotlin.jps.build.JpsUtils.getAllDependencies
@@ -113,6 +116,7 @@ object KotlinBuilderModuleScriptGenerator {
                     moduleSources,
                     findSourceRoots(context, target),
                     findClassPathRoots(target),
+                    findModularJdkRoot(target),
                     targetId.type,
                     (targetType as JavaModuleBuildTargetType).isTests,
                     // this excludes the output directories from the class path, to be removed for true incremental compilation
@@ -162,6 +166,18 @@ object KotlinBuilderModuleScriptGenerator {
         }
     }
 
+    private fun findModularJdkRoot(target: ModuleBuildTarget): File? {
+        // List of paths to JRE modules in the following format:
+        // jrt:///Library/Java/JavaVirtualMachines/jdk-9.jdk/Contents/Home!/java.base
+        val urls = JpsJavaExtensionService.dependencies(target.module)
+                .satisfying { dependency -> dependency is JpsSdkDependency }
+                .classes().urls
+
+        val url = urls.firstOrNull { it.startsWith(StandardFileSystems.JRT_PROTOCOL_PREFIX) } ?: return null
+
+        return File(url.substringAfter(StandardFileSystems.JRT_PROTOCOL_PREFIX).substringBeforeLast(URLUtil.JAR_SEPARATOR))
+    }
+
     private fun findSourceRoots(context: CompileContext, target: ModuleBuildTarget): List<JvmSourceRoot> {
         val roots = context.projectDescriptor.buildRootIndex.getTargetRoots(target, context)
         val result = ContainerUtil.newArrayList<JvmSourceRoot>()

jps-plugin/testData/general/Jre9/kotlinProject.iml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
+    </content>
+    <orderEntry type="jdk" jdkName="IDEA_JDK" jdkType="JavaSDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="module" module-name="kotlinProject" />
+  </component>
+</module>

jps-plugin/testData/general/Jre9/kotlinProject.ipr

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CompilerConfiguration">
+    <option name="DEFAULT_COMPILER" value="Javac" />
+  </component>
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/kotlinProject.iml" filepath="$PROJECT_DIR$/kotlinProject.iml" />
+    </modules>
+  </component>
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_1_6" assert-keyword="true" jdk-15="true" project-jdk-name="IDEA_JDK" project-jdk-type="JavaSDK">
+    <output url="file://$PROJECT_DIR$/out" />
+  </component>
+</project>

jps-plugin/testData/general/Jre9/src/test.kt

@@ -0,0 +1,14 @@
+import java.nio.file.Path
+import java.util.stream.IntStream
+import java.util.function.Consumer
+
+class Foo : Consumer<Path> {
+    override fun accept(path: Path) {}
+}
+
+fun foo(s: IntStream): List<String> {
+    println(s.boxed())
+    Any()
+    if (s.count() == 0L) throw Exception()
+    return object : ArrayList<String>() {}
+}