Support single Java source files in kotlinc arguments

E.g. "kotlinc foo.kt test/Bar.java" will compile foo.kt, and
declarations from Bar.java will be accessible to Kotlin code in foo.kt.

The change in AbstractTopLevelMembersInvocationTest is needed because an
incorrect configuration was created in that test where a library jar was
also a Java source root (the compiler is never configured this way in
production), which led to an exception in
JavaCoreProjectEnvironment#addSourcesToClasspath

 #KT-17697 Fixed

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/K2JVMCompiler.kt

@@ -16,6 +16,7 @@
 
 package org.jetbrains.kotlin.cli.jvm
 
+import com.intellij.ide.highlighter.JavaFileType
 import com.intellij.openapi.Disposable
 import org.jetbrains.kotlin.cli.common.CLICompiler
 import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
@@ -94,11 +95,16 @@ class K2JVMCompiler : CLICompiler<K2JVMCompilerArguments>() {
         }
         else if (arguments.module == null) {
             for (arg in arguments.freeArgs) {
-                configuration.addKotlinSourceRoot(arg)
                 val file = File(arg)
-                if (file.isDirectory) {
+                if (file.extension == JavaFileType.DEFAULT_EXTENSION) {
                     configuration.addJavaSourceRoot(file)
                 }
+                else {
+                    configuration.addKotlinSourceRoot(arg)
+                    if (file.isDirectory) {
+                        configuration.addJavaSourceRoot(file)
+                    }
+                }
             }
         }
 

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCliJavaFileManagerImpl.kt

@@ -26,6 +26,7 @@ import com.intellij.psi.search.GlobalSearchScope
 import gnu.trove.THashMap
 import org.jetbrains.kotlin.cli.jvm.index.JavaRoot
 import org.jetbrains.kotlin.cli.jvm.index.JvmDependenciesIndex
+import org.jetbrains.kotlin.cli.jvm.index.SingleJavaFileRootsIndex
 import org.jetbrains.kotlin.load.java.structure.JavaClass
 import org.jetbrains.kotlin.load.java.structure.impl.JavaClassImpl
 import org.jetbrains.kotlin.load.java.structure.impl.classFiles.BinaryClassSignatureParser
@@ -36,21 +37,23 @@ import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.resolve.jvm.KotlinCliJavaFileManager
 import org.jetbrains.kotlin.util.PerformanceCounter
+import org.jetbrains.kotlin.utils.addIfNotNull
 import java.util.*
-import kotlin.properties.Delegates
 
 // TODO: do not inherit from CoreJavaFileManager to avoid accidental usage of its methods which do not use caches/indices
 // Currently, the only relevant usage of this class as CoreJavaFileManager is at CoreJavaDirectoryService.getPackage,
 // which is indirectly invoked from PsiPackage.getSubPackages
 class KotlinCliJavaFileManagerImpl(private val myPsiManager: PsiManager) : CoreJavaFileManager(myPsiManager), KotlinCliJavaFileManager {
     private val perfCounter = PerformanceCounter.create("Find Java class")
-    private var index: JvmDependenciesIndex by Delegates.notNull()
+    private lateinit var index: JvmDependenciesIndex
+    private lateinit var singleJavaFileRootsIndex: SingleJavaFileRootsIndex
     private val topLevelClassesCache: MutableMap<FqName, VirtualFile?> = THashMap()
     private val allScope = GlobalSearchScope.allScope(myPsiManager.project)
     private var useFastClassFilesReading = false
 
-    fun initialize(packagesCache: JvmDependenciesIndex, useFastClassFilesReading: Boolean) {
-        this.index = packagesCache
+    fun initialize(index: JvmDependenciesIndex, singleJavaFileRootsIndex: SingleJavaFileRootsIndex, useFastClassFilesReading: Boolean) {
+        this.index = index
+        this.singleJavaFileRootsIndex = singleJavaFileRootsIndex
         this.useFastClassFilesReading = useFastClassFilesReading
     }
 
@@ -64,6 +67,7 @@ class KotlinCliJavaFileManagerImpl(private val myPsiManager: PsiManager) : CoreJ
             index.findClass(classId) { dir, type ->
                 findVirtualFileGivenPackage(dir, relativeClassName, type)
             }
+            ?: singleJavaFileRootsIndex.findJavaSourceClass(classId)
         }?.takeIf { it in searchScope }
     }
 
@@ -153,6 +157,13 @@ class KotlinCliJavaFileManagerImpl(private val myPsiManager: PsiManager) : CoreJ
                 // traverse all
                 true
             }
+
+            result.addIfNotNull(
+                    singleJavaFileRootsIndex.findJavaSourceClass(classId)
+                            ?.takeIf { it in scope }
+                            ?.findPsiClassInVirtualFile(relativeClassName)
+            )
+
             if (result.isNotEmpty()) {
                 return@time result.toTypedArray()
             }
@@ -169,10 +180,10 @@ class KotlinCliJavaFileManagerImpl(private val myPsiManager: PsiManager) : CoreJ
             //abort on first found
             false
         }
-        if (found) {
-            return PsiPackageImpl(myPsiManager, packageName)
+        if (!found) {
+            found = singleJavaFileRootsIndex.findJavaSourceClasses(packageFqName).isNotEmpty()
         }
-        return null
+        return if (found) PsiPackageImpl(myPsiManager, packageName) else null
     }
 
     private fun findVirtualFileGivenPackage(
@@ -216,6 +227,11 @@ class KotlinCliJavaFileManagerImpl(private val myPsiManager: PsiManager) : CoreJ
             true
         })
 
+        for (classId in singleJavaFileRootsIndex.findJavaSourceClasses(packageFqName)) {
+            assert(!classId.isNestedClass) { "ClassId of a single .java source class should not be nested: $classId" }
+            result.add(classId.shortClassName.asString())
+        }
+
         return result
     }
 

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt

@@ -72,10 +72,7 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.*
 import org.jetbrains.kotlin.cli.common.toBooleanLenient
 import org.jetbrains.kotlin.cli.jvm.JvmRuntimeVersionsConsistencyChecker
 import org.jetbrains.kotlin.cli.jvm.config.*
-import org.jetbrains.kotlin.cli.jvm.index.JavaRoot
-import org.jetbrains.kotlin.cli.jvm.index.JvmDependenciesDynamicCompoundIndex
-import org.jetbrains.kotlin.cli.jvm.index.JvmDependenciesIndex
-import org.jetbrains.kotlin.cli.jvm.index.JvmUpdateableDependenciesIndexFactory
+import org.jetbrains.kotlin.cli.jvm.index.*
 import org.jetbrains.kotlin.cli.jvm.modules.CoreJrtFileSystem
 import org.jetbrains.kotlin.cli.jvm.modules.JavaModuleInfo
 import org.jetbrains.kotlin.cli.jvm.modules.ModuleGraph
@@ -207,11 +204,16 @@ class KotlinCoreEnvironment private constructor(
         // REPL and kapt2 update classpath dynamically
         val indexFactory = JvmUpdateableDependenciesIndexFactory()
 
-        rootsIndex = indexFactory.makeIndexFor(initialRoots)
+        val (roots, singleJavaFileRoots) =
+                initialRoots.partition { (file) -> file.isDirectory || file.extension != JavaFileType.DEFAULT_EXTENSION }
+        rootsIndex = indexFactory.makeIndexFor(roots)
         updateClasspathFromRootsIndex(rootsIndex)
 
-        (ServiceManager.getService(project, CoreJavaFileManager::class.java) as KotlinCliJavaFileManagerImpl)
-                .initialize(rootsIndex, configuration.getBoolean(JVMConfigurationKeys.USE_FAST_CLASS_FILES_READING))
+        (ServiceManager.getService(project, CoreJavaFileManager::class.java) as KotlinCliJavaFileManagerImpl).initialize(
+                rootsIndex,
+                SingleJavaFileRootsIndex(singleJavaFileRoots),
+                configuration.getBoolean(JVMConfigurationKeys.USE_FAST_CLASS_FILES_READING)
+        )
 
         val finderFactory = CliVirtualFileFinderFactory(rootsIndex)
         project.registerService(MetadataFinderFactory::class.java, finderFactory)
@@ -230,7 +232,7 @@ class KotlinCoreEnvironment private constructor(
 
     private val allJavaFiles: List<File>
         get() = configuration.javaSourceRoots
-                .mapNotNull(this::findLocalDirectory)
+                .mapNotNull(this::findLocalFile)
                 .flatMap { it.javaFiles }
                 .map { File(it.canonicalPath) }
 
@@ -337,13 +339,9 @@ class KotlinCoreEnvironment private constructor(
     fun tryUpdateClasspath(files: Iterable<File>): List<File>? = updateClasspath(files.map(::JvmClasspathRoot))
 
     fun contentRootToVirtualFile(root: JvmContentRoot): VirtualFile? {
-        when (root) {
-            is JvmClasspathRoot -> {
-                return if (root.file.isFile) findJarRoot(root) else findLocalDirectory(root)
-            }
-            is JavaSourceRoot -> {
-                return if (root.file.isDirectory) findLocalDirectory(root) else null
-            }
+        return when (root) {
+            is JvmClasspathRoot -> if (root.file.isFile) findJarRoot(root) else findLocalFile(root)
+            is JavaSourceRoot -> findLocalFile(root)
             else -> throw IllegalStateException("Unexpected root: $root")
         }
     }
@@ -352,9 +350,9 @@ class KotlinCoreEnvironment private constructor(
 
     fun findJarFile(path: String) = applicationEnvironment.jarFileSystem.findFileByPath(path)
 
-    private fun findLocalDirectory(root: JvmContentRoot): VirtualFile? {
+    private fun findLocalFile(root: JvmContentRoot): VirtualFile? {
         val path = root.file
-        val localFile = findLocalDirectory(path.absolutePath)
+        val localFile = findLocalFile(path.absolutePath)
         if (localFile == null) {
             report(STRONG_WARNING, "Classpath entry points to a non-existent location: $path")
             return null
@@ -362,9 +360,6 @@ class KotlinCoreEnvironment private constructor(
         return localFile
     }
 
-    internal fun findLocalDirectory(absolutePath: String): VirtualFile? =
-            applicationEnvironment.localFileSystem.findFileByPath(absolutePath)
-
     private fun findJarRoot(root: JvmClasspathRoot): VirtualFile? =
             applicationEnvironment.jarFileSystem.findFileByPath("${root.file}${URLUtil.JAR_SEPARATOR}")
 

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinToJVMBytecodeCompiler.kt

@@ -362,7 +362,7 @@ object KotlinToJVMBytecodeCompiler {
             override fun analyze(): AnalysisResult {
                 val project = environment.project
                 val moduleOutputs = environment.configuration.get(JVMConfigurationKeys.MODULES)?.mapNotNull { module ->
-                    environment.findLocalDirectory(module.getOutputDirectory())
+                    environment.findLocalFile(module.getOutputDirectory())
                 }.orEmpty()
                 val sourcesOnly = TopDownAnalyzerFacadeForJVM.newModuleSearchScope(project, environment.getSourceFiles())
                 // To support partial and incremental compilation, we add the scope which contains binaries from output directories

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/index/SingleJavaFileRootsIndex.kt

@@ -0,0 +1,115 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.cli.jvm.index
+
+import com.intellij.lang.java.lexer.JavaLexer
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.pom.java.LanguageLevel
+import com.intellij.psi.impl.source.tree.ElementType
+import com.intellij.psi.tree.IElementType
+import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.name.Name
+
+class SingleJavaFileRootsIndex(private val roots: List<JavaRoot>) {
+    init {
+        for ((file) in roots) {
+            assert(!file.isDirectory) { "Should not be a directory: $file" }
+        }
+    }
+
+    private val classIdsInRoots = ArrayList<List<ClassId>>(roots.size)
+
+    fun findJavaSourceClass(classId: ClassId): VirtualFile? =
+            roots.indices
+                    .find { index -> classId in getClassIdsForRootAt(index) }
+                    ?.let { index -> roots[index].file }
+
+    fun findJavaSourceClasses(packageFqName: FqName): List<ClassId> =
+            roots.indices.flatMap(this::getClassIdsForRootAt).filter { root -> root.packageFqName == packageFqName }
+
+    private fun getClassIdsForRootAt(index: Int): List<ClassId> {
+        for (i in classIdsInRoots.size..index) {
+            classIdsInRoots.add(JavaSourceClassIdReader(roots[i].file).readClassIds())
+        }
+        return classIdsInRoots[index]
+    }
+
+    /**
+     * Given a .java file, [readClassIds] uses lexer to determine which classes are declared in that file
+     */
+    private class JavaSourceClassIdReader(file: VirtualFile) {
+        private val lexer = JavaLexer(LanguageLevel.JDK_1_9).apply {
+            start(String(file.contentsToByteArray()))
+        }
+        private var braceBalance = 0
+
+        private fun at(type: IElementType): Boolean = lexer.tokenType == type
+
+        private fun end(): Boolean = lexer.tokenType == null
+
+        private fun advance() {
+            when {
+                at(ElementType.LBRACE) -> braceBalance++
+                at(ElementType.RBRACE) -> braceBalance--
+            }
+            lexer.advance()
+        }
+
+        private fun tokenText(): String = lexer.tokenText
+
+        private fun atClass(): Boolean =
+                braceBalance == 0 && lexer.tokenType in CLASS_KEYWORDS
+
+        fun readClassIds(): List<ClassId> {
+            var packageFqName = FqName.ROOT
+            while (!end() && !at(ElementType.PACKAGE_KEYWORD) && !atClass()) {
+                advance()
+            }
+            if (at(ElementType.PACKAGE_KEYWORD)) {
+                val packageName = StringBuilder()
+                while (!end() && !at(ElementType.SEMICOLON)) {
+                    if (at(ElementType.IDENTIFIER) || at(ElementType.DOT)) {
+                        packageName.append(tokenText())
+                    }
+                    advance()
+                }
+                packageFqName = FqName(packageName.toString())
+            }
+
+            val result = ArrayList<ClassId>(1)
+
+            while (true) {
+                while (!end() && !atClass()) {
+                    advance()
+                }
+                if (end()) break
+                while (!end() && !at(ElementType.IDENTIFIER)) {
+                    advance()
+                }
+                if (end()) break
+                result.add(ClassId(packageFqName, Name.identifier(tokenText())))
+            }
+
+            return result
+        }
+
+        companion object {
+            private val CLASS_KEYWORDS = setOf(ElementType.CLASS_KEYWORD, ElementType.INTERFACE_KEYWORD, ElementType.ENUM_KEYWORD)
+        }
+    }
+}

compiler/testData/cli/jvm/singleJavaFileRoots.args

@@ -0,0 +1,11 @@
+$TESTDATA_DIR$/singleJavaFileRoots/test.kt
+$TESTDATA_DIR$/singleJavaFileRoots/DefaultPackage.java
+$TESTDATA_DIR$/singleJavaFileRoots/lib/A.java
+$TESTDATA_DIR$/singleJavaFileRoots/lib/B.java
+$TESTDATA_DIR$/singleJavaFileRoots/lib/ext/SeveralClasses.java
+$TESTDATA_DIR$/singleJavaFileRoots/lib/classKinds/ClassClass.java
+$TESTDATA_DIR$/singleJavaFileRoots/lib/classKinds/InterfaceClass.java
+$TESTDATA_DIR$/singleJavaFileRoots/lib/classKinds/EnumClass.java
+$TESTDATA_DIR$/singleJavaFileRoots/lib/classKinds/AnnotationClass.java
+-d
+$TEMP_DIR$

compiler/testData/cli/jvm/singleJavaFileRoots.out

@@ -0,0 +1,13 @@
+compiler/testData/cli/jvm/singleJavaFileRoots/test.kt:7:9: error: cannot access class 'C'. Check your module classpath for missing or conflicting dependencies
+    B().c()
+        ^
+compiler/testData/cli/jvm/singleJavaFileRoots/test.kt:8:5: error: unresolved reference: C
+    C().a()
+    ^
+compiler/testData/cli/jvm/singleJavaFileRoots/test.kt:12:5: error: cannot access '<init>': it is public/*package*/ in 'PackageLocal1'
+    PackageLocal1()
+    ^
+compiler/testData/cli/jvm/singleJavaFileRoots/test.kt:13:5: error: cannot access '<init>': it is public/*package*/ in 'PackageLocal2'
+    PackageLocal2()
+    ^
+COMPILATION_ERROR

compiler/testData/cli/jvm/singleJavaFileRoots/DefaultPackage.java

@@ -0,0 +1 @@
+public class DefaultPackage {}

compiler/testData/cli/jvm/singleJavaFileRoots/lib/A.java

@@ -0,0 +1,5 @@
+package lib;
+
+public class A {
+    public B b() {}
+}

compiler/testData/cli/jvm/singleJavaFileRoots/lib/B.java

@@ -0,0 +1,5 @@
+package lib;
+
+public class B {
+    public C c() {}
+}

compiler/testData/cli/jvm/singleJavaFileRoots/lib/C.java

@@ -0,0 +1,5 @@
+package lib;
+
+public class C {
+    public A a() {}
+}

compiler/testData/cli/jvm/singleJavaFileRoots/lib/classKinds/AnnotationClass.java

@@ -0,0 +1,5 @@
+package lib.classKinds;
+
+public @interface AnnotationClass {
+    EnumClass value();
+}

compiler/testData/cli/jvm/singleJavaFileRoots/lib/classKinds/ClassClass.java

@@ -0,0 +1,3 @@
+package lib.classKinds;
+
+public class ClassClass {}

compiler/testData/cli/jvm/singleJavaFileRoots/lib/classKinds/EnumClass.java

@@ -0,0 +1,3 @@
+package lib.classKinds;
+
+public enum EnumClass { ENTRY }

compiler/testData/cli/jvm/singleJavaFileRoots/lib/classKinds/InterfaceClass.java

@@ -0,0 +1,3 @@
+package lib.classKinds;
+
+public interface InterfaceClass {}

compiler/testData/cli/jvm/singleJavaFileRoots/lib/ext/SeveralClasses.java

@@ -0,0 +1,6 @@
+package lib. /* comment */ ext;
+
+class
+PackageLocal1 {}
+
+class /* comment */ PackageLocal2 {}

compiler/testData/cli/jvm/singleJavaFileRoots/test.kt

@@ -0,0 +1,21 @@
+import lib.*
+import lib.ext.*
+import lib.classKinds.*
+
+fun main(args: Array<String>) {
+    A().b()
+    B().c()
+    C().a()
+
+    DefaultPackage()
+
+    PackageLocal1()
+    PackageLocal2()
+}
+
+@AnnotationClass(EnumClass.ENTRY)
+fun classKinds(
+        c: ClassClass,
+        i: InterfaceClass,
+        e: EnumClass
+) {}

compiler/tests-common/org/jetbrains/kotlin/codegen/AbstractTopLevelMembersInvocationTest.java

@@ -61,7 +61,7 @@ public abstract class AbstractTopLevelMembersInvocationTest extends AbstractByte
                 getTestRootDisposable(),
                 KotlinTestUtils.newConfiguration(
                         ConfigurationKind.JDK_ONLY, TestJdkKind.MOCK_JDK,
-                        CollectionsKt.plus(classPath, KotlinTestUtils.getAnnotationsJar()), classPath
+                        CollectionsKt.plus(classPath, KotlinTestUtils.getAnnotationsJar()), Collections.emptyList()
                 ),
                 EnvironmentConfigFiles.JVM_CONFIG_FILES);
 

compiler/tests/org/jetbrains/kotlin/cli/CliTestGenerated.java

@@ -332,6 +332,12 @@ public class CliTestGenerated extends AbstractCliTest {
             doJvmTest(fileName);
         }
 
+        @TestMetadata("singleJavaFileRoots.args")
+        public void testSingleJavaFileRoots() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/cli/jvm/singleJavaFileRoots.args");
+            doJvmTest(fileName);
+        }
+
         @TestMetadata("suppressAllWarningsJvm.args")
         public void testSuppressAllWarningsJvm() throws Exception {
             String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/cli/jvm/suppressAllWarningsJvm.args");

compiler/tests/org/jetbrains/kotlin/cli/jvm/KotlinCliJavaFileManagerTest.kt

@@ -26,6 +26,7 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.cli.jvm.config.JavaSourceRoot
 import org.jetbrains.kotlin.cli.jvm.index.JavaRoot
 import org.jetbrains.kotlin.cli.jvm.index.JvmDependenciesIndexImpl
+import org.jetbrains.kotlin.cli.jvm.index.SingleJavaFileRootsIndex
 import org.jetbrains.kotlin.load.java.structure.impl.JavaClassImpl
 import org.jetbrains.kotlin.load.kotlin.VirtualFileFinder
 import org.jetbrains.kotlin.name.ClassId
@@ -202,6 +203,7 @@ class KotlinCliJavaFileManagerTest : KotlinTestWithEnvironment() {
         val root = environment.contentRootToVirtualFile(JavaSourceRoot(javaFilesDir!!, null))!!
         coreJavaFileManager.initialize(
                 JvmDependenciesIndexImpl(listOf(JavaRoot(root, JavaRoot.RootType.SOURCE))),
+                SingleJavaFileRootsIndex(emptyList()),
                 useFastClassFilesReading = true
         )