From 59712d4bf86bba9d770a97e3504358aa51a8d1f8 Mon Sep 17 00:00:00 2001 From: Ilya Chernikov Date: Mon, 18 Jul 2016 20:51:24 +0200 Subject: [PATCH] Implement updatable dependencies index with usage API in the environment for using in REPL --- ...y.kt => JvmCliVirtualFileFinderFactory.kt} | 5 +- .../cli/jvm/compiler/JvmDependenciesIndex.kt | 86 +++++++++++++++++-- .../cli/jvm/compiler/KotlinCoreEnvironment.kt | 80 +++++++++++------ .../kotlin/cli/jvm/repl/ReplInterpreter.kt | 2 + .../config/CommonConfigurationKeys.java | 3 + .../cli/jvm/KotlinCliJavaFileManagerTest.kt | 2 +- 6 files changed, 139 insertions(+), 39 deletions(-) rename compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/{JvmLazyCliVirtualFileFinderFactory.kt => JvmCliVirtualFileFinderFactory.kt} (84%) diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmLazyCliVirtualFileFinderFactory.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmCliVirtualFileFinderFactory.kt similarity index 84% rename from compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmLazyCliVirtualFileFinderFactory.kt rename to compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmCliVirtualFileFinderFactory.kt index 43b0432d2d4..0ae0175dbe8 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmLazyCliVirtualFileFinderFactory.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmCliVirtualFileFinderFactory.kt @@ -20,9 +20,6 @@ import com.intellij.psi.search.GlobalSearchScope import org.jetbrains.kotlin.load.kotlin.JvmVirtualFileFinder import org.jetbrains.kotlin.load.kotlin.JvmVirtualFileFinderFactory -class JvmLazyCliVirtualFileFinderFactory(private val makeIndex: () -> JvmDependenciesIndex) : JvmVirtualFileFinderFactory { - - private val index by lazy { makeIndex() } - +class JvmCliVirtualFileFinderFactory(private val index: JvmDependenciesIndex) : JvmVirtualFileFinderFactory { override fun create(scope: GlobalSearchScope): JvmVirtualFileFinder = JvmCliVirtualFileFinder(index) } diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmDependenciesIndex.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmDependenciesIndex.kt index c8eb73e0a00..c202000789f 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmDependenciesIndex.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmDependenciesIndex.kt @@ -21,6 +21,9 @@ import com.intellij.util.containers.IntArrayList import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName import java.util.* +import java.util.concurrent.locks.ReentrantReadWriteLock +import kotlin.concurrent.read +import kotlin.concurrent.write data class JavaRoot(val file: VirtualFile, val type: JavaRoot.RootType, val prefixFqName: FqName? = null) { enum class RootType { @@ -34,10 +37,31 @@ data class JavaRoot(val file: VirtualFile, val type: JavaRoot.RootType, val pref } } +interface JvmDependenciesIndex { + + val indexedRoots: Sequence + + fun findClass( + classId: ClassId, + acceptedRootTypes: Set = JavaRoot.SourceAndBinary, + findClassGivenDirectory: (VirtualFile, JavaRoot.RootType) -> T? + ): T? + + fun traverseDirectoriesInPackage( + packageFqName: FqName, + acceptedRootTypes: Set = JavaRoot.SourceAndBinary, + continueSearch: (VirtualFile, JavaRoot.RootType) -> Boolean + ) + + fun collectKnownClassNamesInPackage( + packageFqName: FqName + ): Set +} + // speeds up finding files/classes in classpath/java source roots // NOT THREADSAFE, needs to be adapted/removed if we want compiler to be multithreaded // the main idea of this class is for each package to store roots which contains it to avoid excessive file system traversal -class JvmDependenciesIndex(_roots: List) { +class JvmDependenciesIndexImpl(_roots: List): JvmDependenciesIndex { //these fields are computed based on _roots passed to constructor which are filled in later private val roots: List by lazy { _roots.toList() } @@ -74,11 +98,12 @@ class JvmDependenciesIndex(_roots: List) { // helps improve several scenarios, LazyJavaResolverContext.findClassInJava being the most important private var lastClassSearch: Pair? = null + override val indexedRoots by lazy { roots.asSequence() } // findClassGivenDirectory MUST check whether the class with this classId exists in given package - fun findClass( + override fun findClass( classId: ClassId, - acceptedRootTypes: Set = JavaRoot.SourceAndBinary, + acceptedRootTypes: Set, findClassGivenDirectory: (VirtualFile, JavaRoot.RootType) -> T? ): T? { return search(FindClassRequest(classId, acceptedRootTypes)) { dir, rootType -> @@ -87,9 +112,9 @@ class JvmDependenciesIndex(_roots: List) { } } - fun traverseDirectoriesInPackage( + override fun traverseDirectoriesInPackage( packageFqName: FqName, - acceptedRootTypes: Set = JavaRoot.SourceAndBinary, + acceptedRootTypes: Set, continueSearch: (VirtualFile, JavaRoot.RootType) -> Boolean ) { search(TraverseRequest(packageFqName, acceptedRootTypes)) { dir, rootType -> @@ -97,7 +122,7 @@ class JvmDependenciesIndex(_roots: List) { } } - fun collectKnownClassNamesInPackage( + override fun collectKnownClassNamesInPackage( packageFqName: FqName ): Set { var result = hashSetOf() @@ -285,5 +310,54 @@ class JvmDependenciesIndex(_roots: List) { } } +class JvmDependenciesDynamicCompoundIndex() : JvmDependenciesIndex { + + private val indices = arrayListOf() + private val lock = ReentrantReadWriteLock() + + fun addIndex(index: JvmDependenciesIndex) { + lock.write { + indices.add(index) + } + } + + fun addNewIndexForRoots(roots: Iterable): JvmDependenciesIndex? = + lock.read { + val alreadyIndexed = indexedRoots.toHashSet() + val newRoots = roots.filter { !alreadyIndexed.contains(it) } + if (newRoots.isEmpty()) null + else { + val index = JvmDependenciesIndexImpl(newRoots) + addIndex(index) + index + } + } + + override val indexedRoots: Sequence get() = indices.asSequence().flatMap { it.indexedRoots } + + override fun findClass( + classId: ClassId, + acceptedRootTypes: Set, + findClassGivenDirectory: (VirtualFile, JavaRoot.RootType) -> T? + ): T? = + lock.read { + indices.asSequence().mapNotNull { it.findClass(classId, acceptedRootTypes, findClassGivenDirectory) }.firstOrNull() + } + + override fun traverseDirectoriesInPackage( + packageFqName: FqName, + acceptedRootTypes: Set, + continueSearch: (VirtualFile, JavaRoot.RootType) -> Boolean + ) { + lock.read { + indices.forEach { it.traverseDirectoriesInPackage(packageFqName, acceptedRootTypes, continueSearch) } + } + } + + override fun collectKnownClassNamesInPackage(packageFqName: FqName): Set = lock.read { + indices.fold(hashSetOf(), { s, index -> s.addAll(index.collectKnownClassNamesInPackage(packageFqName)); s }) + } +} + private fun IntArrayList.lastOrNull() = if (isEmpty) null else get(size() - 1) private val IntArrayList.indices: IntRange get() = 0..(size() - 1) \ No newline at end of file diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt index 01dfbb438e8..96d65fec661 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt @@ -52,6 +52,7 @@ import com.intellij.psi.impl.file.impl.JavaFileManager import com.intellij.psi.meta.MetaDataContributor import com.intellij.psi.stubs.BinaryFileStubBuilders import com.intellij.psi.util.JavaClassSupers +import com.intellij.util.io.URLUtil import org.jetbrains.annotations.TestOnly import org.jetbrains.kotlin.asJava.LightClassGenerationSupport import org.jetbrains.kotlin.asJava.classes.KtLightClassForFacade @@ -71,9 +72,7 @@ import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots import org.jetbrains.kotlin.codegen.extensions.ClassBuilderInterceptorExtension import org.jetbrains.kotlin.codegen.extensions.ExpressionCodegenExtension import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar -import org.jetbrains.kotlin.config.CompilerConfiguration -import org.jetbrains.kotlin.config.JVMConfigurationKeys -import org.jetbrains.kotlin.config.kotlinSourceRoots +import org.jetbrains.kotlin.config.* import org.jetbrains.kotlin.extensions.ExternalDeclarationsProvider import org.jetbrains.kotlin.extensions.StorageComponentContainerContributor import org.jetbrains.kotlin.idea.KotlinFileType @@ -108,7 +107,7 @@ class KotlinCoreEnvironment private constructor( } } private val sourceFiles = ArrayList() - private val javaRoots = ArrayList() + private val rootsIndex: JvmDependenciesIndex val configuration: CompilerConfiguration = configuration.copy().apply { setReadOnly(true) } @@ -124,10 +123,6 @@ class KotlinCoreEnvironment private constructor( registerProjectServicesForCLI(projectEnvironment) registerProjectServices(projectEnvironment) - val fileManager = ServiceManager.getService(project, CoreJavaFileManager::class.java) - val index = JvmDependenciesIndex(javaRoots) - (fileManager as KotlinCliJavaFileManagerImpl).initIndex(index) - sourceFiles.addAll(CompileEnvironmentUtil.getKtFiles(project, getSourceRootsCheckingForDuplicates(), this.configuration, { message -> report(ERROR, message) @@ -158,7 +153,20 @@ class KotlinCoreEnvironment private constructor( } } - project.registerService(JvmVirtualFileFinderFactory::class.java, JvmLazyCliVirtualFileFinderFactory({ fillClasspath(configuration); index } )) + val initialRoots = configuration.getList(JVMConfigurationKeys.CONTENT_ROOTS).classpathRoots() + val initialIndex = JvmDependenciesIndexImpl(initialRoots) + updateClasspathFromRootsIndex(initialIndex) + + // replacing index with updatable one only for REPL mode, so we can add roots to classpath then compiling + // subsequent lines in REPL + rootsIndex = if (!configuration.getBoolean(CommonConfigurationKeys.REPL_MODE)) initialIndex + else JvmDependenciesDynamicCompoundIndex().apply { + addIndex(initialIndex) + } + (ServiceManager.getService(project, CoreJavaFileManager::class.java) + as KotlinCliJavaFileManagerImpl).initIndex(rootsIndex) + + project.registerService(JvmVirtualFileFinderFactory::class.java, JvmCliVirtualFileFinderFactory(rootsIndex)) ExternalDeclarationsProvider.registerExtensionPoint(project) ExpressionCodegenExtension.registerExtensionPoint(project) @@ -189,33 +197,49 @@ class KotlinCoreEnvironment private constructor( StringUtil.getLineBreakCount(it.text) + (if (StringUtil.endsWithLineBreak(text)) 0 else 1) } - private fun fillClasspath(configuration: CompilerConfiguration) { - for (root in configuration.getList(JVMConfigurationKeys.CONTENT_ROOTS)) { - val javaRoot = root as? JvmContentRoot ?: continue - val virtualFile = contentRootToVirtualFile(javaRoot) ?: continue + private fun Iterable.classpathRoots(): List = + filterIsInstance(JvmContentRoot::class.java).mapNotNull { + javaRoot -> contentRootToVirtualFile(javaRoot)?.let { virtualFile -> - projectEnvironment.addSourcesToClasspath(virtualFile) - - val prefixPackageFqName = (javaRoot as? JavaSourceRoot)?.packagePrefix?.let { - if (isValidJavaFqName(it)) { - FqName(it) + val prefixPackageFqName = (javaRoot as? JavaSourceRoot)?.packagePrefix?.let { + if (isValidJavaFqName(it)) { + FqName(it) + } + else { + report(WARNING, "Invalid package prefix name is ignored: $it") + null + } } - else { - report(WARNING, "Invalid package prefix name is ignored: $it") - null + + val rootType = when (javaRoot) { + is JavaSourceRoot -> JavaRoot.RootType.SOURCE + is JvmClasspathRoot -> JavaRoot.RootType.BINARY + else -> throw IllegalStateException() } - } - val rootType = when (javaRoot) { - is JavaSourceRoot -> JavaRoot.RootType.SOURCE - is JvmClasspathRoot -> JavaRoot.RootType.BINARY - else -> throw IllegalStateException() + JavaRoot(virtualFile, rootType, prefixPackageFqName) } + } - javaRoots.add(JavaRoot(virtualFile, rootType, prefixPackageFqName)) + private fun updateClasspathFromRootsIndex(index: JvmDependenciesIndex) { + index.indexedRoots.forEach { + projectEnvironment.addSourcesToClasspath(it.file) } } + @Suppress("unused") // used externally + fun tryUpdateClasspath(files: Iterable): List? = + if (rootsIndex !is JvmDependenciesDynamicCompoundIndex) { + report(WARNING, "Unable to update classpath after initialization, it is only allowed in REPL") + null + } + else { + rootsIndex.addNewIndexForRoots(files.map { JvmClasspathRoot(it) }.classpathRoots())?.let { + updateClasspathFromRootsIndex(it) + it.indexedRoots.mapNotNull { File(it.file.path.substringBefore(URLUtil.JAR_SEPARATOR)) }.toList() + } ?: emptyList() + } + fun contentRootToVirtualFile(root: JvmContentRoot): VirtualFile? { when (root) { is JvmClasspathRoot -> { @@ -240,7 +264,7 @@ class KotlinCoreEnvironment private constructor( private fun findJarRoot(root: JvmClasspathRoot): VirtualFile? { val path = root.file - val jarFile = applicationEnvironment.jarFileSystem.findFileByPath("${path}!/") + val jarFile = applicationEnvironment.jarFileSystem.findFileByPath("${path}${URLUtil.JAR_SEPARATOR}") if (jarFile == null) { report(WARNING, "Classpath entry points to a file that is not a JAR archive: $path") return null diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/repl/ReplInterpreter.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/repl/ReplInterpreter.kt index 1487fe270f4..042a7a39bb8 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/repl/ReplInterpreter.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/repl/ReplInterpreter.kt @@ -33,6 +33,7 @@ import org.jetbrains.kotlin.codegen.ClassBuilderFactories import org.jetbrains.kotlin.codegen.CompilationErrorHandler import org.jetbrains.kotlin.codegen.KotlinCodegenFacade import org.jetbrains.kotlin.codegen.state.GenerationState +import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.JVMConfigurationKeys import org.jetbrains.kotlin.descriptors.ScriptDescriptor @@ -64,6 +65,7 @@ class ReplInterpreter( private val environment = run { configuration.add(JVMConfigurationKeys.SCRIPT_DEFINITIONS, REPL_LINE_AS_SCRIPT_DEFINITION) + configuration.put(CommonConfigurationKeys.REPL_MODE, true) KotlinCoreEnvironment.createForProduction(disposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES) } diff --git a/compiler/frontend/src/org/jetbrains/kotlin/config/CommonConfigurationKeys.java b/compiler/frontend/src/org/jetbrains/kotlin/config/CommonConfigurationKeys.java index f64b915adb3..6b869a8527a 100644 --- a/compiler/frontend/src/org/jetbrains/kotlin/config/CommonConfigurationKeys.java +++ b/compiler/frontend/src/org/jetbrains/kotlin/config/CommonConfigurationKeys.java @@ -28,4 +28,7 @@ public class CommonConfigurationKeys { public static final CompilerConfigurationKey MODULE_NAME = CompilerConfigurationKey.create("module name"); + + public static final CompilerConfigurationKey REPL_MODE = + CompilerConfigurationKey.create("REPL mode"); } diff --git a/compiler/tests/org/jetbrains/kotlin/cli/jvm/KotlinCliJavaFileManagerTest.kt b/compiler/tests/org/jetbrains/kotlin/cli/jvm/KotlinCliJavaFileManagerTest.kt index 2603829dcac..6b00bf03b21 100644 --- a/compiler/tests/org/jetbrains/kotlin/cli/jvm/KotlinCliJavaFileManagerTest.kt +++ b/compiler/tests/org/jetbrains/kotlin/cli/jvm/KotlinCliJavaFileManagerTest.kt @@ -196,7 +196,7 @@ class KotlinCliJavaFileManagerTest : KotlinTestWithEnvironment() { val coreJavaFileManager = ServiceManager.getService(project, CoreJavaFileManager::class.java) as KotlinCliJavaFileManagerImpl val root = environment.contentRootToVirtualFile(JavaSourceRoot(javaFilesDir!!, null))!! - coreJavaFileManager.initIndex(JvmDependenciesIndex(listOf(JavaRoot(root, JavaRoot.RootType.SOURCE)))) + coreJavaFileManager.initIndex(JvmDependenciesIndexImpl(listOf(JavaRoot(root, JavaRoot.RootType.SOURCE)))) return coreJavaFileManager }