Split compiler and ide specific parts of script dependency caching

Move all caching logic to ide specific KotlinScriptConfigurationManager Clean up apis Remove logging when updating caches
2017-05-25 17:53:56 +03:00
parent e43a145614
commit b7fc909821
6 changed files with 129 additions and 169 deletions
@@ -185,9 +185,10 @@ class KotlinCoreEnvironment private constructor(
            scriptDefinitionProvider.setScriptDefinitions(
                    configuration.getList(JVMConfigurationKeys.SCRIPT_DEFINITIONS))

-            KotlinScriptExternalImportsProvider.getInstance(project)?.run {
+            KotlinScriptExternalImportsProvider.getInstance(project)?.let { importsProvider ->
                configuration.addJvmClasspathRoots(
-                        getCombinedClasspathFor(sourceFiles)
+                        sourceFiles.mapNotNull(importsProvider::getExternalImports)
+                                .flatMap { it.classpath }
                                .distinctBy { it.absolutePath })
            }
        }
@@ -20,8 +20,6 @@ import com.intellij.openapi.components.ServiceManager
 import com.intellij.openapi.diagnostic.Logger
 import com.intellij.openapi.project.Project
 import java.io.File
-import java.lang.management.ManagementFactory
-import java.util.concurrent.TimeUnit
 import java.util.concurrent.locks.ReentrantReadWriteLock
 import kotlin.concurrent.read
 import kotlin.concurrent.write
@@ -39,10 +37,6 @@ class KotlinScriptExternalImportsProviderImpl(
        calculateExternalDependencies(file)
    }

-    fun <TF: Any> getExternalImports(files: Iterable<TF>): List<KotlinScriptExternalDependencies> = cacheLock.read {
-        files.mapNotNull { calculateExternalDependencies(it) }
-    }
-
    private fun <TF: Any> calculateExternalDependencies(file: TF): KotlinScriptExternalDependencies? {
        val path = getFilePath(file)
        val cached = cache[path]
@@ -62,107 +56,6 @@ class KotlinScriptExternalImportsProviderImpl(
        }
    }

-    // optimized for initial caching, additional handling of possible duplicates to save a call to distinct
-    // returns list of cached files
-    override fun <TF: Any> cacheExternalImports(files: Iterable<TF>): Iterable<TF> = cacheLock.write {
-        var filesCount = 0
-        var additionsCount = 0
-        val (res, time) = measureThreadTimeMillis {
-            files.mapNotNull { file ->
-                filesCount += 1
-                val scriptDef = scriptDefinitionProvider.findScriptDefinition(file)
-                if (scriptDef != null) {
-                    val path = getFilePath(file)
-                    if (isValidFile(file) && !cache.containsKey(path)) {
-                        val deps = scriptDef.getDependenciesFor(file, project, null)
-                        cache.put(path, deps)
-                        if (deps != null) {
-                            log.info("[kts] cached deps for $path: ${deps.classpath.joinToString(File.pathSeparator)}")
-                            additionsCount += 1
-                            file
-                        }
-                        else null
-                    }
-                    else null
-                }
-                else null
-            }
-        }
-        log.info("[kts] cache creation: $filesCount checked, $additionsCount added (in ${time}ms)")
-        res
-    }
-
-    // optimized for update, no special duplicates handling
-    // returns files with valid script definition (or deleted from cache - which in fact should have script def too)
-    // TODO: this is the badly designed contract, since it mixes the entities, but these files are needed on the calling site now. Find out other solution
-    override fun <TF: Any> updateExternalImportsCache(files: Iterable<TF>): Iterable<TF> = cacheLock.write {
-        var filesCount = 0
-        var updatesCount = 0
-        val (res, time) = measureThreadTimeMillis {
-            files.mapNotNull { file ->
-                filesCount += 1
-                val scriptDef = scriptDefinitionProvider.findScriptDefinition(file)
-                if (scriptDef != null) {
-                    val path = getFilePath(file)
-                    if (!isValidFile(file)) {
-                        if (cache.remove(path) != null) {
-                            log.debug("[kts] removed deps for file $path")
-                            updatesCount += 1
-                            file
-                        } // cleared
-                        else null // unknown
-                    }
-                    else {
-                        val oldDeps = cache[path]
-                        val deps = scriptDef.getDependenciesFor(file, project, oldDeps)
-                        when {
-                            deps != null && (oldDeps == null ||
-                                             !deps.classpath.isSamePathListAs(oldDeps.classpath) || !deps.sources.isSamePathListAs(oldDeps.sources)) -> {
-                                // changed or new
-                                log.info("[kts] updated/new cached deps for $path: ${deps.classpath.joinToString(File.pathSeparator)}")
-                                cache.put(path, deps)
-                                updatesCount += 1
-                                file
-                            }
-                            deps != null -> {
-                                // same as before
-                                null
-                            }
-                            cache.remove(path) != null -> {
-                                log.debug("[kts] removed deps for $path")
-                                updatesCount += 1
-                                file
-                            }
-                            else -> null // unknown
-                        }
-                    }
-                }
-                else null // not a script
-            }
-        }
-        if (updatesCount > 0) {
-            log.info("[kts] cache update check: $filesCount checked, $updatesCount updated (in ${time}ms)")
-        }
-        res
-    }
-
-    override fun invalidateCaches() {
-        cacheLock.write(cache::clear)
-    }
-
-    override fun getKnownCombinedClasspath(): List<File> = cacheLock.read {
-        cache.values.flatMap { it?.classpath ?: emptyList() }
-    }.distinct()
-
-    override fun getKnownSourceRoots(): List<File> = cacheLock.read {
-        cache.values.flatMap { it?.sources ?: emptyList() }
-    }.distinct()
-
-    override fun <TF: Any> getCombinedClasspathFor(files: Iterable<TF>): List<File> =
-        getExternalImports(files)
-                .flatMap { it.classpath }
-                .distinct()
-
    companion object {
        @JvmStatic
        fun getInstance(project: Project): KotlinScriptExternalImportsProvider? =
@@ -170,18 +63,3 @@ class KotlinScriptExternalImportsProviderImpl(
        internal val log = Logger.getInstance(KotlinScriptExternalImportsProvider::class.java)
    }
 }
-
-private fun Iterable<File>.isSamePathListAs(other: Iterable<File>): Boolean =
-        with (Pair(iterator(), other.iterator())) {
-            while (first.hasNext() && second.hasNext()) {
-                if (first.next().canonicalPath != second.next().canonicalPath) return false
-            }
-            !(first.hasNext() || second.hasNext())
-        }
-
-private inline fun<T> measureThreadTimeMillis(body: () -> T): Pair<T, Long> {
-    val mxBeans = ManagementFactory.getThreadMXBean()
-    val startTime = mxBeans.currentThreadCpuTime
-    val res = body()
-    return res to TimeUnit.NANOSECONDS.toMillis(mxBeans.currentThreadCpuTime - startTime)
-}
@@ -19,23 +19,16 @@ package org.jetbrains.kotlin.script
 import com.intellij.openapi.components.ServiceManager
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.vfs.VirtualFile
-import java.io.File
 import kotlin.script.dependencies.KotlinScriptExternalDependencies

 interface KotlinScriptExternalImportsProvider {
    fun <TF: Any> getExternalImports(file: TF): KotlinScriptExternalDependencies?
-    fun <TF: Any> cacheExternalImports(files: Iterable<TF>): Iterable<TF>
-    fun <TF: Any> updateExternalImportsCache(files: Iterable<TF>): Iterable<TF>
-    fun invalidateCaches()
-    fun getKnownCombinedClasspath(): List<File>
-    fun getKnownSourceRoots(): List<File>
-    fun <TF: Any> getCombinedClasspathFor(files: Iterable<TF>): List<File>

    companion object {
-        fun getInstance(project: Project): KotlinScriptExternalImportsProvider? =
+        fun getInstance(project: Project): KotlinScriptExternalImportsProvider =
                ServiceManager.getService(project, KotlinScriptExternalImportsProvider::class.java)
    }
 }

 fun getScriptExternalDependencies(file: VirtualFile, project: Project): KotlinScriptExternalDependencies?  =
-        KotlinScriptExternalImportsProvider.getInstance(project)?.getExternalImports(file)
+        KotlinScriptExternalImportsProvider.getInstance(project).getExternalImports(file)
@@ -44,7 +44,7 @@ data class ScriptModuleInfo(val project: Project, val scriptFile: VirtualFile,
        get() = ModuleOrigin.OTHER

    val externalDependencies: KotlinScriptExternalDependencies?
-        get() = KotlinScriptExternalImportsProvider.getInstance(project)?.getExternalImports(scriptFile)
+        get() = KotlinScriptConfigurationManager.getInstance(project).getExternalImports(scriptFile)

    override val name: Name = Name.special("<script ${scriptFile.name} ${scriptDefinition.name}>")

@@ -40,27 +40,35 @@ import com.intellij.psi.search.NonClasspathDirectoriesScope
 import com.intellij.util.io.URLUtil
 import org.jetbrains.annotations.TestOnly
 import org.jetbrains.kotlin.idea.util.application.runWriteAction
-import org.jetbrains.kotlin.script.KotlinScriptDefinitionProvider
-import org.jetbrains.kotlin.script.KotlinScriptExternalImportsProvider
-import org.jetbrains.kotlin.script.makeScriptDefsFromTemplatesProviderExtensions
+import org.jetbrains.kotlin.script.*
 import java.io.File
 import java.util.concurrent.locks.ReentrantReadWriteLock
 import kotlin.concurrent.read
 import kotlin.concurrent.write
+import kotlin.script.dependencies.KotlinScriptExternalDependencies
+
+
+class IdeScriptExternalImportsProvider(
+        private val scriptConfigurationManager: KotlinScriptConfigurationManager
+) : KotlinScriptExternalImportsProvider {
+    override fun <TF : Any> getExternalImports(file: TF): KotlinScriptExternalDependencies? {
+        return scriptConfigurationManager.getExternalImports(file)
+    }
+}

-@Suppress("SimplifyAssertNotNull")
 class KotlinScriptConfigurationManager(
        private val project: Project,
-        private val scriptDefinitionProvider: KotlinScriptDefinitionProvider,
-        private val scriptExternalImportsProvider: KotlinScriptExternalImportsProvider
+        private val scriptDefinitionProvider: KotlinScriptDefinitionProvider
 ) {

+    private val cacheLock = ReentrantReadWriteLock()
+
    init {
        reloadScriptDefinitions()

        StartupManager.getInstance(project).runWhenProjectIsInitialized {
            DumbService.getInstance(project).smartInvokeLater {
-                if (cacheAllScriptsExtraImports()) {
+                if (populateCache()) {
                    invalidateLocalCaches()
                    notifyRootsChanged()
                }
@@ -74,14 +82,13 @@ class KotlinScriptConfigurationManager(

            override fun after(events: List<VFileEvent>) {
                if (updateExternalImportsCache(events.mapNotNull {
-                        // The check is partly taken from the BuildManager.java
-                        it.file?.takeIf {
-                            // the isUnitTestMode check fixes ScriptConfigurationHighlighting & Navigation tests, since they are not trigger proper update mechanims
-                            // TODO: find out the reason, then consider to fix tests and remove this check
-                            (application.isUnitTestMode || projectFileIndex.isInContent(it)) && !ProjectUtil.isProjectOrWorkspaceFile(it)
-                        }
-                    }))
-                {
+                    // The check is partly taken from the BuildManager.java
+                    it.file?.takeIf {
+                        // the isUnitTestMode check fixes ScriptConfigurationHighlighting & Navigation tests, since they are not trigger proper update mechanims
+                        // TODO: find out the reason, then consider to fix tests and remove this check
+                        (application.isUnitTestMode || projectFileIndex.isInContent(it)) && !ProjectUtil.isProjectOrWorkspaceFile(it)
+                    }
+                })) {
                    invalidateLocalCaches()
                    notifyRootsChanged()
                }
@@ -89,10 +96,9 @@ class KotlinScriptConfigurationManager(
        })
    }

-    private val cacheLock = ReentrantReadWriteLock()
-
    private val allScriptsClasspathCache = ClearableLazyValue(cacheLock) {
-        toVfsRoots(scriptExternalImportsProvider.getKnownCombinedClasspath().distinct())
+        val files = cache.values.flatMap { it?.classpath ?: emptyList() }.distinct()
+        toVfsRoots(files)
    }

    private val allScriptsClasspathScope = ClearableLazyValue(cacheLock) {
@@ -100,7 +106,7 @@ class KotlinScriptConfigurationManager(
    }

    private val allLibrarySourcesCache = ClearableLazyValue(cacheLock) {
-        toVfsRoots(scriptExternalImportsProvider.getKnownSourceRoots().distinct())
+        toVfsRoots(cache.values.flatMap { it?.sources ?: emptyList() }.distinct())
    }

    private val allLibrarySourcesScope = ClearableLazyValue(cacheLock) {
@@ -127,7 +133,7 @@ class KotlinScriptConfigurationManager(
    }

    fun getScriptClasspath(file: VirtualFile): List<VirtualFile> = toVfsRoots(
-            scriptExternalImportsProvider.getExternalImports(file)?.classpath ?: emptyList()
+            getExternalImports(file)?.classpath ?: emptyList()
    )

    fun getAllScriptsClasspath(): List<VirtualFile> = allScriptsClasspathCache.get()
@@ -143,16 +149,86 @@ class KotlinScriptConfigurationManager(
        scriptDefinitionProvider.setScriptDefinitions(def)
    }

-    private fun cacheAllScriptsExtraImports(): Boolean =
-        scriptExternalImportsProvider.run {
-            invalidateCaches()
-            cacheExternalImports(
-                    scriptDefinitionProvider.getAllKnownFileTypes()
-                            .flatMap { FileTypeIndex.getFiles(it, GlobalSearchScope.allScope(project)) }
-            ).any()
+    private fun populateCache(): Boolean {
+        cacheLock.write(cache::clear)
+        return cacheExternalImports(
+                scriptDefinitionProvider.getAllKnownFileTypes()
+                        .flatMap { FileTypeIndex.getFiles(it, GlobalSearchScope.allScope(project)) }
+        ).any()
+    }
+
+    private val cache = hashMapOf<String, KotlinScriptExternalDependencies?>()
+
+    fun <TF : Any> getExternalImports(file: TF): KotlinScriptExternalDependencies? = cacheLock.read {
+        val path = getFilePath(file)
+        cache[path]?.let { return it }
+
+        val scriptDef = scriptDefinitionProvider.findScriptDefinition(file) ?: return null
+        return scriptDef.getDependenciesFor(file, project, null).also {
+            cacheLock.write {
+                cache.put(path, it)
+            }
+        }
+    }
+
+    // optimized for initial caching, additional handling of possible duplicates to save a call to distinct
+    // returns list of cached files
+    fun <TF : Any> cacheExternalImports(files: Iterable<TF>): Iterable<TF> = cacheLock.write {
+        return files.mapNotNull { file ->
+            scriptDefinitionProvider.findScriptDefinition(file)?.let {
+                cacheForFile(file, getFilePath(file), it)
+            }
+        }
+    }
+
+    private fun <TF : Any> cacheForFile(file: TF, path: String, scriptDef: KotlinScriptDefinition): TF? {
+        if (!isValidFile(file) || cache.containsKey(path)) return null
+
+        val deps = scriptDef.getDependenciesFor(file, project, null)
+        cache.put(path, deps)
+
+        return file.takeIf { deps != null }
+    }
+
+    private fun updateExternalImportsCache(files: Iterable<VirtualFile>) = cacheLock.write {
+        files.mapNotNull { file ->
+            scriptDefinitionProvider.findScriptDefinition(file)?.let {
+                updateForFile(file, it)
+            }
+        }
+    }.contains(true)
+
+    private fun <TF : Any> updateForFile(file: TF, scriptDef: KotlinScriptDefinition): Boolean {
+        if (!isValidFile(file)) {
+            return cache.remove(getFilePath(file)) != null
        }

-    private fun updateExternalImportsCache(files: Iterable<VirtualFile>) = scriptExternalImportsProvider.updateExternalImportsCache(files).any()
+        return updateSync(file, scriptDef)
+    }
+
+    private fun <TF : Any> updateSync(file: TF, scriptDef: KotlinScriptDefinition): Boolean {
+        val path = getFilePath(file)
+        val oldDeps = cache[path]
+        val deps = scriptDef.getDependenciesFor(file, project, oldDeps)
+        return cacheNewDependencies(deps, oldDeps, path)
+    }
+
+    private fun cacheNewDependencies(
+            new: KotlinScriptExternalDependencies?,
+            old: KotlinScriptExternalDependencies?,
+            path: String
+    ): Boolean {
+        return when {
+            new != null && (old == null || !(new.match(old))) -> {
+                // changed or new
+                cache.put(path, new)
+                true
+            }
+            new != null -> false // same
+            cache.remove(path) != null -> true
+            else -> false // unknown
+        }
+    }

    private fun invalidateLocalCaches() {
        allScriptsClasspathCache.clear()
@@ -162,13 +238,12 @@ class KotlinScriptConfigurationManager(

        val kotlinScriptDependenciesClassFinder =
                Extensions.getArea(project).getExtensionPoint(PsiElementFinder.EP_NAME).extensions
-                .filterIsInstance<KotlinScriptDependenciesClassFinder>()
-                .single()
+                        .filterIsInstance<KotlinScriptDependenciesClassFinder>()
+                        .single()

        kotlinScriptDependenciesClassFinder.clearCache()
    }

-
    companion object {
        @JvmStatic
        fun getInstance(project: Project): KotlinScriptConfigurationManager =
@@ -188,13 +263,14 @@ class KotlinScriptConfigurationManager(
            // TODO: report this somewhere, but do not throw: assert(res != null, { "Invalid classpath entry '$this': exists: ${exists()}, is directory: $isDirectory, is file: $isFile" })
            return res
        }
+
        internal val log = Logger.getInstance(KotlinScriptConfigurationManager::class.java)

        @TestOnly
        fun reloadScriptDefinitions(project: Project) {
-            with (getInstance(project)) {
+            with(getInstance(project)) {
                reloadScriptDefinitions()
-                scriptExternalImportsProvider.invalidateCaches()
+                cacheLock.write(cache::clear)
                invalidateLocalCaches()
            }
        }
@@ -220,4 +296,16 @@ private class ClearableLazyValue<out T : Any>(private val lock: ReentrantReadWri
            value = null
        }
    }
-}
+}
+
+private fun KotlinScriptExternalDependencies.match(other: KotlinScriptExternalDependencies)
+        = classpath.isSamePathListAs(other.classpath) && sources.isSamePathListAs(other.sources)
+
+
+private fun Iterable<File>.isSamePathListAs(other: Iterable<File>): Boolean =
+        with(Pair(iterator(), other.iterator())) {
+            while (first.hasNext() && second.hasNext()) {
+                if (first.next().canonicalPath != second.next().canonicalPath) return false
+            }
+            !(first.hasNext() || second.hasNext())
+        }
@@ -267,7 +267,7 @@
                    serviceImplementation="org.jetbrains.kotlin.script.KotlinScriptDefinitionProvider"/>

    <projectService serviceInterface="org.jetbrains.kotlin.script.KotlinScriptExternalImportsProvider"
-                    serviceImplementation="org.jetbrains.kotlin.script.KotlinScriptExternalImportsProviderImpl"/>
+                    serviceImplementation="org.jetbrains.kotlin.idea.core.script.IdeScriptExternalImportsProvider"/>

    <projectService serviceInterface="org.jetbrains.kotlin.idea.core.script.KotlinScriptConfigurationManager"
                    serviceImplementation="org.jetbrains.kotlin.idea.core.script.KotlinScriptConfigurationManager"/>