Move caching to after all configuration refinements and ...

move compilation cache setting to the scripting host configuration
The cache implementation can now take into account all external
dependencies that may appear during refinement.
Also change initial refinement location accordingly and rearrange
caching interfaces to make it available in scripting plugin.

libraries/scripting/jvm-host-test/test/kotlin/script/experimental/jvmhost/test/CachingTest.kt

@@ -18,12 +18,10 @@ import kotlin.script.experimental.api.*
 import kotlin.script.experimental.host.toScriptSource
 import kotlin.script.experimental.host.with
 import kotlin.script.experimental.jvm.*
+import kotlin.script.experimental.jvm.BasicJvmScriptEvaluator
 import kotlin.script.experimental.jvm.impl.KJvmCompiledScript
 import kotlin.script.experimental.jvm.util.KotlinJars
-import kotlin.script.experimental.jvmhost.BasicJvmScriptingHost
-import kotlin.script.experimental.jvmhost.CompiledJvmScriptsCache
-import kotlin.script.experimental.jvmhost.CompiledScriptJarsCache
-import kotlin.script.experimental.jvmhost.JvmScriptCompiler
+import kotlin.script.experimental.jvmhost.*
 
 class CachingTest : TestCase() {
 
@@ -104,9 +102,10 @@ class CachingTest : TestCase() {
         val hostConfiguration = defaultJvmScriptingHostConfiguration.with {
             jvm {
                 baseClassLoader.replaceOnlyDefault(null)
+                compilationCache(cache)
             }
         }
-        val compiler = JvmScriptCompiler(hostConfiguration, cache = cache)
+        val compiler = JvmScriptCompiler(hostConfiguration)
         val evaluator = BasicJvmScriptEvaluator()
         val host = BasicJvmScriptingHost(compiler = compiler, evaluator = evaluator)
 

libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/CompiledScriptJarsCache.kt

@@ -13,12 +13,14 @@ import kotlin.script.experimental.api.ScriptCompilationConfiguration
 import kotlin.script.experimental.api.SourceCode
 import kotlin.script.experimental.api.hostConfiguration
 import kotlin.script.experimental.host.ScriptingHostConfiguration
+import kotlin.script.experimental.jvm.CompiledJvmScriptsCache
 import kotlin.script.experimental.jvm.baseClassLoader
 import kotlin.script.experimental.jvm.impl.KJvmCompiledScript
 import kotlin.script.experimental.jvm.impl.createScriptFromClassLoader
 import kotlin.script.experimental.jvm.jvm
 
-open class CompiledScriptJarsCache(val scriptToFile: (SourceCode, ScriptCompilationConfiguration) -> File?) : CompiledJvmScriptsCache {
+open class CompiledScriptJarsCache(val scriptToFile: (SourceCode, ScriptCompilationConfiguration) -> File?) :
+    CompiledJvmScriptsCache {
 
     override fun get(script: SourceCode, scriptCompilationConfiguration: ScriptCompilationConfiguration): CompiledScript<*>? {
         val file = scriptToFile(script, scriptCompilationConfiguration)

libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/jvmScriptCompilation.kt

@@ -14,26 +14,9 @@ import kotlin.script.experimental.host.ScriptingHostConfiguration
 import kotlin.script.experimental.jvm.defaultJvmScriptingHostConfiguration
 import kotlin.script.experimental.jvmhost.impl.withDefaults
 
-interface CompiledJvmScriptsCache {
-    fun get(script: SourceCode, scriptCompilationConfiguration: ScriptCompilationConfiguration): CompiledScript<*>?
-    fun store(compiledScript: CompiledScript<*>, script: SourceCode, scriptCompilationConfiguration: ScriptCompilationConfiguration)
-
-    object NoCache : CompiledJvmScriptsCache {
-        override fun get(
-            script: SourceCode, scriptCompilationConfiguration: ScriptCompilationConfiguration
-        ): CompiledScript<*>? = null
-
-        override fun store(
-            compiledScript: CompiledScript<*>, script: SourceCode, scriptCompilationConfiguration: ScriptCompilationConfiguration
-        ) {
-        }
-    }
-}
-
 open class JvmScriptCompiler(
     baseHostConfiguration: ScriptingHostConfiguration = defaultJvmScriptingHostConfiguration,
-    compilerProxy: ScriptJvmCompilerProxy? = null,
-    val cache: CompiledJvmScriptsCache = CompiledJvmScriptsCache.NoCache
+    compilerProxy: ScriptJvmCompilerProxy? = null
 ) : ScriptCompiler {
 
     val hostConfiguration = baseHostConfiguration.withDefaults()
@@ -43,24 +26,12 @@ open class JvmScriptCompiler(
     override suspend operator fun invoke(
         script: SourceCode,
         scriptCompilationConfiguration: ScriptCompilationConfiguration
-    ): ResultWithDiagnostics<CompiledScript<*>> {
-
-        // TODO: implement caching deeper in the compilation pipeline - the actual configuration should be calculated first, with dependencies and imported scripts
-        // Note that previous implementation wasn't a solution for that and added problems with cache usage. Now it is consistent although shallow.
-
-        val cached = cache.get(script, scriptCompilationConfiguration)
-        if (cached != null) return cached.asSuccess()
-
-        return scriptCompilationConfiguration.with {
-            hostConfiguration(this@JvmScriptCompiler.hostConfiguration)
-        }.refineBeforeParsing(script).onSuccess { refinedConfiguration ->
-
-            compilerProxy.compile(script, refinedConfiguration).also {
-                if (it is ResultWithDiagnostics.Success) {
-                    cache.store(it.value, script, scriptCompilationConfiguration)
-                }
+    ): ResultWithDiagnostics<CompiledScript<*>> =
+        compilerProxy.compile(
+            script,
+            scriptCompilationConfiguration.with {
+                hostConfiguration(this@JvmScriptCompiler.hostConfiguration)
             }
-        }
-    }
+        )
 }
 

libraries/scripting/jvm/src/kotlin/script/experimental/jvm/jvmScriptCaching.kt

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.script.experimental.jvm
+
+import kotlin.script.experimental.api.CompiledScript
+import kotlin.script.experimental.api.ScriptCompilationConfiguration
+import kotlin.script.experimental.api.SourceCode
+import kotlin.script.experimental.util.PropertiesCollection
+
+interface CompiledJvmScriptsCache {
+    fun get(script: SourceCode, scriptCompilationConfiguration: ScriptCompilationConfiguration): CompiledScript<*>?
+    fun store(compiledScript: CompiledScript<*>, script: SourceCode, scriptCompilationConfiguration: ScriptCompilationConfiguration)
+
+    object NoCache : CompiledJvmScriptsCache {
+        override fun get(
+            script: SourceCode, scriptCompilationConfiguration: ScriptCompilationConfiguration
+        ): CompiledScript<*>? = null
+
+        override fun store(
+            compiledScript: CompiledScript<*>, script: SourceCode, scriptCompilationConfiguration: ScriptCompilationConfiguration
+        ) {
+        }
+    }
+}
+
+val JvmScriptingHostConfigurationKeys.compilationCache by PropertiesCollection.key<CompiledJvmScriptsCache>(isTransient = true)
+

plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/impl/ScriptJvmCompilerImpls.kt

@@ -18,9 +18,13 @@ import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.scripting.compiler.plugin.ScriptJvmCompilerProxy
+import org.jetbrains.kotlin.scripting.compiler.plugin.dependencies.ScriptsCompilationDependencies
 import org.jetbrains.kotlin.scripting.definitions.ScriptDependenciesProvider
 import kotlin.script.experimental.api.*
 import kotlin.script.experimental.host.ScriptingHostConfiguration
+import kotlin.script.experimental.jvm.compilationCache
+import kotlin.script.experimental.jvm.impl.KJvmCompiledScript
+import kotlin.script.experimental.jvm.jvm
 
 class ScriptJvmCompilerIsolated(val hostConfiguration: ScriptingHostConfiguration) : ScriptJvmCompilerProxy {
 
@@ -30,8 +34,12 @@ class ScriptJvmCompilerIsolated(val hostConfiguration: ScriptingHostConfiguratio
     ): ResultWithDiagnostics<CompiledScript<*>> =
         withMessageCollectorAndDisposable(script = script) { messageCollector, disposable ->
 
+            val initialConfiguration = scriptCompilationConfiguration.refineBeforeParsing(script).valueOr {
+                return it
+            }
+
             val context = createIsolatedCompilationContext(
-                scriptCompilationConfiguration, hostConfiguration, messageCollector, disposable
+                initialConfiguration, hostConfiguration, messageCollector, disposable
             )
 
             compileImpl(script, context, messageCollector)
@@ -47,7 +55,11 @@ class ScriptJvmCompilerFromEnvironment(val environment: KotlinCoreEnvironment) :
         val parentMessageCollector = environment.configuration[CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY]
         return withMessageCollector(script = script, parentMessageCollector = parentMessageCollector) { messageCollector ->
 
-            val context = createCompilationContextFromEnvironment(scriptCompilationConfiguration, environment, messageCollector)
+            val initialConfiguration = scriptCompilationConfiguration.refineBeforeParsing(script).valueOr {
+                return it
+            }
+
+            val context = createCompilationContextFromEnvironment(initialConfiguration, environment, messageCollector)
 
             try {
                 environment.configuration.put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, messageCollector)
@@ -83,6 +95,33 @@ private fun compileImpl(
     )
 
     val dependenciesProvider = ScriptDependenciesProvider.getInstance(context.environment.project)
+    val getScriptConfiguration = { ktFile: KtFile ->
+        dependenciesProvider?.getScriptConfigurationResult(ktFile)?.valueOrNull()?.configuration
+            ?: context.baseScriptCompilationConfiguration
+    }
+
+    val refinedConfiguration = getScriptConfiguration(mainKtFile)
+
+    val cache = refinedConfiguration[ScriptCompilationConfiguration.hostConfiguration]?.get(ScriptingHostConfiguration.jvm.compilationCache)
+
+    val cached = cache?.get(script, refinedConfiguration)
+
+    return cached?.asSuccess(messageCollector.diagnostics)
+        ?: doCompile(context, script, sourceFiles, sourceDependencies, messageCollector, getScriptConfiguration).also {
+            if (cache != null && it is ResultWithDiagnostics.Success) {
+                cache.store(it.value, script, refinedConfiguration)
+            }
+        }
+}
+
+private fun doCompile(
+    context: SharedScriptCompilationContext,
+    script: SourceCode,
+    sourceFiles: List<KtFile>,
+    sourceDependencies: List<ScriptsCompilationDependencies.SourceDependencies>,
+    messageCollector: ScriptDiagnosticsMessageCollector,
+    getScriptConfiguration: (KtFile) -> ScriptCompilationConfiguration
+): ResultWithDiagnostics<KJvmCompiledScript<Any>> {
     val analysisResult = analyze(sourceFiles, context.environment)
 
     if (!analysisResult.shouldGenerateCode) return failure(
@@ -102,11 +141,9 @@ private fun compileImpl(
             generationState,
             script,
             sourceFiles.first(),
-            sourceDependencies
-        ) { ktFile ->
-            dependenciesProvider?.getScriptConfigurationResult(ktFile)?.valueOrNull()?.configuration
-                ?: context.baseScriptCompilationConfiguration
-        }
+            sourceDependencies,
+            getScriptConfiguration
+        )
 
     return ResultWithDiagnostics.Success(compiledScript, messageCollector.diagnostics)
 }