PR-1021: Merge Keplin project scripting code into Kotlin core.

Overhauls the scripting layers (GenericRepl and related, and JSR223 and related)
Adds repeating modes (none, only latest eval'd line, or random order)
Also adds better thread-safe IO capturing, default imports, SimpleRepl wrapper, more unit tests

NOTE: the script-util part of the pull request was rejected due to various problems and incompatibilities.
It may be incorporated into the code later.

(originally cherry picked from commit 6f7d517)

compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/KotlinRemoteReplClient.kt

@@ -35,15 +35,15 @@ open class KotlinRemoteReplClientBase(
         targetPlatform: CompileService.TargetPlatform,
         templateClasspath: List<File>,
         templateClassName: String,
-        scriptArgs: Array<Any?>? = null,
-        scriptArgsTypes: Array<Class<*>>? = null,
+        scriptArgs: Array<out Any?>? = null,
+        scriptArgsTypes: Array<out Class<out Any>>? = null,
         compilerMessagesOutputStream: OutputStream = System.err,
         evalOutputStream: OutputStream? = null,
         evalErrorStream: OutputStream? = null,
         evalInputStream: InputStream? = null,
         port: Int = SOCKET_ANY_FREE_PORT,
         operationsTracer: RemoteOperationsTracer? = null
-) : ReplChecker {
+) {
 
     val sessionId = compileService.leaseReplSession(
             clientAliveFlagFile?.absolutePath,
@@ -70,10 +70,6 @@ open class KotlinRemoteReplClientBase(
             }
         })
     }
-
-    override fun check(codeLine: ReplCodeLine, history: List<ReplCodeLine>): ReplCheckResult {
-        return compileService.remoteReplLineCheck(sessionId, codeLine, history).get()
-    }
 }
 
 class KotlinRemoteReplCompiler(
@@ -99,10 +95,20 @@ class KotlinRemoteReplCompiler(
         evalInputStream = null,
         port = port,
         operationsTracer = operationsTracer
-), ReplCompiler {
+), ReplCompiler, ReplCheckAction {
+    override fun resetToLine(lineNumber: Int): List<ReplCodeLine> {
+        return emptyList() // TODO: not implemented, no current need
+    }
 
-    override fun compile(codeLine: ReplCodeLine, history: List<ReplCodeLine>): ReplCompileResult {
-        return compileService.remoteReplLineCompile(sessionId, codeLine, history).get()
+    override val history: List<ReplCodeLine>
+        get() = emptyList() // TODO: not implemented, no current need
+
+    override fun check(codeLine: ReplCodeLine): ReplCheckResult {
+        return compileService.remoteReplLineCheck(sessionId, codeLine).get()
+    }
+
+    override fun compile(codeLine: ReplCodeLine, verifyHistory: List<ReplCodeLine>?): ReplCompileResult {
+        return compileService.remoteReplLineCompile(sessionId, codeLine, verifyHistory).get()
     }
 }
 
@@ -114,8 +120,8 @@ class KotlinRemoteReplEvaluator(
         targetPlatform: CompileService.TargetPlatform,
         templateClasspath: List<File>,
         templateClassName: String,
-        scriptArgs: Array<Any?>?,
-        scriptArgsTypes: Array<Class<*>>?,
+        scriptArgs: Array<out Any?>? = null,
+        scriptArgsTypes: Array<out Class<out Any>>? = null,
         compilerMessagesOutputStream: OutputStream,
         evalOutputStream: OutputStream?,
         evalErrorStream: OutputStream?,
@@ -137,12 +143,20 @@ class KotlinRemoteReplEvaluator(
         evalInputStream = evalInputStream,
         port = port,
         operationsTracer = operationsTracer
-), ReplEvaluator {
+), ReplAtomicEvalAction, ReplEvaluatorExposedInternalHistory, ReplCheckAction {
 
-    override val lastEvaluatedScript: ClassWithInstance? = null // not implemented, no need so far
+    override val lastEvaluatedScripts: List<EvalHistoryType> = emptyList() // not implemented, no need so far
 
     // TODO: invokeWrapper is ignored here, and in the daemon the session wrapper is used instead; So consider to make it per call (avoid performance penalties though)
-    override fun eval(codeLine: ReplCodeLine, history: List<ReplCodeLine>, invokeWrapper: InvokeWrapper?): ReplEvalResult {
-        return compileService.remoteReplLineEval(sessionId, codeLine, history).get()
+    // TODO: scriptArgs are ignored here, they should be passed through
+    override fun compileAndEval(codeLine: ReplCodeLine,
+                                scriptArgs: ScriptArgsWithTypes?,
+                                verifyHistory: List<ReplCodeLine>?,
+                                invokeWrapper: InvokeWrapper?): ReplEvalResult {
+        return compileService.remoteReplLineEval(sessionId, codeLine, verifyHistory).get()
+    }
+
+    override fun check(codeLine: ReplCodeLine): ReplCheckResult {
+        return compileService.remoteReplLineCheck(sessionId, codeLine).get()
     }
 }

compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompileService.kt

@@ -16,8 +16,10 @@
 
 package org.jetbrains.kotlin.daemon.common
 
-import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
-import org.jetbrains.kotlin.cli.common.repl.*
+import org.jetbrains.kotlin.cli.common.repl.ReplCheckResult
+import org.jetbrains.kotlin.cli.common.repl.ReplCodeLine
+import org.jetbrains.kotlin.cli.common.repl.ReplCompileResult
+import org.jetbrains.kotlin.cli.common.repl.ReplEvalResult
 import java.io.File
 import java.io.Serializable
 import java.rmi.Remote
@@ -148,8 +150,8 @@ interface CompileService : Remote {
             servicesFacade: CompilerCallbackServicesFacade,
             templateClasspath: List<File>,
             templateClassName: String,
-            scriptArgs: Array<Any?>?,
-            scriptArgsTypes: Array<Class<*>>?,
+            scriptArgs: Array<out Any?>?,
+            scriptArgsTypes: Array<out Class<out Any>>?,
             compilerMessagesOutputStream: RemoteOutputStream,
             evalOutputStream: RemoteOutputStream?,
             evalErrorStream: RemoteOutputStream?,
@@ -163,21 +165,20 @@ interface CompileService : Remote {
     @Throws(RemoteException::class)
     fun remoteReplLineCheck(
             sessionId: Int,
-            codeLine: ReplCodeLine,
-            history: List<ReplCodeLine>
+            codeLine: ReplCodeLine
     ): CallResult<ReplCheckResult>
 
     @Throws(RemoteException::class)
     fun remoteReplLineCompile(
             sessionId: Int,
             codeLine: ReplCodeLine,
-            history: List<ReplCodeLine>
+            history: List<ReplCodeLine>?
     ): CallResult<ReplCompileResult>
 
     @Throws(RemoteException::class)
     fun remoteReplLineEval(
             sessionId: Int,
             codeLine: ReplCodeLine,
-            history: List<ReplCodeLine>
+            history: List<ReplCodeLine>?
     ): CallResult<ReplEvalResult>
 }

compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt

@@ -23,29 +23,36 @@ import com.intellij.openapi.vfs.impl.jar.CoreJarFileSystem
 import org.jetbrains.kotlin.cli.common.CLICompiler
 import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.cli.common.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY
-import org.jetbrains.kotlin.cli.common.arguments.*
+import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
+import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
+import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
+import org.jetbrains.kotlin.cli.common.arguments.K2MetadataCompilerArguments
 import org.jetbrains.kotlin.cli.common.messages.*
 import org.jetbrains.kotlin.cli.common.modules.ModuleXmlParser
-import org.jetbrains.kotlin.cli.common.repl.ReplCheckResult
-import org.jetbrains.kotlin.cli.common.repl.ReplCodeLine
-import org.jetbrains.kotlin.cli.common.repl.ReplCompileResult
-import org.jetbrains.kotlin.cli.common.repl.ReplEvalResult
+import org.jetbrains.kotlin.cli.common.repl.*
 import org.jetbrains.kotlin.cli.js.K2JSCompiler
 import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.cli.metadata.K2MetadataCompiler
-import org.jetbrains.kotlin.config.IncrementalCompilation
 import org.jetbrains.kotlin.config.Services
 import org.jetbrains.kotlin.daemon.common.*
-import org.jetbrains.kotlin.daemon.incremental.*
-import org.jetbrains.kotlin.daemon.report.*
+import org.jetbrains.kotlin.daemon.incremental.RemoteAnnotationsFileUpdater
+import org.jetbrains.kotlin.daemon.incremental.RemoteArtifactChangesProvider
+import org.jetbrains.kotlin.daemon.incremental.RemoteChangesRegostry
+import org.jetbrains.kotlin.daemon.report.CompileServicesFacadeMessageCollector
+import org.jetbrains.kotlin.daemon.report.DaemonMessageReporter
+import org.jetbrains.kotlin.daemon.report.DaemonMessageReporterPrintStreamAdapter
+import org.jetbrains.kotlin.daemon.report.RemoteICReporter
 import org.jetbrains.kotlin.incremental.*
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
 import org.jetbrains.kotlin.modules.Module
 import org.jetbrains.kotlin.progress.CompilationCanceledStatus
 import org.jetbrains.kotlin.utils.addToStdlib.check
 import org.jetbrains.kotlin.utils.stackTraceStr
-import java.io.*
+import java.io.BufferedOutputStream
+import java.io.ByteArrayOutputStream
+import java.io.File
+import java.io.PrintStream
 import java.rmi.NoSuchObjectException
 import java.rmi.registry.Registry
 import java.rmi.server.UnicastRemoteObject
@@ -445,8 +452,8 @@ class CompileServiceImpl(
             servicesFacade: CompilerCallbackServicesFacade,
             templateClasspath: List<File>,
             templateClassName: String,
-            scriptArgs: Array<Any?>?,
-            scriptArgsTypes: Array<Class<*>>?,
+            scriptArgs: Array<out Any?>?,
+            scriptArgsTypes: Array<out Class<out Any>>?,
             compilerMessagesOutputStream: RemoteOutputStream,
             evalOutputStream: RemoteOutputStream?,
             evalErrorStream: RemoteOutputStream?,
@@ -457,7 +464,9 @@ class CompileServiceImpl(
             CompileService.CallResult.Error("Sorry, only JVM target platform is supported now")
         else {
             val disposable = Disposer.newDisposable()
-            val repl = KotlinJvmReplService(disposable, templateClasspath, templateClassName, scriptArgs, scriptArgsTypes, compilerMessagesOutputStream, operationsTracer)
+            val repl = KotlinJvmReplService(disposable, templateClasspath, templateClassName,
+                                            scriptArgs?.let { ScriptArgsWithTypes(it, scriptArgsTypes?.map { it.kotlin }?.toTypedArray() ?: emptyArray()) },
+                                            compilerMessagesOutputStream, operationsTracer)
             val sessionId = state.sessions.leaseSession(ClientOrSessionProxy(aliveFlagPath, repl, disposable))
 
             CompileService.CallResult.Good(sessionId)
@@ -467,14 +476,14 @@ class CompileServiceImpl(
     // TODO: add more checks (e.g. is it a repl session)
     override fun releaseReplSession(sessionId: Int): CompileService.CallResult<Nothing> = releaseCompileSession(sessionId)
 
-    override fun remoteReplLineCheck(sessionId: Int, codeLine: ReplCodeLine, history: List<ReplCodeLine>): CompileService.CallResult<ReplCheckResult> =
+    override fun remoteReplLineCheck(sessionId: Int, codeLine: ReplCodeLine): CompileService.CallResult<ReplCheckResult> =
             ifAlive(minAliveness = Aliveness.Alive) {
                 withValidRepl(sessionId) {
-                    check(codeLine, history)
+                    check(codeLine)
                 }
             }
 
-    override fun remoteReplLineCompile(sessionId: Int, codeLine: ReplCodeLine, history: List<ReplCodeLine>): CompileService.CallResult<ReplCompileResult> =
+    override fun remoteReplLineCompile(sessionId: Int, codeLine: ReplCodeLine, history: List<ReplCodeLine>?): CompileService.CallResult<ReplCompileResult> =
             ifAlive(minAliveness = Aliveness.Alive) {
                 withValidRepl(sessionId) {
                     compile(codeLine, history)
@@ -484,11 +493,11 @@ class CompileServiceImpl(
     override fun remoteReplLineEval(
             sessionId: Int,
             codeLine: ReplCodeLine,
-            history: List<ReplCodeLine>
+            history: List<ReplCodeLine>?
     ): CompileService.CallResult<ReplEvalResult> =
             ifAlive(minAliveness = Aliveness.Alive) {
                 withValidRepl(sessionId) {
-                    eval(codeLine, history)
+                    compileAndEval(codeLine, verifyHistory = history)
                 }
             }
 

compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinRemoteReplService.kt

@@ -22,7 +22,6 @@ import org.jetbrains.kotlin.cli.common.repl.*
 import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots
 import org.jetbrains.kotlin.cli.jvm.config.jvmClasspathRoots
 import org.jetbrains.kotlin.cli.jvm.repl.GenericReplCompiler
-import org.jetbrains.kotlin.cli.jvm.repl.compileAndEval
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.daemon.common.DummyProfiler
@@ -40,12 +39,10 @@ open class KotlinJvmReplService(
         disposable: Disposable,
         templateClasspath: List<File>,
         templateClassName: String,
-        scriptArgs: Array<Any?>?,
-        scriptArgsTypes: Array<Class<*>>?,
+        protected val fallbackScriptArgs: ScriptArgsWithTypes?,
         compilerOutputStreamProxy: RemoteOutputStream,
-        val operationsTracer: RemoteOperationsTracer?
-) : ReplCompiler, ReplEvaluator {
-
+        protected val operationsTracer: RemoteOperationsTracer?
+) : ReplCompileAction, ReplAtomicEvalAction, ReplCheckAction, ReplEvaluatorExposedInternalHistory {
     protected val compilerMessagesStream = PrintStream(BufferedOutputStream(RemoteOutputStreamClient(compilerOutputStreamProxy, DummyProfiler()), REMOTE_STREAM_BUFFER_SIZE))
 
     protected class KeepFirstErrorMessageCollector(compilerMessagesStream: PrintStream) : MessageCollector {
@@ -105,23 +102,24 @@ open class KotlinJvmReplService(
 
     private val scriptDef = makeScriptDefinition(templateClasspath, templateClassName)
 
-    private val replCompiler : GenericReplCompiler? by lazy {
+    private val replCompiler: ReplCompiler? by lazy {
         if (scriptDef == null) null
         else GenericReplCompiler(disposable, scriptDef, configuration, messageCollector)
     }
 
-    private val compiledEvaluator : GenericReplCompiledEvaluator by lazy {
-        GenericReplCompiledEvaluator(configuration.jvmClasspathRoots, null, scriptArgs, scriptArgsTypes)
+    private val replEvaluator: ReplFullEvaluator? by lazy {
+        replCompiler?.let { compiler ->
+            GenericReplCompilingEvaluator(compiler, configuration.jvmClasspathRoots, null, fallbackScriptArgs, ReplRepeatingMode.NONE)
+        }
     }
 
-    override val lastEvaluatedScript: ClassWithInstance? get() = compiledEvaluator.lastEvaluatedScript
+    override val lastEvaluatedScripts: List<EvalHistoryType> get() = replEvaluator?.lastEvaluatedScripts ?: emptyList()
 
-    override fun check(codeLine: ReplCodeLine, history: List<ReplCodeLine>): ReplCheckResult {
+    override fun check(codeLine: ReplCodeLine): ReplCheckResult {
         operationsTracer?.before("check")
         try {
-            return replCompiler?.check(codeLine, history)
-                   ?: ReplCheckResult.Error(history,
-                                            messageCollector.firstErrorMessage ?: "Unknown error",
+            return replCompiler?.check(codeLine)
+                   ?: ReplCheckResult.Error(messageCollector.firstErrorMessage ?: "Unknown error",
                                             messageCollector.firstErrorLocation ?: CompilerMessageLocation.NO_LOCATION)
         }
         finally {
@@ -129,11 +127,11 @@ open class KotlinJvmReplService(
         }
     }
 
-    override fun compile(codeLine: ReplCodeLine, history: List<ReplCodeLine>): ReplCompileResult {
+    override fun compile(codeLine: ReplCodeLine, verifyHistory: List<ReplCodeLine>?): ReplCompileResult {
         operationsTracer?.before("compile")
         try {
-            return replCompiler?.compile(codeLine, history)
-                   ?: ReplCompileResult.Error(history,
+            return replCompiler?.compile(codeLine, verifyHistory)
+                   ?: ReplCompileResult.Error(verifyHistory ?: emptyList(),
                                               messageCollector.firstErrorMessage ?: "Unknown error",
                                               messageCollector.firstErrorLocation ?: CompilerMessageLocation.NO_LOCATION)
         }
@@ -142,13 +140,13 @@ open class KotlinJvmReplService(
         }
     }
 
-    override fun eval(codeLine: ReplCodeLine, history: List<ReplCodeLine>, invokeWrapper: InvokeWrapper?): ReplEvalResult = synchronized(this) {
+    override fun compileAndEval(codeLine: ReplCodeLine, scriptArgs: ScriptArgsWithTypes?, verifyHistory: List<ReplCodeLine>?, invokeWrapper: InvokeWrapper?): ReplEvalResult {
         operationsTracer?.before("eval")
         try {
-            return replCompiler?.let { compileAndEval(it, compiledEvaluator, codeLine, history, invokeWrapper) }
-                   ?: ReplEvalResult.Error.CompileTime(history,
-                                                       messageCollector.firstErrorMessage ?: "Unknown error",
-                                                       messageCollector.firstErrorLocation ?: CompilerMessageLocation.NO_LOCATION)
+            return replEvaluator?.compileAndEval(codeLine, scriptArgs ?: fallbackScriptArgs, verifyHistory, invokeWrapper)
+                   ?: ReplEvalResult.Error.CompileTime(verifyHistory ?: replEvaluator?.history ?: emptyList(),
+                                                         messageCollector.firstErrorMessage ?: "Unknown error",
+                                                         messageCollector.firstErrorLocation ?: CompilerMessageLocation.NO_LOCATION)
         }
         finally {
             operationsTracer?.after("eval")