Reintroduce history check between compile and eval, place generation into code line and id

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/repl/AggregatedReplState.kt

@@ -80,5 +80,9 @@ open class AggregatedReplStageState<T1, T2>(val state1: IReplStageState<T1>, val
                 target.isAssignableFrom(state2::class.java) -> state2 as StateT
                 else -> super.asState(target)
             }
+
+    override fun getNextLineNo() = state1.getNextLineNo()
+
+    override val currentGeneration: Int get() = state1.currentGeneration
 }
 

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/repl/BasicReplState.kt

@@ -18,18 +18,19 @@ package org.jetbrains.kotlin.cli.common.repl
 
 import java.io.Serializable
 import java.util.*
+import java.util.concurrent.atomic.AtomicInteger
 import java.util.concurrent.locks.ReentrantReadWriteLock
 import kotlin.concurrent.write
 
-data class LineId(override val no: Int, private val codeHash: Int) : ILineId, Serializable {
+data class LineId(override val no: Int, override val generation: Int, private val codeHash: Int) : ILineId, Serializable {
 
-    constructor(no: Int, code: String): this(no, code.hashCode())
-    constructor(codeLine: ReplCodeLine): this(codeLine.no, codeLine.code.hashCode())
+    constructor(no: Int, generation: Int, code: String): this(no, generation, code.hashCode())
+    constructor(codeLine: ReplCodeLine): this(codeLine.no, codeLine.generation, codeLine.code.hashCode())
 
     override fun compareTo(other: ILineId): Int = (other as? LineId)?.let {
-        val c = no.compareTo(it.no)
-        if (c == 0) codeHash.compareTo(it.codeHash)
-        else c
+        no.compareTo(it.no).takeIf { it != 0 }
+        ?: generation.compareTo(it.generation).takeIf { it != 0 }
+        ?: codeHash.compareTo(it.codeHash)
     } ?: -1 // TODO: check if it doesn't break something
 
     companion object {
@@ -39,6 +40,8 @@ data class LineId(override val no: Int, private val codeHash: Int) : ILineId, Se
 
 open class BasicReplStageHistory<T>(override val lock: ReentrantReadWriteLock = ReentrantReadWriteLock()) : IReplStageHistory<T>, ArrayList<ReplHistoryRecord<T>>() {
 
+    val currentGeneration = AtomicInteger(REPL_CODE_LINE_FIRST_GEN)
+
     override fun push(id: ILineId, item: T) {
         lock.write {
             add(ReplHistoryRecord(id, item))
@@ -54,6 +57,7 @@ open class BasicReplStageHistory<T>(override val lock: ReentrantReadWriteLock =
             if (idx < lastIndex) {
                 val removed = asSequence().drop(idx + 1).map { it.id }.toList()
                 removeRange(idx + 1, lastIndex)
+                currentGeneration.incrementAndGet()
                 return removed
             }
             else return emptyList()
@@ -61,7 +65,9 @@ open class BasicReplStageHistory<T>(override val lock: ReentrantReadWriteLock =
     }
 }
 
-class BasicReplStageState<HistoryItemT>: IReplStageState<HistoryItemT> {
+open class BasicReplStageState<HistoryItemT>: IReplStageState<HistoryItemT> {
+
+    override val currentGeneration: Int get() = history.currentGeneration.get()
 
     override val lock = ReentrantReadWriteLock()
 

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/repl/GenericEvaluatorState.kt

@@ -26,6 +26,8 @@ open class GenericReplEvaluatorState(baseClasspath: Iterable<File>, baseClassloa
 {
     override val history: IReplStageHistory<EvalClassWithInstanceAndLoader> = BasicReplStageHistory(lock)
 
+    override val currentGeneration: Int get() = (history as BasicReplStageHistory<*>).currentGeneration.get()
+
     val topClassLoader: ReplClassLoader = makeReplClassLoader(baseClassloader, baseClasspath)
 
     val currentClasspath: List<File> get() = lock.read {

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/repl/GenericReplCompilingEvaluator.kt

@@ -36,7 +36,6 @@ class GenericReplCompilingEvaluator(val compiler: ReplCompiler,
             val compiled = compiler.compile(state, codeLine)
             when (compiled) {
                 is ReplCompileResult.Error -> ReplEvalResult.Error.CompileTime(compiled.message, compiled.location)
-                is ReplCompileResult.HistoryMismatch -> ReplEvalResult.HistoryMismatch(compiled.lineNo)
                 is ReplCompileResult.Incomplete -> ReplEvalResult.Incomplete()
                 is ReplCompileResult.CompiledClasses -> {
                     val result = eval(state, compiled, scriptArgs, invokeWrapper)

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/repl/GenericReplEvaluator.kt

@@ -113,10 +113,10 @@ open class GenericReplEvaluator(val baseClasspath: Iterable<File>,
                 }
             }
 
-//            val firstMismatch = historyActor.firstMismatch(verifyHistory)
-//            if (firstMismatch != null) {
-//                return@eval ReplEvalResult.HistoryMismatch(firstMismatch)
-//            }
+            val firstMismatch = historyActor.firstMismatch(compileResult.previousLines.asSequence())
+            if (firstMismatch != null) {
+                return@eval ReplEvalResult.HistoryMismatch(firstMismatch.first?.id?.no ?: firstMismatch.second?.no ?: -1 /* means error? */)
+            }
 
             val (classLoader, scriptClass) = try {
                 historyActor.processClasses(compileResult)
@@ -172,7 +172,7 @@ private open class HistoryActionsForNoRepeat(val state: GenericReplEvaluatorStat
 
     open val effectiveHistory: List<EvalClassWithInstanceAndLoader> get() = state.history.map { it.item }
 
-    open fun<OtherT> firstMismatch(other: IReplStageHistory<OtherT>): Pair<ReplHistoryRecord<EvalClassWithInstanceAndLoader>?, ReplHistoryRecord<OtherT>?>? = state.history.firstMismatch(other)
+    open fun firstMismatch(other: Sequence<ILineId>): Pair<ReplHistoryRecord<EvalClassWithInstanceAndLoader>?, ILineId?>? = state.history.firstMismatch(other)
 
     open fun addPlaceholder(lineId: ILineId, value: EvalClassWithInstanceAndLoader) { state.history.push(lineId, value) }
 
@@ -215,7 +215,7 @@ private open class HistoryActionsForRepeatRecentOnly(state: GenericReplEvaluator
 
     override val effectiveHistory: List<EvalClassWithInstanceAndLoader> get() = super.effectiveHistory.dropLast(1)
 
-    override fun<OtherT> firstMismatch(other: IReplStageHistory<OtherT>): Pair<ReplHistoryRecord<EvalClassWithInstanceAndLoader>?, ReplHistoryRecord<OtherT>?>? =
+    override fun firstMismatch(other: Sequence<ILineId>): Pair<ReplHistoryRecord<EvalClassWithInstanceAndLoader>?, ILineId?>? =
             state.history.firstMismatchFiltered(other) { it.id != currentLast.id }
 
     override fun addPlaceholder(lineId: ILineId, value: EvalClassWithInstanceAndLoader) {}
@@ -235,7 +235,7 @@ private open class HistoryActionsForRepeatAny(state: GenericReplEvaluatorState,
 
     override val effectiveHistory: List<EvalClassWithInstanceAndLoader> get() = state.history.takeWhile { it.id != matchingLine.id }.map { it.item }
 
-    override fun<OtherT> firstMismatch(other: IReplStageHistory<OtherT>): Pair<ReplHistoryRecord<EvalClassWithInstanceAndLoader>?, ReplHistoryRecord<OtherT>?>? =
+    override fun firstMismatch(other: Sequence<ILineId>): Pair<ReplHistoryRecord<EvalClassWithInstanceAndLoader>?, ILineId?>? =
             state.history.firstMismatchWhile(other) { it.id != matchingLine.id }
 
     override fun addPlaceholder(lineId: ILineId, value: EvalClassWithInstanceAndLoader) {}

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/repl/KotlinJsr223JvmScriptEngineBase.kt

@@ -23,17 +23,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock
 import javax.script.*
 
 val KOTLIN_SCRIPT_STATE_BINDINGS_KEY = "kotlin.script.state"
-val KOTLIN_SCRIPT_LINE_NUMBER_BINDINGS_KEY = "kotlin.script.line.number"
-
-// TODO consider additional error handling
-var Bindings.kotlinScriptState: IReplStageState<*>
-    @Suppress("UNCHECKED_CAST")
-    get() = get(KOTLIN_SCRIPT_STATE_BINDINGS_KEY) as IReplStageState<*>
-    set(v) { put(KOTLIN_SCRIPT_STATE_BINDINGS_KEY, v) }
-
-val Bindings.kotlinScriptLineNumber: AtomicInteger
-    @Suppress("UNCHECKED_CAST")
-    get() = getOrPut(KOTLIN_SCRIPT_LINE_NUMBER_BINDINGS_KEY, { AtomicInteger(0) }) as AtomicInteger
 
 abstract class KotlinJsr223JvmScriptEngineBase(protected val myFactory: ScriptEngineFactory) : AbstractScriptEngine(), ScriptEngine, Compilable {
 
@@ -53,7 +42,7 @@ abstract class KotlinJsr223JvmScriptEngineBase(protected val myFactory: ScriptEn
     override fun getFactory(): ScriptEngineFactory = myFactory
 
     // the parameter could be used in the future when we decide to keep state completely in the context and solve appropriate problems (now e.g. replCompiler keeps separate state)
-    fun nextCodeLine(context: ScriptContext, code: String) = ReplCodeLine(context.getBindings(ScriptContext.ENGINE_SCOPE).kotlinScriptLineNumber.incrementAndGet(), code)
+    fun nextCodeLine(context: ScriptContext, code: String) = getCurrentState(context).let { ReplCodeLine(it.getNextLineNo(), it.currentGeneration, code) }
 
     protected open fun createState(lock: ReentrantReadWriteLock = ReentrantReadWriteLock()): IReplStageState<*> = AggregatedReplStageState(replCompiler.createState(lock), replEvaluator.createState(lock), lock)
 
@@ -83,7 +72,6 @@ abstract class KotlinJsr223JvmScriptEngineBase(protected val myFactory: ScriptEn
         val compiled = when (result) {
             is ReplCompileResult.Error -> throw ScriptException("Error${result.locationString()}: ${result.message}")
             is ReplCompileResult.Incomplete -> throw ScriptException("error: incomplete code")
-            is ReplCompileResult.HistoryMismatch -> throw ScriptException("Repl history mismatch at line: ${result.lineNo}")
             is ReplCompileResult.CompiledClasses -> result
         }
         return CompiledKotlinScript(this, codeLine, compiled)

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/repl/ReplApi.kt

@@ -23,7 +23,10 @@ import java.util.*
 import java.util.concurrent.locks.ReentrantReadWriteLock
 import kotlin.reflect.KClass
 
-data class ReplCodeLine(val no: Int, val code: String) : Serializable {
+const val REPL_CODE_LINE_FIRST_NO = 1
+const val REPL_CODE_LINE_FIRST_GEN = 1
+
+data class ReplCodeLine(val no: Int, val generation: Int, val code: String) : Serializable {
     companion object {
         private val serialVersionUID: Long = 8228357578L
     }
@@ -77,6 +80,7 @@ interface ReplCompileAction {
 
 sealed class ReplCompileResult : Serializable {
     class CompiledClasses(val lineId: LineId,
+                          val previousLines: List<ILineId>,
                           val mainClassName: String,
                           val classes: List<CompiledClassData>,
                           val hasResult: Boolean,
@@ -84,8 +88,6 @@ sealed class ReplCompileResult : Serializable {
 
     class Incomplete : ReplCompileResult()
 
-    class HistoryMismatch(val lineNo: Int) : ReplCompileResult()
-
     class Error(val message: String,
                 val location: CompilerMessageLocation = CompilerMessageLocation.NO_LOCATION) : ReplCompileResult() {
         override fun toString(): String = "Error(message = \"$message\""

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/repl/ReplState.kt

@@ -23,14 +23,13 @@ import kotlin.concurrent.write
 
 interface ILineId : Comparable<ILineId> {
     val no: Int
+    val generation: Int
 }
 
 data class ReplHistoryRecord<out T> (val id: ILineId, val item: T)
 
 interface IReplStageHistory<T> : List<ReplHistoryRecord<T>> {
 
-    fun backwardIterator(): Iterator<ReplHistoryRecord<T>> = lock.read { asReversed().iterator() } // TODO: check perf
-
     fun peek(): ReplHistoryRecord<T>? = lock.read { lastOrNull() }
 
     fun push(id: ILineId, item: T)
@@ -44,31 +43,6 @@ interface IReplStageHistory<T> : List<ReplHistoryRecord<T>> {
 
     fun resetTo(id: ILineId): Iterable<ILineId>
 
-    fun <OtherT> firstMismatch(other: IReplStageHistory<OtherT>): Pair<ReplHistoryRecord<T>?, ReplHistoryRecord<OtherT>?>? =
-            lock.read { other.lock.read {
-                iterator().asSequence().zip(other.asSequence()).firstOrNull { it.first.id != it.second.id }?.let { it.first to it.second }
-            } }
-
-    fun<OtherT> firstMismatchFiltered(other: IReplStageHistory<OtherT>, predicate: (ReplHistoryRecord<T>) -> Boolean): Pair<ReplHistoryRecord<T>?, ReplHistoryRecord<OtherT>?>? =
-            lock.read { other.lock.read {
-                iterator().asSequence().filter(predicate).zip(other.asSequence()).firstOrNull { it.first.id != it.second.id }?.let { it.first to it.second }
-            } }
-
-    fun<OtherT> firstMismatchWhile(other: IReplStageHistory<OtherT>, predicate: (ReplHistoryRecord<T>) -> Boolean): Pair<ReplHistoryRecord<T>?, ReplHistoryRecord<OtherT>?>? =
-            lock.read { other.lock.read {
-                iterator().asSequence().takeWhile(predicate).zip(other.asSequence()).firstOrNull { it.first.id != it.second.id }?.let { it.first to it.second }
-            } }
-
-    fun <OtherT> matches(other: IReplStageHistory<OtherT>): Boolean =
-            lock.read { other.lock.read {
-                size == other.size && firstMismatch(other) == null
-            } }
-
-    fun <OtherT> isProperPrefix(other: IReplStageHistory<OtherT>): Boolean =
-            lock.read { other.lock.read {
-                size == other.size - 1 && firstMismatch(other) == null
-            } }
-
     val lock: ReentrantReadWriteLock
 }
 
@@ -77,9 +51,28 @@ interface IReplStageState<T> {
 
     val lock: ReentrantReadWriteLock
 
+    val currentGeneration: Int
+
+    fun getNextLineNo(): Int = history.peek()?.id?.no?.let { it + 1 } ?: REPL_CODE_LINE_FIRST_NO // TODO: it should be more robust downstream (e.g. use atomic)
+
     fun <StateT : IReplStageState<*>> asState(target: Class<out StateT>): StateT =
             if (target.isAssignableFrom(this::class.java)) this as StateT
             else throw IllegalArgumentException("$this is not an expected instance of IReplStageState")
 }
 
 
+fun <T> IReplStageHistory<T>.firstMismatch(other: Sequence<ILineId>): Pair<ReplHistoryRecord<T>?, ILineId?>? =
+        lock.read {
+            iterator().asSequence().zip(other.asSequence()).firstOrNull { it.first.id != it.second }?.let { it.first to it.second }
+        }
+
+fun<T> IReplStageHistory<T>.firstMismatchFiltered(other: Sequence<ILineId>, predicate: (ReplHistoryRecord<T>) -> Boolean): Pair<ReplHistoryRecord<T>?, ILineId?>? =
+        lock.read {
+            iterator().asSequence().filter(predicate).zip(other.asSequence()).firstOrNull { it.first.id != it.second }?.let { it.first to it.second }
+        }
+
+fun<T> IReplStageHistory<T>.firstMismatchWhile(other: Sequence<ILineId>, predicate: (ReplHistoryRecord<T>) -> Boolean): Pair<ReplHistoryRecord<T>?, ILineId?>? =
+        lock.read {
+            iterator().asSequence().takeWhile(predicate).zip(other.asSequence()).firstOrNull { it.first.id != it.second }?.let { it.first to it.second }
+        }
+

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/repl/replUtil.kt

@@ -20,8 +20,8 @@ import com.google.common.base.Throwables
 import java.io.File
 import java.net.URLClassLoader
 
-fun makeScriptBaseName(codeLine: ReplCodeLine, generation: Long) =
-        "Line_${codeLine.no}${if (generation > 1) "_gen_$generation" else ""}"
+fun makeScriptBaseName(codeLine: ReplCodeLine) =
+        "Line_${codeLine.no}${if (codeLine.generation > REPL_CODE_LINE_FIRST_GEN) "_gen_${codeLine.generation}" else ""}"
 
 fun renderReplStackTrace(cause: Throwable, startFromMethodName: String): String {
     val newTrace = arrayListOf<StackTraceElement>()

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/repl/GenericCompilerState.kt

@@ -28,7 +28,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock
 class ReplCompilerStageHistory(private val state: GenericReplCompilerState) : BasicReplStageHistory<ScriptDescriptor>(state.lock) {
 
     override fun resetTo(id: ILineId): Iterable<ILineId> {
-        state.generation.incrementAndGet()
         val removedCompiledLines = super.resetTo(id)
         val removedAnalyzedLines = state.analyzerEngine.resetToLine(id.no)
 
@@ -49,13 +48,14 @@ abstract class GenericReplCheckerState: IReplStageState<ScriptDescriptor> {
             val psiFile: KtFile,
             val errorHolder: DiagnosticMessageHolder)
 
-    val generation = AtomicLong(1)
     var lastLineState: LineState? = null // for transferring state to the compiler in most typical case
 }
 
 class GenericReplCompilerState(environment: KotlinCoreEnvironment, override val lock: ReentrantReadWriteLock = ReentrantReadWriteLock()) : IReplStageState<ScriptDescriptor>, GenericReplCheckerState() {
 
-    override val history: IReplStageHistory<ScriptDescriptor> = ReplCompilerStageHistory(this)
+    override val history = ReplCompilerStageHistory(this)
+
+    override val currentGeneration: Int get() = (history as BasicReplStageHistory<*>).currentGeneration.get()
 
     val analyzerEngine = ReplCodeAnalyzer(environment)
 

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/repl/GenericReplChecker.kt

@@ -71,7 +71,7 @@ open class GenericReplChecker(
     override fun check(state: IReplStageState<*>, codeLine: ReplCodeLine): ReplCheckResult {
         state.lock.write {
             val checkerState = state.asState(GenericReplCheckerState::class.java)
-            val scriptFileName = makeScriptBaseName(codeLine, checkerState.generation.get())
+            val scriptFileName = makeScriptBaseName(codeLine)
             val virtualFile =
                     LightVirtualFile("$scriptFileName${KotlinParserDefinition.STD_SCRIPT_EXT}", KotlinLanguage.INSTANCE, StringUtil.convertLineSeparators(codeLine.code)).apply {
                         charset = CharsetToolkit.UTF8_CHARSET

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/repl/GenericReplCompiler.kt

@@ -49,8 +49,6 @@ open class GenericReplCompiler(disposable: Disposable,
         state.lock.write {
             val compilerState = state.asState(GenericReplCompilerState::class.java)
 
-            val currentGeneration = compilerState.generation.get()
-
             val (psiFile, errorHolder) = run {
                 if (compilerState.lastLineState == null || compilerState.lastLineState!!.codeLine != codeLine) {
                     val res = checker.check(state, codeLine)
@@ -95,10 +93,11 @@ open class GenericReplCompiler(disposable: Disposable,
                     setOf(psiFile.script!!.containingKtFile),
                     org.jetbrains.kotlin.codegen.CompilationErrorHandler.THROW_EXCEPTION)
 
-            val generatedClassname = makeScriptBaseName(codeLine, currentGeneration)
+            val generatedClassname = makeScriptBaseName(codeLine)
             compilerState.history.push(LineId(codeLine), scriptDescriptor)
 
             return ReplCompileResult.CompiledClasses(LineId(codeLine),
+                                                     compilerState.history.map { it.id },
                                                      generatedClassname,
                                                      generationState.factory.asList().map { CompiledClassData(it.relativePath, it.asByteArray()) },
                                                      generationState.replSpecific.hasResult,

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/repl/ReplInterpreter.kt

@@ -98,7 +98,7 @@ class ReplInterpreter(
             return LineResult.Error.CompileTime(errorHolder.renderedDiagnostics)
         }
 
-        val analysisResult = analyzerEngine.analyzeReplLine(psiFile, ReplCodeLine(lineNumber, "fake line"))
+        val analysisResult = analyzerEngine.analyzeReplLine(psiFile, ReplCodeLine(lineNumber, 0, "fake line"))
         AnalyzerWithCompilerReport.reportDiagnostics(analysisResult.diagnostics, errorHolder)
         val scriptDescriptor = when (analysisResult) {
             is ReplCodeAnalyzer.ReplLineAnalysisResult.WithErrors -> return LineResult.Error.CompileTime(errorHolder.renderedDiagnostics)