[JS SCRIPTING] create CoreCompiler for scripting

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/compiler.kt

@@ -13,11 +13,23 @@ import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.ir.backend.js.lower.moveBodilessDeclarationsToSeparatePlace
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformer
 import org.jetbrains.kotlin.ir.backend.js.utils.JsMainFunctionDetector
+import org.jetbrains.kotlin.ir.backend.js.utils.NameTables
+import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
 import org.jetbrains.kotlin.ir.util.ExternalDependenciesGenerator
 import org.jetbrains.kotlin.ir.util.patchDeclarationParents
 import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.utils.DFS
+
+fun sortDependencies(dependencies: Collection<IrModuleFragment>): Collection<IrModuleFragment> {
+    val mapping = dependencies.map { it.descriptor to it }.toMap()
+
+    return DFS.topologicalOrder(dependencies) { m ->
+        val descriptor = m.descriptor
+        descriptor.allDependencyModules.filter { it != descriptor }.map { mapping[it] }
+    }.reversed()
+}
 
 class CompilerResult(
     val jsCode: String,
@@ -73,3 +85,15 @@ fun compile(
     val transformer = IrModuleToJsTransformer(context, mainFunction, mainArguments)
     return transformer.generateModule(moduleFragment)
 }
+
+fun compileForRepl(
+    context: JsIrBackendContext,
+    moduleFragment: IrModuleFragment,
+    nameTables: NameTables
+): String {
+    moveBodilessDeclarationsToSeparatePlace(context, moduleFragment)
+    jsPhases.invokeToplevel(PhaseConfig(jsPhases), context, moduleFragment)
+
+    val transformer = IrModuleToJsTransformer(context, null, null, true, nameTables)
+    return transformer.generateModule(moduleFragment).jsCode
+}

compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt

@@ -222,13 +222,31 @@ private fun runAnalysisAndPreparePsi2Ir(depsDescriptors: ModulesStructure): Gene
     )
 }
 
-private fun GeneratorContext.generateModuleFragment(files: List<KtFile>, deserializer: JsIrLinker? = null) =
+fun GeneratorContext.generateModuleFragment(files: List<KtFile>, deserializer: IrDeserializer? = null) =
     Psi2IrTranslator(languageVersionSettings, configuration).generateModuleFragment(this, files, deserializer)
 
 
 private fun createBuiltIns(storageManager: StorageManager) = object : KotlinBuiltIns(storageManager) {}
 val JsFactories = KlibMetadataFactories(::createBuiltIns, DynamicTypeDeserializer)
 
+fun getModuleDescriptorByLibrary(
+    current: KotlinLibrary
+): ModuleDescriptorImpl {
+    val parts = loadKlibMetadataParts(current)
+    val isBuiltIns = parts.importedModules.isEmpty()
+    return loadKlibMetadata(
+        parts,
+        current,
+        isBuiltIns,
+        LookupTracker.DO_NOTHING,
+        LockBasedStorageManager("ModulesStructure"),
+        JsKlibMetadataVersion.INSTANCE,
+        LanguageVersionSettingsImpl.DEFAULT,
+        null,
+        emptyList()
+    )
+}
+
 private class ModulesStructure(
     private val project: Project,
     private val files: List<KtFile>,
@@ -298,7 +316,6 @@ private class ModulesStructure(
             getModuleDescriptor(builtInsDep)
         else
             null // null in case compiling builtInModule itself
-
 }
 
 private fun getDescriptorForElement(

libraries/scripting/jvm/src/kotlin/script/experimental/jvm/jvmScriptCompilation.kt

@@ -25,6 +25,8 @@ class JvmDependencyFromClassLoader(val classLoaderGetter: ClassLoaderByConfigura
     fun getClassLoader(configuration: ScriptCompilationConfiguration): ClassLoader = classLoaderGetter(configuration)
 }
 
+data class JsDependency(val path: String) : ScriptDependency
+
 interface JvmScriptCompilationConfigurationKeys
 
 open class JvmScriptCompilationConfigurationBuilder : PropertiesCollection.Builder(), JvmScriptCompilationConfigurationKeys {

plugins/scripting/scripting-compiler/build.gradle.kts

@@ -12,8 +12,11 @@ dependencies {
     compileOnly(project(":compiler:psi"))
     compileOnly(project(":compiler:plugin-api"))
     compileOnly(project(":compiler:cli"))
+    compileOnly(project(":compiler:backend.js"))
     compileOnly(project(":core:descriptors.runtime"))
     compile(project(":kotlin-scripting-common"))
+    compile(project(":kotlin-scripting-js"))
+    compile(project(":kotlin-util-klib"))
     compile(project(":kotlin-scripting-jvm"))
     compile(project(":kotlin-scripting-compiler-impl"))
     compile(kotlinStdlib())
@@ -25,6 +28,7 @@ dependencies {
     testCompile(project(":compiler:cli"))
     testCompile(project(":compiler:cli-common"))
     testCompile(project(":compiler:frontend.java"))
+    testCompile(project(":compiler:backend.js"))
     testCompile(projectTests(":compiler:tests-common"))
     testCompile(commonDep("junit:junit"))
 

plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/repl/js/CoreScriptingJsCompiler.kt

@@ -0,0 +1,115 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.scripting.repl.js
+
+import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
+import org.jetbrains.kotlin.cli.common.environment.setIdeaIoUseFallback
+import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport
+import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import org.jetbrains.kotlin.cli.common.repl.*
+import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
+import org.jetbrains.kotlin.config.languageVersionSettings
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.ir.backend.js.*
+import org.jetbrains.kotlin.ir.backend.js.utils.NameTables
+import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
+import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
+import org.jetbrains.kotlin.ir.util.ExternalDependenciesGenerator
+import org.jetbrains.kotlin.ir.util.IrDeserializer
+import org.jetbrains.kotlin.ir.util.SymbolTable
+import org.jetbrains.kotlin.js.config.JSConfigurationKeys
+import org.jetbrains.kotlin.psi2ir.Psi2IrTranslator
+import org.jetbrains.kotlin.serialization.js.ModuleKind
+import kotlin.script.experimental.api.valueOr
+import kotlin.script.experimental.host.StringScriptSource
+
+class DeserializerWithDependencies(val deserializer: IrDeserializer, val dependencies: List<IrModuleFragment>)
+
+class CoreScriptingJsCompiler(
+    private val environment: KotlinCoreEnvironment,
+    private val nameTables: NameTables,
+    private val dependencyDescriptors: List<ModuleDescriptor>,
+    private val createDeserializer: (ModuleDescriptor, SymbolTable, IrBuiltIns) -> DeserializerWithDependencies? = { _, _, _ -> null }
+) {
+    private val analyzerEngine: JsReplCodeAnalyzer = JsReplCodeAnalyzer(environment.project, dependencyDescriptors)
+    private val symbolTable: SymbolTable = SymbolTable()
+
+    fun compile(codeLine: ReplCodeLine): ReplCompileResult {
+        val snippet = codeLine.code
+        val snippetId = codeLine.no
+
+        val messageCollector = environment.configuration[CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY] as MessageCollector
+
+        setIdeaIoUseFallback()
+
+        val sourceCode = StringScriptSource(snippet, "line-$snippetId.kts")
+        val snippetKtFile = getScriptKtFile(
+            sourceCode,
+            snippet,
+            environment.project
+        ).valueOr { return ReplCompileResult.Error(it.reports.joinToString { r -> r.message }) }
+
+        analyzerEngine.analyzeReplLine(snippetKtFile, codeLine).also {
+            AnalyzerWithCompilerReport.reportDiagnostics(it, messageCollector)
+            if (messageCollector.hasErrors()) return ReplCompileResult.Error("Error while analysis")
+        }
+
+        val psi2ir = Psi2IrTranslator(environment.configuration.languageVersionSettings)
+        val psi2irContext = psi2ir.createGeneratorContext(
+            analyzerEngine.context.module,
+            analyzerEngine.trace.bindingContext,
+            symbolTable
+        )
+
+        val deserializerWithDependencies = createDeserializer(psi2irContext.moduleDescriptor, symbolTable, psi2irContext.irBuiltIns)
+
+        val irModuleFragment = psi2irContext.generateModuleFragment(listOf(snippetKtFile), deserializerWithDependencies?.deserializer)
+
+        val deserializedFragments = deserializerWithDependencies?.dependencies ?: emptyList()
+
+        val irFiles = sortDependencies(deserializedFragments).flatMap { it.files } + irModuleFragment.files
+        irModuleFragment.files.clear()
+        irModuleFragment.files += irFiles
+
+
+        val context = JsIrBackendContext(
+            irModuleFragment.descriptor,
+            psi2irContext.irBuiltIns,
+            psi2irContext.symbolTable,
+            irModuleFragment,
+            emptySet(),
+            environment.configuration,
+            true
+        )
+
+        ExternalDependenciesGenerator(
+            irModuleFragment.descriptor,
+            psi2irContext.symbolTable,
+            psi2irContext.irBuiltIns,
+            deserializer = deserializerWithDependencies?.deserializer
+        ).generateUnboundSymbolsAsDependencies()
+
+        with(context.implicitDeclarationFile) {
+            if (!irModuleFragment.files.contains(this)) {
+                irModuleFragment.files += this
+            }
+        }
+        context.implicitDeclarationFile.declarations.clear()
+
+        environment.configuration.put(JSConfigurationKeys.MODULE_KIND, ModuleKind.PLAIN)
+
+        val code = compileForRepl(
+            context,
+            irModuleFragment,
+            nameTables
+        )
+
+        return createCompileResult(
+            LineId(codeLine),
+            code
+        )
+    }
+}

plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/repl/js/JsReplCodeAnalyzer.kt

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.scripting.repl.js
+
+import com.intellij.openapi.project.Project
+import org.jetbrains.kotlin.builtins.KotlinBuiltIns
+import org.jetbrains.kotlin.cli.common.repl.ReplCodeLine
+import org.jetbrains.kotlin.cli.jvm.compiler.NoScopeRecordCliBindingTrace
+import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl
+import org.jetbrains.kotlin.context.ContextForNewModule
+import org.jetbrains.kotlin.context.MutableModuleContext
+import org.jetbrains.kotlin.context.ProjectContext
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
+import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
+import org.jetbrains.kotlin.diagnostics.Severity
+import org.jetbrains.kotlin.frontend.js.di.createTopDownAnalyzerForJs
+import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
+import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.resolve.BindingTraceContext
+import org.jetbrains.kotlin.resolve.LazyTopDownAnalyzer
+import org.jetbrains.kotlin.resolve.TopDownAnalysisMode
+import org.jetbrains.kotlin.resolve.diagnostics.Diagnostics
+import org.jetbrains.kotlin.resolve.lazy.declarations.FileBasedDeclarationProviderFactory
+import org.jetbrains.kotlin.scripting.compiler.plugin.repl.ReplCodeAnalyzer
+import org.jetbrains.kotlin.scripting.definitions.ScriptPriorities
+
+class JsReplCodeAnalyzer(private val project: Project, private val dependencies: List<ModuleDescriptor>) {
+
+    private val replState = ReplCodeAnalyzer.ResettableAnalyzerState()
+    val trace: BindingTraceContext = NoScopeRecordCliBindingTrace()
+
+    private val builtIns: KotlinBuiltIns = dependencies.single { it.allDependencyModules.isEmpty() }.builtIns
+    lateinit var context: MutableModuleContext
+    private fun createTopDownAnalyzerJS(files: Collection<KtFile>): LazyTopDownAnalyzer {
+        context = ContextForNewModule(
+            ProjectContext(project, "TopDownAnalyzer for JS"),
+            Name.special("<script>"),
+            builtIns,
+            platform = null
+        )
+
+        val languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT
+        val lookupTracker = LookupTracker.DO_NOTHING
+        val expectActualTracker = ExpectActualTracker.DoNothing
+        val additionalPackages = mutableListOf<PackageFragmentProvider>()
+
+        context.module.setDependencies(dependencies.map { it as ModuleDescriptorImpl } + context.module)
+
+        return createTopDownAnalyzerForJs(
+            context, trace,
+            FileBasedDeclarationProviderFactory(context.storageManager, files),
+            languageVersionSettings,
+            lookupTracker,
+            expectActualTracker,
+            additionalPackages
+        )
+    }
+
+    fun analyzeReplLine(linePsi: KtFile, codeLine: ReplCodeLine): Diagnostics {
+        trace.clearDiagnostics()
+
+        linePsi.script!!.putUserData(ScriptPriorities.PRIORITY_KEY, codeLine.no)
+
+        replState.submitLine(linePsi, codeLine)
+
+        val analyzer = createTopDownAnalyzerJS(listOf(linePsi))
+        val context = analyzer.analyzeDeclarations(TopDownAnalysisMode.TopLevelDeclarations, listOf(linePsi))
+
+        val diagnostics = trace.bindingContext.diagnostics
+        val hasErrors = diagnostics.any { it.severity == Severity.ERROR }
+
+        if (hasErrors) {
+            replState.lineFailure(linePsi, codeLine)
+        } else {
+            val scriptDescriptor = context.scripts[linePsi.script]!!
+            replState.lineSuccess(linePsi, codeLine, scriptDescriptor)
+        }
+        return diagnostics
+    }
+}

plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/repl/js/JsReplUtils.kt

@@ -0,0 +1,182 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.scripting.repl.js
+
+import com.intellij.openapi.project.Project
+import com.intellij.psi.PsiFileFactory
+import com.intellij.psi.impl.PsiFileFactoryImpl
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import org.jetbrains.kotlin.cli.common.repl.*
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.idea.KotlinLanguage
+import org.jetbrains.kotlin.ir.backend.js.loadKlib
+import org.jetbrains.kotlin.ir.backend.js.getModuleDescriptorByLibrary
+import org.jetbrains.kotlin.ir.backend.js.utils.NameTables
+import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.psi.KtScript
+import org.jetbrains.kotlin.scripting.configuration.ScriptingConfigurationKeys
+import org.jetbrains.kotlin.scripting.resolve.ScriptLightVirtualFile
+import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
+import java.io.*
+import java.nio.charset.Charset
+import kotlin.script.experimental.api.*
+import kotlin.script.experimental.host.FileBasedScriptSource
+import kotlin.script.experimental.jvm.JsDependency
+
+fun getScriptKtFile(
+    script: SourceCode,
+    scriptText: String,
+    project: Project
+): ResultWithDiagnostics<KtFile> {
+    val psiFileFactory: PsiFileFactoryImpl = PsiFileFactory.getInstance(project) as PsiFileFactoryImpl
+    val virtualFile = ScriptLightVirtualFile(
+        script.name!!,
+        (script as? FileBasedScriptSource)?.file?.path,
+        scriptText
+    )
+    val ktFile = psiFileFactory.trySetupPsiForFile(virtualFile, KotlinLanguage.INSTANCE, true, false) as KtFile?
+    return when {
+        ktFile == null -> ResultWithDiagnostics.Failure(
+            ScriptDiagnostic(
+                message = "Cannot create PSI",
+                severity = ScriptDiagnostic.Severity.ERROR
+            )
+        )
+        ktFile.declarations.firstIsInstanceOrNull<KtScript>() == null -> ResultWithDiagnostics.Failure(
+            ScriptDiagnostic(
+                message = "There is not Script",
+                severity = ScriptDiagnostic.Severity.ERROR
+            )
+        )
+        else -> ktFile.asSuccess()
+    }
+}
+
+fun makeReplCodeLine(no: Int, code: String): ReplCodeLine = ReplCodeLine(no, 0, code)
+
+//TODO: remove and use collector from kotlin-scripting-compiler
+class ReplMessageCollector : MessageCollector {
+    private var hasErrors = false
+    private var messages = mutableListOf<Pair<CompilerMessageSeverity, String>>()
+
+    override fun clear() {
+        hasErrors = false
+        messages.clear()
+    }
+
+    override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation?) {
+        if (severity == CompilerMessageSeverity.ERROR) hasErrors = true
+        messages.add(Pair(severity, message))
+    }
+
+    override fun hasErrors(): Boolean {
+        return hasErrors
+    }
+
+    fun hasNotErrors(): Boolean {
+        return !hasErrors
+    }
+
+    fun getMessage(): String {
+        val resultMessage = StringBuilder("Found ${messages.size} problems:\n")
+        for (m in messages) {
+            resultMessage.append(m.first.toString() + " : " + m.second + "\n")
+        }
+        return resultMessage.toString()
+    }
+}
+
+fun readLibrariesFromConfiguration(configuration: CompilerConfiguration): List<ModuleDescriptor> {
+    val scriptConfig = configuration[ScriptingConfigurationKeys.SCRIPT_DEFINITIONS]!!
+    val scriptCompilationConfig = scriptConfig.find { (it).platform == "JS" }!!.compilationConfiguration
+    val scriptDependencies = scriptCompilationConfig[ScriptCompilationConfiguration.dependencies]!!
+    return scriptDependencies.map { loadKlib((it as JsDependency).path) }.map { getModuleDescriptorByLibrary(it) }
+}
+
+fun createCompileResult(code: String) = createCompileResult(LineId(ReplCodeLine(0, 0, "")), code)
+
+fun createCompileResult(lineId: LineId, code: String): ReplCompileResult.CompiledClasses {
+    return ReplCompileResult.CompiledClasses(
+        lineId,
+        emptyList(),
+        "",
+        emptyList(),
+        false,
+        emptyList(),
+        "Any?",
+        code
+    )
+}
+
+class DependencyLoader {
+    private val commonPath = "compiler/ir/serialization.js/build/fullRuntime/klib"
+    private val mappedNamesPath = "$commonPath/mappedNames.txt"
+    private val scriptDependencyBinaryPath = "$commonPath/scriptDependencyBinary.js"
+
+    fun saveNames(nameTables: NameTables, path: String = mappedNamesPath) {
+        writeDataByPath(writeNames(nameTables), path)
+    }
+
+    fun loadNames(path: String = mappedNamesPath): NameTables {
+        return readNames(readDataByPath(path))
+    }
+
+    fun saveScriptDependencyBinary(stdlibCompiledResult: String, path: String = scriptDependencyBinaryPath) {
+        writeDataByPath(writeScriptDependencyBinary(stdlibCompiledResult), path)
+    }
+
+    fun loadScriptDependencyBinary(path: String = scriptDependencyBinaryPath): String {
+        return readScriptDependencyBinary(readDataByPath(path))
+    }
+
+
+    fun writeNames(nameTables: NameTables): ByteArray {
+        val result = StringBuilder()
+        for (entry in nameTables.mappedNames) {
+            result.append("${entry.key} ${entry.value}" + System.lineSeparator())
+        }
+        return result.toString().toByteArray(Charset.defaultCharset())
+    }
+
+    fun readNames(data: ByteArray): NameTables {
+        val mappedNames = mutableMapOf<String, String>()
+        val reserved = mutableSetOf<String>()
+
+        BufferedReader(InputStreamReader(data.inputStream())).use { reader ->
+            for (line in reader.readLines()) {
+                val (key, value) = line.split(" ")
+                mappedNames[key] = value
+                reserved += value
+            }
+        }
+
+        return NameTables(emptyList(), mappedNames = mappedNames, reservedForGlobal = reserved)
+    }
+
+    fun writeScriptDependencyBinary(stdlibCompiledResult: String): ByteArray {
+        return stdlibCompiledResult.toByteArray(Charset.defaultCharset())
+    }
+
+    fun readScriptDependencyBinary(data: ByteArray): String {
+        return data.toString(Charset.defaultCharset())
+    }
+
+    fun readDataByPath(path: String): ByteArray {
+        FileReader(path).use { reader ->
+            val stdlibCompiledResult = reader.readText()
+            return stdlibCompiledResult.toByteArray(Charset.defaultCharset())
+        }
+    }
+
+    fun writeDataByPath(data: ByteArray, path: String) {
+        FileOutputStream(path).use {
+            it.write(data)
+        }
+    }
+}