[JS SCRIPTING] create CoreCompiler for scripting

This commit is contained in:
Vitaliy.Tikhonov
2019-08-30 14:56:26 +03:00
committed by romanart
parent cf5a1615ea
commit d79279d8a5
7 changed files with 433 additions and 3 deletions
@@ -13,11 +13,23 @@ import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.ir.backend.js.lower.moveBodilessDeclarationsToSeparatePlace
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformer
import org.jetbrains.kotlin.ir.backend.js.utils.JsMainFunctionDetector
import org.jetbrains.kotlin.ir.backend.js.utils.NameTables
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
import org.jetbrains.kotlin.ir.util.ExternalDependenciesGenerator
import org.jetbrains.kotlin.ir.util.patchDeclarationParents
import org.jetbrains.kotlin.library.KotlinLibrary
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.utils.DFS
fun sortDependencies(dependencies: Collection<IrModuleFragment>): Collection<IrModuleFragment> {
val mapping = dependencies.map { it.descriptor to it }.toMap()
return DFS.topologicalOrder(dependencies) { m ->
val descriptor = m.descriptor
descriptor.allDependencyModules.filter { it != descriptor }.map { mapping[it] }
}.reversed()
}
class CompilerResult(
val jsCode: String,
@@ -72,4 +84,16 @@ fun compile(
val transformer = IrModuleToJsTransformer(context, mainFunction, mainArguments)
return transformer.generateModule(moduleFragment)
}
}
fun compileForRepl(
context: JsIrBackendContext,
moduleFragment: IrModuleFragment,
nameTables: NameTables
): String {
moveBodilessDeclarationsToSeparatePlace(context, moduleFragment)
jsPhases.invokeToplevel(PhaseConfig(jsPhases), context, moduleFragment)
val transformer = IrModuleToJsTransformer(context, null, null, true, nameTables)
return transformer.generateModule(moduleFragment).jsCode
}
@@ -222,13 +222,31 @@ private fun runAnalysisAndPreparePsi2Ir(depsDescriptors: ModulesStructure): Gene
)
}
private fun GeneratorContext.generateModuleFragment(files: List<KtFile>, deserializer: JsIrLinker? = null) =
fun GeneratorContext.generateModuleFragment(files: List<KtFile>, deserializer: IrDeserializer? = null) =
Psi2IrTranslator(languageVersionSettings, configuration).generateModuleFragment(this, files, deserializer)
private fun createBuiltIns(storageManager: StorageManager) = object : KotlinBuiltIns(storageManager) {}
val JsFactories = KlibMetadataFactories(::createBuiltIns, DynamicTypeDeserializer)
fun getModuleDescriptorByLibrary(
current: KotlinLibrary
): ModuleDescriptorImpl {
val parts = loadKlibMetadataParts(current)
val isBuiltIns = parts.importedModules.isEmpty()
return loadKlibMetadata(
parts,
current,
isBuiltIns,
LookupTracker.DO_NOTHING,
LockBasedStorageManager("ModulesStructure"),
JsKlibMetadataVersion.INSTANCE,
LanguageVersionSettingsImpl.DEFAULT,
null,
emptyList()
)
}
private class ModulesStructure(
private val project: Project,
private val files: List<KtFile>,
@@ -298,7 +316,6 @@ private class ModulesStructure(
getModuleDescriptor(builtInsDep)
else
null // null in case compiling builtInModule itself
}
private fun getDescriptorForElement(
@@ -25,6 +25,8 @@ class JvmDependencyFromClassLoader(val classLoaderGetter: ClassLoaderByConfigura
fun getClassLoader(configuration: ScriptCompilationConfiguration): ClassLoader = classLoaderGetter(configuration)
}
data class JsDependency(val path: String) : ScriptDependency
interface JvmScriptCompilationConfigurationKeys
open class JvmScriptCompilationConfigurationBuilder : PropertiesCollection.Builder(), JvmScriptCompilationConfigurationKeys {
@@ -12,8 +12,11 @@ dependencies {
compileOnly(project(":compiler:psi"))
compileOnly(project(":compiler:plugin-api"))
compileOnly(project(":compiler:cli"))
compileOnly(project(":compiler:backend.js"))
compileOnly(project(":core:descriptors.runtime"))
compile(project(":kotlin-scripting-common"))
compile(project(":kotlin-scripting-js"))
compile(project(":kotlin-util-klib"))
compile(project(":kotlin-scripting-jvm"))
compile(project(":kotlin-scripting-compiler-impl"))
compile(kotlinStdlib())
@@ -25,6 +28,7 @@ dependencies {
testCompile(project(":compiler:cli"))
testCompile(project(":compiler:cli-common"))
testCompile(project(":compiler:frontend.java"))
testCompile(project(":compiler:backend.js"))
testCompile(projectTests(":compiler:tests-common"))
testCompile(commonDep("junit:junit"))
@@ -0,0 +1,115 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.scripting.repl.js
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
import org.jetbrains.kotlin.cli.common.environment.setIdeaIoUseFallback
import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.cli.common.repl.*
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
import org.jetbrains.kotlin.config.languageVersionSettings
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.ir.backend.js.*
import org.jetbrains.kotlin.ir.backend.js.utils.NameTables
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
import org.jetbrains.kotlin.ir.util.ExternalDependenciesGenerator
import org.jetbrains.kotlin.ir.util.IrDeserializer
import org.jetbrains.kotlin.ir.util.SymbolTable
import org.jetbrains.kotlin.js.config.JSConfigurationKeys
import org.jetbrains.kotlin.psi2ir.Psi2IrTranslator
import org.jetbrains.kotlin.serialization.js.ModuleKind
import kotlin.script.experimental.api.valueOr
import kotlin.script.experimental.host.StringScriptSource
class DeserializerWithDependencies(val deserializer: IrDeserializer, val dependencies: List<IrModuleFragment>)
class CoreScriptingJsCompiler(
private val environment: KotlinCoreEnvironment,
private val nameTables: NameTables,
private val dependencyDescriptors: List<ModuleDescriptor>,
private val createDeserializer: (ModuleDescriptor, SymbolTable, IrBuiltIns) -> DeserializerWithDependencies? = { _, _, _ -> null }
) {
private val analyzerEngine: JsReplCodeAnalyzer = JsReplCodeAnalyzer(environment.project, dependencyDescriptors)
private val symbolTable: SymbolTable = SymbolTable()
fun compile(codeLine: ReplCodeLine): ReplCompileResult {
val snippet = codeLine.code
val snippetId = codeLine.no
val messageCollector = environment.configuration[CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY] as MessageCollector
setIdeaIoUseFallback()
val sourceCode = StringScriptSource(snippet, "line-$snippetId.kts")
val snippetKtFile = getScriptKtFile(
sourceCode,
snippet,
environment.project
).valueOr { return ReplCompileResult.Error(it.reports.joinToString { r -> r.message }) }
analyzerEngine.analyzeReplLine(snippetKtFile, codeLine).also {
AnalyzerWithCompilerReport.reportDiagnostics(it, messageCollector)
if (messageCollector.hasErrors()) return ReplCompileResult.Error("Error while analysis")
}
val psi2ir = Psi2IrTranslator(environment.configuration.languageVersionSettings)
val psi2irContext = psi2ir.createGeneratorContext(
analyzerEngine.context.module,
analyzerEngine.trace.bindingContext,
symbolTable
)
val deserializerWithDependencies = createDeserializer(psi2irContext.moduleDescriptor, symbolTable, psi2irContext.irBuiltIns)
val irModuleFragment = psi2irContext.generateModuleFragment(listOf(snippetKtFile), deserializerWithDependencies?.deserializer)
val deserializedFragments = deserializerWithDependencies?.dependencies ?: emptyList()
val irFiles = sortDependencies(deserializedFragments).flatMap { it.files } + irModuleFragment.files
irModuleFragment.files.clear()
irModuleFragment.files += irFiles
val context = JsIrBackendContext(
irModuleFragment.descriptor,
psi2irContext.irBuiltIns,
psi2irContext.symbolTable,
irModuleFragment,
emptySet(),
environment.configuration,
true
)
ExternalDependenciesGenerator(
irModuleFragment.descriptor,
psi2irContext.symbolTable,
psi2irContext.irBuiltIns,
deserializer = deserializerWithDependencies?.deserializer
).generateUnboundSymbolsAsDependencies()
with(context.implicitDeclarationFile) {
if (!irModuleFragment.files.contains(this)) {
irModuleFragment.files += this
}
}
context.implicitDeclarationFile.declarations.clear()
environment.configuration.put(JSConfigurationKeys.MODULE_KIND, ModuleKind.PLAIN)
val code = compileForRepl(
context,
irModuleFragment,
nameTables
)
return createCompileResult(
LineId(codeLine),
code
)
}
}
@@ -0,0 +1,86 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.scripting.repl.js
import com.intellij.openapi.project.Project
import org.jetbrains.kotlin.builtins.KotlinBuiltIns
import org.jetbrains.kotlin.cli.common.repl.ReplCodeLine
import org.jetbrains.kotlin.cli.jvm.compiler.NoScopeRecordCliBindingTrace
import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl
import org.jetbrains.kotlin.context.ContextForNewModule
import org.jetbrains.kotlin.context.MutableModuleContext
import org.jetbrains.kotlin.context.ProjectContext
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.diagnostics.Severity
import org.jetbrains.kotlin.frontend.js.di.createTopDownAnalyzerForJs
import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.resolve.BindingTraceContext
import org.jetbrains.kotlin.resolve.LazyTopDownAnalyzer
import org.jetbrains.kotlin.resolve.TopDownAnalysisMode
import org.jetbrains.kotlin.resolve.diagnostics.Diagnostics
import org.jetbrains.kotlin.resolve.lazy.declarations.FileBasedDeclarationProviderFactory
import org.jetbrains.kotlin.scripting.compiler.plugin.repl.ReplCodeAnalyzer
import org.jetbrains.kotlin.scripting.definitions.ScriptPriorities
class JsReplCodeAnalyzer(private val project: Project, private val dependencies: List<ModuleDescriptor>) {
private val replState = ReplCodeAnalyzer.ResettableAnalyzerState()
val trace: BindingTraceContext = NoScopeRecordCliBindingTrace()
private val builtIns: KotlinBuiltIns = dependencies.single { it.allDependencyModules.isEmpty() }.builtIns
lateinit var context: MutableModuleContext
private fun createTopDownAnalyzerJS(files: Collection<KtFile>): LazyTopDownAnalyzer {
context = ContextForNewModule(
ProjectContext(project, "TopDownAnalyzer for JS"),
Name.special("<script>"),
builtIns,
platform = null
)
val languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT
val lookupTracker = LookupTracker.DO_NOTHING
val expectActualTracker = ExpectActualTracker.DoNothing
val additionalPackages = mutableListOf<PackageFragmentProvider>()
context.module.setDependencies(dependencies.map { it as ModuleDescriptorImpl } + context.module)
return createTopDownAnalyzerForJs(
context, trace,
FileBasedDeclarationProviderFactory(context.storageManager, files),
languageVersionSettings,
lookupTracker,
expectActualTracker,
additionalPackages
)
}
fun analyzeReplLine(linePsi: KtFile, codeLine: ReplCodeLine): Diagnostics {
trace.clearDiagnostics()
linePsi.script!!.putUserData(ScriptPriorities.PRIORITY_KEY, codeLine.no)
replState.submitLine(linePsi, codeLine)
val analyzer = createTopDownAnalyzerJS(listOf(linePsi))
val context = analyzer.analyzeDeclarations(TopDownAnalysisMode.TopLevelDeclarations, listOf(linePsi))
val diagnostics = trace.bindingContext.diagnostics
val hasErrors = diagnostics.any { it.severity == Severity.ERROR }
if (hasErrors) {
replState.lineFailure(linePsi, codeLine)
} else {
val scriptDescriptor = context.scripts[linePsi.script]!!
replState.lineSuccess(linePsi, codeLine, scriptDescriptor)
}
return diagnostics
}
}
@@ -0,0 +1,182 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.scripting.repl.js
import com.intellij.openapi.project.Project
import com.intellij.psi.PsiFileFactory
import com.intellij.psi.impl.PsiFileFactoryImpl
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.cli.common.repl.*
import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.idea.KotlinLanguage
import org.jetbrains.kotlin.ir.backend.js.loadKlib
import org.jetbrains.kotlin.ir.backend.js.getModuleDescriptorByLibrary
import org.jetbrains.kotlin.ir.backend.js.utils.NameTables
import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.psi.KtScript
import org.jetbrains.kotlin.scripting.configuration.ScriptingConfigurationKeys
import org.jetbrains.kotlin.scripting.resolve.ScriptLightVirtualFile
import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
import java.io.*
import java.nio.charset.Charset
import kotlin.script.experimental.api.*
import kotlin.script.experimental.host.FileBasedScriptSource
import kotlin.script.experimental.jvm.JsDependency
fun getScriptKtFile(
script: SourceCode,
scriptText: String,
project: Project
): ResultWithDiagnostics<KtFile> {
val psiFileFactory: PsiFileFactoryImpl = PsiFileFactory.getInstance(project) as PsiFileFactoryImpl
val virtualFile = ScriptLightVirtualFile(
script.name!!,
(script as? FileBasedScriptSource)?.file?.path,
scriptText
)
val ktFile = psiFileFactory.trySetupPsiForFile(virtualFile, KotlinLanguage.INSTANCE, true, false) as KtFile?
return when {
ktFile == null -> ResultWithDiagnostics.Failure(
ScriptDiagnostic(
message = "Cannot create PSI",
severity = ScriptDiagnostic.Severity.ERROR
)
)
ktFile.declarations.firstIsInstanceOrNull<KtScript>() == null -> ResultWithDiagnostics.Failure(
ScriptDiagnostic(
message = "There is not Script",
severity = ScriptDiagnostic.Severity.ERROR
)
)
else -> ktFile.asSuccess()
}
}
fun makeReplCodeLine(no: Int, code: String): ReplCodeLine = ReplCodeLine(no, 0, code)
//TODO: remove and use collector from kotlin-scripting-compiler
class ReplMessageCollector : MessageCollector {
private var hasErrors = false
private var messages = mutableListOf<Pair<CompilerMessageSeverity, String>>()
override fun clear() {
hasErrors = false
messages.clear()
}
override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation?) {
if (severity == CompilerMessageSeverity.ERROR) hasErrors = true
messages.add(Pair(severity, message))
}
override fun hasErrors(): Boolean {
return hasErrors
}
fun hasNotErrors(): Boolean {
return !hasErrors
}
fun getMessage(): String {
val resultMessage = StringBuilder("Found ${messages.size} problems:\n")
for (m in messages) {
resultMessage.append(m.first.toString() + " : " + m.second + "\n")
}
return resultMessage.toString()
}
}
fun readLibrariesFromConfiguration(configuration: CompilerConfiguration): List<ModuleDescriptor> {
val scriptConfig = configuration[ScriptingConfigurationKeys.SCRIPT_DEFINITIONS]!!
val scriptCompilationConfig = scriptConfig.find { (it).platform == "JS" }!!.compilationConfiguration
val scriptDependencies = scriptCompilationConfig[ScriptCompilationConfiguration.dependencies]!!
return scriptDependencies.map { loadKlib((it as JsDependency).path) }.map { getModuleDescriptorByLibrary(it) }
}
fun createCompileResult(code: String) = createCompileResult(LineId(ReplCodeLine(0, 0, "")), code)
fun createCompileResult(lineId: LineId, code: String): ReplCompileResult.CompiledClasses {
return ReplCompileResult.CompiledClasses(
lineId,
emptyList(),
"",
emptyList(),
false,
emptyList(),
"Any?",
code
)
}
class DependencyLoader {
private val commonPath = "compiler/ir/serialization.js/build/fullRuntime/klib"
private val mappedNamesPath = "$commonPath/mappedNames.txt"
private val scriptDependencyBinaryPath = "$commonPath/scriptDependencyBinary.js"
fun saveNames(nameTables: NameTables, path: String = mappedNamesPath) {
writeDataByPath(writeNames(nameTables), path)
}
fun loadNames(path: String = mappedNamesPath): NameTables {
return readNames(readDataByPath(path))
}
fun saveScriptDependencyBinary(stdlibCompiledResult: String, path: String = scriptDependencyBinaryPath) {
writeDataByPath(writeScriptDependencyBinary(stdlibCompiledResult), path)
}
fun loadScriptDependencyBinary(path: String = scriptDependencyBinaryPath): String {
return readScriptDependencyBinary(readDataByPath(path))
}
fun writeNames(nameTables: NameTables): ByteArray {
val result = StringBuilder()
for (entry in nameTables.mappedNames) {
result.append("${entry.key} ${entry.value}" + System.lineSeparator())
}
return result.toString().toByteArray(Charset.defaultCharset())
}
fun readNames(data: ByteArray): NameTables {
val mappedNames = mutableMapOf<String, String>()
val reserved = mutableSetOf<String>()
BufferedReader(InputStreamReader(data.inputStream())).use { reader ->
for (line in reader.readLines()) {
val (key, value) = line.split(" ")
mappedNames[key] = value
reserved += value
}
}
return NameTables(emptyList(), mappedNames = mappedNames, reservedForGlobal = reserved)
}
fun writeScriptDependencyBinary(stdlibCompiledResult: String): ByteArray {
return stdlibCompiledResult.toByteArray(Charset.defaultCharset())
}
fun readScriptDependencyBinary(data: ByteArray): String {
return data.toString(Charset.defaultCharset())
}
fun readDataByPath(path: String): ByteArray {
FileReader(path).use { reader ->
val stdlibCompiledResult = reader.readText()
return stdlibCompiledResult.toByteArray(Charset.defaultCharset())
}
}
fun writeDataByPath(data: ByteArray, path: String) {
FileOutputStream(path).use {
it.write(data)
}
}
}