azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[JS IR] tie together IC code in tests and K2JsIrCompiler

Browse Source
This commit is contained in:
Anton Bannykh
2021-11-12 13:59:41 +03:00
committed by TeamCityServer
parent ff02ea5840
commit 75368a2c06
7 changed files with 121 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt
+25 -4
View File
@@ -77,7 +77,7 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
}
@Suppress("UNUSED_PARAMETER")
private fun usePerFileInvalidator(configuration: CompilerConfiguration): Boolean = false
private fun usePerFileInvalidator(configuration: CompilerConfiguration): Boolean = true
private fun IcCacheInfo.toICCacheMap(): Map<String, ICCache> {
return data.map { it.key to ICCache(PersistentCacheProvider.EMPTY, PersistentCacheConsumer.EMPTY, it.value) }.toMap()
@@ -212,12 +212,12 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
MainModule.Klib(includes)
}
val perFileCache = usePerFileInvalidator(configuration)
val perFileCache = usePerFileInvalidator(configurationJs)
val start = System.currentTimeMillis()
val updated = if (perFileCache) {
actualizeCacheForModule(includes, outputFilePath, configuration, libraries, icCaches, IrFactoryImpl)
actualizeCacheForModule(includes, outputFilePath, configurationJs, libraries, icCaches, IrFactoryImpl)
} else {
buildCache(
cachePath = outputFilePath,
@@ -282,6 +282,27 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
messageCollector.report(INFO,"Produce executable: $outputFilePath")
messageCollector.report(INFO, arguments.cacheDirectories ?: "")
if (icCaches.isNotEmpty()) {
val beforeIc2Js = System.currentTimeMillis()
val caches = loadModuleCaches(icCaches)
val moduleKind = configurationJs[JSConfigurationKeys.MODULE_KIND]!!
val compiledModule = generateJsFromAst(moduleName, moduleKind, caches)
val outputs = compiledModule.outputs!!
outputFile.write(outputs)
outputs.dependencies.forEach { (name, content) ->
outputFile.resolveSibling("$name.js").write(content)
}
messageCollector.report(INFO, "Executable production duration (IC): ${System.currentTimeMillis() - beforeIc2Js}ms")
return OK
}
val phaseConfig = createPhaseConfig(jsPhases, arguments, messageCollector)
val module = if (includes != null) {
@@ -378,7 +399,7 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
arguments.irSafeExternalBooleanDiagnostic,
messageCollector
),
lowerPerModule = icCaches.isNotEmpty(),
lowerPerModule = false,//icCaches.isNotEmpty(),
granularity = granularity,
icCompatibleIr2Js = arguments.irNewIr2Js,
)
compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/compilerWithIC.kt
+6 -5
View File
@@ -126,15 +126,16 @@ fun lowerPreservingTags(modules: Iterable<IrModuleFragment>, context: JsIrBacken
@Suppress("UNUSED_PARAMETER")
fun generateJsFromAst(
mainModule: String,
caches: Map<String, ModuleCache>
mainModuleName: String,
moduleKind: ModuleKind,
caches: Map<String, ModuleCache>,
): CompilerResult {
val deserializer = JsIrAstDeserializer()
val fragments = JsIrProgram(caches.values.map { JsIrModule(it.name, it.name, it.asts.values.mapNotNull { it.ast?.let { deserializer.deserialize(ByteArrayInputStream(it))} }) })
val fragments = JsIrProgram(caches.values.map { JsIrModule(it.name, it.name, it.asts.values.sortedBy { it.name }.mapNotNull { it.ast?.let { deserializer.deserialize(ByteArrayInputStream(it))} }) })
return CompilerResult(
generateSingleWrappedModuleBody(
"main",
ModuleKind.PLAIN,
mainModuleName,
moduleKind,
fragments.modules.flatMap { it.fragments },
sourceMapsInfo = null,
generateScriptModule = false,
compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/invalidation.kt
+31 -4
View File
@@ -191,7 +191,7 @@ private fun buildCacheForModule(
}
// TODO: actual way of building a cache could change in future
buildCacheForModuleFiles(irModule, dependencies, deserializer, configuration, dirtyFiles, cacheConsumer)
buildCacheForModuleFiles(irModule, dependencies, deserializer, configuration, dirtyFiles, cacheConsumer, emptySet(), null) // TODO: main arguments?
cacheConsumer.commitLibraryPath(libraryPath)
}
@@ -434,6 +434,8 @@ fun rebuildCacheForDirtyFiles(
dirtyFiles: Collection<String>?,
cacheConsumer: PersistentCacheConsumer,
irFactory: IrFactory,
exportedDeclarations: Set<FqName>,
mainArguments: List<String>?,
) {
val loadedModules = loadModules(configuration.languageVersionSettings, dependencyGraph)
@@ -462,7 +464,16 @@ fun rebuildCacheForDirtyFiles(
val currentIrModule = irModules.find { it.second == library }?.first!!
buildCacheForModuleFiles(currentIrModule, irModules.map { it.first }, jsIrLinker, configuration, dirtyFiles, cacheConsumer)
buildCacheForModuleFiles(
currentIrModule,
irModules.map { it.first },
jsIrLinker,
configuration,
dirtyFiles,
cacheConsumer,
exportedDeclarations,
mainArguments
)
}
@Suppress("UNUSED_PARAMETER")
@@ -472,14 +483,17 @@ private fun buildCacheForModuleFiles(
deserializer: JsIrLinker,
configuration: CompilerConfiguration,
dirtyFiles: Collection<String>?, // if null consider the whole module dirty
cacheConsumer: PersistentCacheConsumer
cacheConsumer: PersistentCacheConsumer,
exportedDeclarations: Set<FqName>,
mainArguments: List<String>?,
) {
compileWithIC(
currentModule,
configuration = configuration,
deserializer = deserializer,
dependencies = dependencies,
exportedDeclarations = setOf(FqName("box")),
mainArguments = mainArguments,
exportedDeclarations = exportedDeclarations,
filesToLower = dirtyFiles?.toSet(),
cacheConsumer = cacheConsumer,
)
@@ -489,3 +503,16 @@ private fun buildCacheForModuleFiles(
// val dirtyS = if (dirtyFiles == null) "[ALL]" else dirtyFiles.joinToString(",", "[", "]") { it }
// println("Dirty files -> $dirtyS")
}
fun loadModuleCaches(icCachePaths: Collection<String>): Map<String, ModuleCache> {
val icCacheMap: Map<ModulePath, String> = loadCacheInfo(icCachePaths)
return icCacheMap.entries.associate { (lib, cache) ->
val provider = createCacheProvider(cache)
val files = provider.filePaths()
lib to ModuleCache(lib, files.associate { f ->
f to FileCache(f, provider.binaryAst(f), provider.dts(f), provider.sourceMap(f))
})
}
}
compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheAccessors.kt
+13
View File
@@ -50,6 +50,8 @@ interface PersistentCacheProvider {
fun sourceMap(path: String): ByteArray?
fun filePaths(): Iterable<String>
companion object {
val EMPTY = object : PersistentCacheProvider {
override fun fileFingerPrint(path: String): Hash {
@@ -83,6 +85,8 @@ interface PersistentCacheProvider {
override fun sourceMap(path: String): ByteArray? {
return null
}
override fun filePaths(): Iterable<String> = emptyList()
}
}
}
@@ -178,6 +182,15 @@ class PersistentCacheProviderImpl(private val cachePath: String) : PersistentCac
override fun sourceMap(path: String): ByteArray? {
return readBytesFromCacheFile(path, fileSourceMap)
}
override fun filePaths(): Iterable<String> {
return File(cachePath).listFiles()!!.filter { it.isDirectory }.mapNotNull { f ->
val fileInfo = File(f, fileInfoFile)
if (fileInfo.exists()) {
fileInfo.readLines()[0]
} else null
}
}
}
interface PersistentCacheConsumer {
js/js.tests/test/org/jetbrains/kotlin/js/test/converters/JsIrBackendFacade.kt
+6 -3
View File
@@ -19,6 +19,7 @@ import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerDesc
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformer
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformerTmp
import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl
import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC
import org.jetbrains.kotlin.ir.util.IrMessageLogger
import org.jetbrains.kotlin.ir.util.SymbolTable
import org.jetbrains.kotlin.js.config.ErrorTolerancePolicy
@@ -94,7 +95,9 @@ class JsIrBackendFacade(
val outputFile = File(JsEnvironmentConfigurator.getJsModuleArtifactPath(testServices, module.name) + ".js")
File(outputFile.parentFile, outputFile.nameWithoutExtension + "-recompiled.js")
}
val compiledModule = generateJsFromAst(inputArtifact.outputFile.absolutePath, testServices.jsIrIncrementalDataProvider.getCaches())
val moduleName = configuration.getNotNull(CommonConfigurationKeys.MODULE_NAME)
val moduleKind = configuration.get(JSConfigurationKeys.MODULE_KIND, ModuleKind.PLAIN)
val compiledModule = generateJsFromAst(moduleName, moduleKind, testServices.jsIrIncrementalDataProvider.getCaches())
return BinaryArtifacts.Js.JsIrArtifact(
outputFile, compiledModule, testServices.jsIrIncrementalDataProvider.getCacheForModule(module)
).dump(module)
@@ -180,7 +183,7 @@ class JsIrBackendFacade(
val filesToLoad = module.files.takeIf { !firstTimeCompilation }?.map { "/${it.relativePath}" }?.toSet()
val messageLogger = configuration.get(IrMessageLogger.IR_MESSAGE_LOGGER) ?: IrMessageLogger.None
val symbolTable = SymbolTable(IdSignatureDescriptor(JsManglerDesc), IrFactoryImpl)
val symbolTable = SymbolTable(IdSignatureDescriptor(JsManglerDesc), IrFactoryImplForJsIC(WholeWorldStageController()),)
val moduleDescriptor = testServices.moduleDescriptorProvider.getModuleDescriptor(module)
val mainModuleLib = testServices.jsLibraryProvider.getCompiledLibraryByDescriptor(moduleDescriptor)
@@ -210,7 +213,7 @@ class JsIrBackendFacade(
): IrModuleInfo {
val errorPolicy = configuration.get(JSConfigurationKeys.ERROR_TOLERANCE_POLICY) ?: ErrorTolerancePolicy.DEFAULT
val messageLogger = configuration.get(IrMessageLogger.IR_MESSAGE_LOGGER) ?: IrMessageLogger.None
val symbolTable = SymbolTable(IdSignatureDescriptor(JsManglerDesc), IrFactoryImpl)
val symbolTable = SymbolTable(IdSignatureDescriptor(JsManglerDesc), IrFactoryImplForJsIC(WholeWorldStageController()),)
val verifySignatures = JsEnvironmentConfigurationDirectives.SKIP_MANGLE_VERIFICATION !in module.directives
val psi2Ir = Psi2IrTranslator(
js/js.tests/test/org/jetbrains/kotlin/js/test/handlers/JsIrRecompiledArtifactsIdentityHandler.kt
+5 -3
View File
@@ -5,10 +5,12 @@
package org.jetbrains.kotlin.js.test.handlers
import com.intellij.openapi.util.io.FileUtil
import org.jetbrains.kotlin.test.backend.handlers.JsBinaryArtifactHandler
import org.jetbrains.kotlin.test.model.BinaryArtifacts.Js
import org.jetbrains.kotlin.test.model.TestModule
import org.jetbrains.kotlin.test.services.TestServices
import org.jetbrains.kotlin.test.services.assertions
class JsIrRecompiledArtifactsIdentityHandler(testServices: TestServices) : JsBinaryArtifactHandler(testServices) {
override fun processModule(module: TestModule, info: Js) {
@@ -44,9 +46,9 @@ class JsIrRecompiledArtifactsIdentityHandler(testServices: TestServices) : JsBin
// }
// }
//
// val originalOutput = FileUtil.loadFile(originalArtifact.outputFile)
// val recompiledOutput = FileUtil.loadFile(incrementalArtifact.outputFile)
// testServices.assertions.assertEquals(originalOutput, recompiledOutput) { "Output file changed after recompilation" }
val originalOutput = FileUtil.loadFile(originalArtifact.outputFile)
val recompiledOutput = FileUtil.loadFile(incrementalArtifact.outputFile)
testServices.assertions.assertEquals(originalOutput, recompiledOutput) { "Output file changed after recompilation" }
}
override fun processAfterAllModules(someAssertionWasFailed: Boolean) {}
js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt
+35 -4
View File
@@ -6,13 +6,17 @@
package org.jetbrains.kotlin.js.test.utils
import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.ir.backend.js.WholeWorldStageController
import org.jetbrains.kotlin.ir.backend.js.ic.*
import org.jetbrains.kotlin.ir.backend.js.moduleName
import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl
import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC
import org.jetbrains.kotlin.ir.util.IdSignature
import org.jetbrains.kotlin.js.test.handlers.JsBoxRunner
import org.jetbrains.kotlin.konan.properties.propertyList
import org.jetbrains.kotlin.library.KLIB_PROPERTY_DEPENDS
import org.jetbrains.kotlin.library.KotlinLibrary
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.test.directives.JsEnvironmentConfigurationDirectives
import org.jetbrains.kotlin.test.model.TestModule
import org.jetbrains.kotlin.test.services.TestService
@@ -59,6 +63,10 @@ class TestModuleCache(val moduleName: String, val files: MutableMap<String, File
override fun sourceMap(path: String): ByteArray? {
return files[path]?.sourceMap
}
override fun filePaths(): Iterable<String> {
return files.keys
}
}
}
@@ -161,8 +169,11 @@ class JsIrIncrementalDataProvider(private val testServices: TestServices) : Test
}
val configuration = testServices.compilerConfigurationProvider.getCompilerConfiguration(module)
val mainArguments = JsEnvironmentConfigurator.getMainCallParametersForModule(module)
.run { if (shouldBeGenerated()) arguments() else null }
runtimeKlibPath.forEach {
recordIncrementalData(it, null, libs, configuration)
recordIncrementalData(it, null, libs, configuration, mainArguments)
}
}
@@ -173,11 +184,20 @@ class JsIrIncrementalDataProvider(private val testServices: TestServices) : Test
val path = JsEnvironmentConfigurator.getJsKlibArtifactPath(testServices, module.name)
val configuration = testServices.compilerConfigurationProvider.getCompilerConfiguration(module)
val mainArguments = JsEnvironmentConfigurator.getMainCallParametersForModule(module)
.run { if (shouldBeGenerated()) arguments() else null }
val allDependencies = JsEnvironmentConfigurator.getAllRecursiveLibrariesFor(module, testServices).keys.toList()
recordIncrementalData(path, dirtyFiles, allDependencies + library, configuration)
recordIncrementalData(path, dirtyFiles, allDependencies + library, configuration, mainArguments)
}
private fun recordIncrementalData(path: String, dirtyFiles: List<String>?, allDependencies: List<KotlinLibrary>, configuration: CompilerConfiguration) {
private fun recordIncrementalData(
path: String,
dirtyFiles: List<String>?,
allDependencies: List<KotlinLibrary>,
configuration: CompilerConfiguration,
mainArguments: List<String>?
) {
val canonicalPath = File(path).canonicalPath
var moduleCache = predefinedKlibHasIcCache[canonicalPath]
@@ -196,7 +216,18 @@ class JsIrIncrementalDataProvider(private val testServices: TestServices) : Test
val currentLib = libs[File(canonicalPath).canonicalPath] ?: error("Expected library at $canonicalPath")
rebuildCacheForDirtyFiles(currentLib, configuration, dependencyGraph, dirtyFiles, moduleCache.cacheConsumer(), IrFactoryImpl)
val testPackage = extractTestPackage(testServices)
rebuildCacheForDirtyFiles(
currentLib,
configuration,
dependencyGraph,
dirtyFiles,
moduleCache.cacheConsumer(),
IrFactoryImplForJsIC(WholeWorldStageController()),
setOf(FqName.fromSegments(listOfNotNull(testPackage, JsBoxRunner.TEST_FUNCTION))),
mainArguments,
)
if (canonicalPath in predefinedKlibHasIcCache) {
predefinedKlibHasIcCache[canonicalPath] = moduleCache