From d776e9211174e645e0bc689531e005be5b9aaebe Mon Sep 17 00:00:00 2001 From: Alexey Tsvetkov Date: Mon, 28 Oct 2019 09:55:46 +0300 Subject: [PATCH] Use Klib JS serializer protocol for IC --- .../kotlin/incremental/IncrementalJsCache.kt | 51 ++++++++++-------- .../incremental/IncrementalCachesManager.kt | 6 ++- .../IncrementalJsCompilerRunner.kt | 53 +++++++++++-------- .../jps/incremental/JpsIncrementalCache.kt | 3 +- 4 files changed, 65 insertions(+), 48 deletions(-) diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt index 637cc9f3b0b..5429c9495cb 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt @@ -24,20 +24,20 @@ import org.jetbrains.kotlin.incremental.storage.* import org.jetbrains.kotlin.metadata.ProtoBuf import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl import org.jetbrains.kotlin.metadata.deserialization.getExtensionOrNull -import org.jetbrains.kotlin.metadata.js.JsProtoBuf import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.name.parentOrNull +import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol import org.jetbrains.kotlin.serialization.deserialization.getClassId -import org.jetbrains.kotlin.serialization.js.JsSerializerProtocol import java.io.DataInput import java.io.DataOutput import java.io.File open class IncrementalJsCache( cachesDir: File, - pathConverter: FileToPathConverter + pathConverter: FileToPathConverter, + serializerProtocol: SerializerExtensionProtocol ) : AbstractIncrementalCache(cachesDir, pathConverter) { companion object { private const val TRANSLATION_RESULT_MAP = "translation-result" @@ -49,9 +49,11 @@ open class IncrementalJsCache( fun hasHeaderFile(cachesDir: File) = File(cachesDir, HEADER_FILE_NAME).exists() } + private val protoData = ProtoDataProvider(serializerProtocol) + override val sourceToClassesMap = registerMap(SourceToFqNameMap(SOURCE_TO_CLASSES.storageFile, pathConverter)) override val dirtyOutputClassesMap = registerMap(DirtyClassesFqNameMap(DIRTY_OUTPUT_CLASSES.storageFile)) - private val translationResults = registerMap(TranslationResultMap(TRANSLATION_RESULT_MAP.storageFile, pathConverter)) + private val translationResults = registerMap(TranslationResultMap(TRANSLATION_RESULT_MAP.storageFile, pathConverter, protoData)) private val irTranslationResults = registerMap(IrTranslationResultMap(IR_TRANSLATION_RESULT_MAP.storageFile, pathConverter)) private val inlineFunctions = registerMap(InlineFunctionsMap(INLINE_FUNCTIONS.storageFile, pathConverter)) private val packageMetadata = registerMap(PackageMetadataMap(PACKAGE_META_FILE.storageFile)) @@ -86,8 +88,8 @@ open class IncrementalJsCache( dirtySources.remove(srcFile) val (binaryMetadata, binaryAst, inlineData) = data - val oldProtoMap = translationResults[srcFile]?.metadata?.let { getProtoData(srcFile, it) } ?: emptyMap() - val newProtoMap = getProtoData(srcFile, binaryMetadata) + val oldProtoMap = translationResults[srcFile]?.metadata?.let { protoData(srcFile, it) } ?: emptyMap() + val newProtoMap = protoData(srcFile, binaryMetadata) for ((classId, protoData) in newProtoMap) { registerOutputForFile(srcFile, classId.asSingleFqName()) @@ -192,7 +194,8 @@ private object TranslationResultValueExternalizer : DataExternalizer(storageFile, TranslationResultValueExternalizer) { override fun dumpValue(value: TranslationResultValue): String = @@ -212,7 +215,7 @@ private class TranslationResultMap( fun remove(sourceFile: File, changesCollector: ChangesCollector) { val path = pathConverter.toPath(sourceFile) val protoBytes = storage[path]!!.metadata - val protoMap = getProtoData(sourceFile, protoBytes) + val protoMap = protoData(sourceFile, protoBytes) for ((_, protoData) in protoMap) { changesCollector.collectProtoChanges(oldData = protoData, newData = null) @@ -282,23 +285,25 @@ private class IrTranslationResultMap( } } -fun getProtoData(sourceFile: File, metadata: ByteArray): Map { - val classes = hashMapOf() - val proto = ProtoBuf.PackageFragment.parseFrom(metadata, JsSerializerProtocol.extensionRegistry) - val nameResolver = NameResolverImpl(proto.strings, proto.qualifiedNames) +private class ProtoDataProvider(private val serializerProtocol: SerializerExtensionProtocol) { + operator fun invoke(sourceFile: File, metadata: ByteArray): Map { + val classes = hashMapOf() + val proto = ProtoBuf.PackageFragment.parseFrom(metadata, serializerProtocol.extensionRegistry) + val nameResolver = NameResolverImpl(proto.strings, proto.qualifiedNames) - proto.class_List.forEach { - val classId = nameResolver.getClassId(it.fqName) - classes[classId] = ClassProtoData(it, nameResolver) + proto.class_List.forEach { + val classId = nameResolver.getClassId(it.fqName) + classes[classId] = ClassProtoData(it, nameResolver) + } + + proto.`package`.apply { + val packageFqName = getExtensionOrNull(serializerProtocol.packageFqName)?.let(nameResolver::getPackageFqName)?.let(::FqName) ?: FqName.ROOT + val packagePartClassId = ClassId(packageFqName, Name.identifier(sourceFile.nameWithoutExtension.capitalize() + "Kt")) + classes[packagePartClassId] = PackagePartProtoData(this, nameResolver, packageFqName) + } + + return classes } - - proto.`package`.apply { - val packageFqName = getExtensionOrNull(JsProtoBuf.packageFqName)?.let(nameResolver::getPackageFqName)?.let(::FqName) ?: FqName.ROOT - val packagePartClassId = ClassId(packageFqName, Name.identifier(sourceFile.nameWithoutExtension.capitalize() + "Kt")) - classes[packagePartClassId] = PackagePartProtoData(this, nameResolver, packageFqName) - } - - return classes } private class InlineFunctionsMap( diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt index 1749655b51f..68d63bc85ad 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt @@ -18,6 +18,7 @@ package org.jetbrains.kotlin.incremental import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner import org.jetbrains.kotlin.incremental.storage.FileToCanonicalPathConverter +import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol import java.io.File private val PATH_CONVERTER = FileToCanonicalPathConverter @@ -75,9 +76,10 @@ class IncrementalJvmCachesManager( class IncrementalJsCachesManager( cachesRootDir: File, - reporter: ICReporter + reporter: ICReporter, + serializerProtocol: SerializerExtensionProtocol ) : IncrementalCachesManager(cachesRootDir, reporter) { private val jsCacheFile = File(cachesRootDir, "js").apply { mkdirs() } - override val platformCache = IncrementalJsCache(jsCacheFile, PATH_CONVERTER).apply { registerCache() } + override val platformCache = IncrementalJsCache(jsCacheFile, PATH_CONVERTER, serializerProtocol).apply { registerCache() } } \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt index 788b582bdf6..ecd42d99965 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt @@ -28,24 +28,27 @@ import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.js.* import org.jetbrains.kotlin.incremental.multiproject.EmptyModulesApiHistory import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory +import org.jetbrains.kotlin.library.metadata.KlibMetadataSerializerProtocol +import org.jetbrains.kotlin.serialization.js.JsSerializerProtocol import java.io.File fun makeJsIncrementally( - cachesDir: File, - sourceRoots: Iterable, - args: K2JSCompilerArguments, - messageCollector: MessageCollector = MessageCollector.NONE, - reporter: ICReporter = EmptyICReporter + cachesDir: File, + sourceRoots: Iterable, + args: K2JSCompilerArguments, + messageCollector: MessageCollector = MessageCollector.NONE, + reporter: ICReporter = EmptyICReporter ) { val allKotlinFiles = sourceRoots.asSequence().flatMap { it.walk() } - .filter { it.isFile && it.extension.equals("kt", ignoreCase = true) }.toList() + .filter { it.isFile && it.extension.equals("kt", ignoreCase = true) }.toList() val buildHistoryFile = File(cachesDir, "build-history.bin") withJsIC { val compiler = IncrementalJsCompilerRunner( cachesDir, reporter, buildHistoryFile = buildHistoryFile, - modulesApiHistory = EmptyModulesApiHistory) + modulesApiHistory = EmptyModulesApiHistory + ) compiler.compile(allKotlinFiles, args, messageCollector, providedChangedFiles = null) } } @@ -64,24 +67,30 @@ inline fun withJsIC(fn: () -> R): R { class IncrementalJsCompilerRunner( workingDir: File, reporter: ICReporter, - buildHistoryFile: File, - private val modulesApiHistory: ModulesApiHistory + buildHistoryFile: File, + private val modulesApiHistory: ModulesApiHistory ) : IncrementalCompilerRunner( workingDir, "caches-js", reporter, - buildHistoryFile = buildHistoryFile + buildHistoryFile = buildHistoryFile ) { override fun isICEnabled(): Boolean = IncrementalCompilation.isEnabledForJs() - override fun createCacheManager(args: K2JSCompilerArguments): IncrementalJsCachesManager = - IncrementalJsCachesManager(cacheDirectory, reporter) + override fun createCacheManager(args: K2JSCompilerArguments): IncrementalJsCachesManager { + val serializerProtocol = if (!args.irBackend) JsSerializerProtocol else KlibMetadataSerializerProtocol + return IncrementalJsCachesManager(cacheDirectory, reporter, serializerProtocol) + } override fun destinationDir(args: K2JSCompilerArguments): File = File(args.outputFile).parentFile - override fun calculateSourcesToCompile(caches: IncrementalJsCachesManager, changedFiles: ChangedFiles.Known, args: K2JSCompilerArguments): CompilationMode { + override fun calculateSourcesToCompile( + caches: IncrementalJsCachesManager, + changedFiles: ChangedFiles.Known, + args: K2JSCompilerArguments + ): CompilationMode { val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) ?: return CompilationMode.Rebuild { "No information on previous build" } @@ -130,10 +139,10 @@ class IncrementalJsCompilerRunner( } override fun updateCaches( - services: Services, - caches: IncrementalJsCachesManager, - generatedFiles: List, - changesCollector: ChangesCollector + services: Services, + caches: IncrementalJsCachesManager, + generatedFiles: List, + changesCollector: ChangesCollector ) { val incrementalResults = services.get(IncrementalResultsConsumer::class.java) as IncrementalResultsConsumerImpl @@ -145,11 +154,11 @@ class IncrementalJsCompilerRunner( } override fun runCompiler( - sourcesToCompile: Set, - args: K2JSCompilerArguments, - caches: IncrementalJsCachesManager, - services: Services, - messageCollector: MessageCollector + sourcesToCompile: Set, + args: K2JSCompilerArguments, + caches: IncrementalJsCachesManager, + services: Services, + messageCollector: MessageCollector ): ExitCode { val freeArgsBackup = args.freeArgs diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt index ae74d0345c2..3e4203fc14d 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/JpsIncrementalCache.kt @@ -27,6 +27,7 @@ import org.jetbrains.kotlin.incremental.IncrementalJvmCache import org.jetbrains.kotlin.incremental.storage.FileToPathConverter import org.jetbrains.kotlin.jps.build.KotlinBuilder import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget +import org.jetbrains.kotlin.serialization.js.JsSerializerProtocol import java.io.File interface JpsIncrementalCache : IncrementalCacheCommon, StorageOwner { @@ -53,7 +54,7 @@ class JpsIncrementalJsCache( target: ModuleBuildTarget, paths: BuildDataPaths, pathConverter: FileToPathConverter -) : IncrementalJsCache(paths.getTargetDataRoot(target), pathConverter), JpsIncrementalCache { +) : IncrementalJsCache(paths.getTargetDataRoot(target), pathConverter, JsSerializerProtocol), JpsIncrementalCache { override fun addJpsDependentCache(cache: JpsIncrementalCache) { if (cache is JpsIncrementalJsCache) { addDependentCache(cache)