azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[JS KLIB IC] Implement incremental cache for IR

Browse Source
This commit is contained in:
Roman Artemev
2019-08-23 14:27:08 +03:00
committed by romanart
parent 7840cf70f0
commit 3753b75609
10 changed files with 145 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt
+80
View File
@@ -18,6 +18,7 @@ package org.jetbrains.kotlin.incremental
import com.intellij.util.io.DataExternalizer
import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl
import org.jetbrains.kotlin.incremental.js.IrTranslationResultValue
import org.jetbrains.kotlin.incremental.js.TranslationResultValue
import org.jetbrains.kotlin.incremental.storage.*
import org.jetbrains.kotlin.metadata.ProtoBuf
@@ -40,6 +41,7 @@ open class IncrementalJsCache(
) : AbstractIncrementalCache<FqName>(cachesDir, pathConverter) {
companion object {
private const val TRANSLATION_RESULT_MAP = "translation-result"
private const val IR_TRANSLATION_RESULT_MAP = "ir-translation-result"
private const val INLINE_FUNCTIONS = "inline-functions"
private const val HEADER_FILE_NAME = "header.meta"
private const val PACKAGE_META_FILE = "packages-meta"
@@ -50,6 +52,7 @@ open class IncrementalJsCache(
override val sourceToClassesMap = registerMap(SourceToFqNameMap(SOURCE_TO_CLASSES.storageFile, pathConverter))
override val dirtyOutputClassesMap = registerMap(DirtyClassesFqNameMap(DIRTY_OUTPUT_CLASSES.storageFile))
private val translationResults = registerMap(TranslationResultMap(TRANSLATION_RESULT_MAP.storageFile, pathConverter))
private val irTranslationResults = registerMap(IrTranslationResultMap(IR_TRANSLATION_RESULT_MAP.storageFile, pathConverter))
private val inlineFunctions = registerMap(InlineFunctionsMap(INLINE_FUNCTIONS.storageFile, pathConverter))
private val packageMetadata = registerMap(PackageMetadataMap(PACKAGE_META_FILE.storageFile))
@@ -108,6 +111,11 @@ open class IncrementalJsCache(
for ((packageName, metadata) in incrementalResults.packageMetadata) {
packageMetadata.put(packageName, metadata)
}
for ((srcFile, irData) in incrementalResults.irFileData) {
val (fileData, symbols, types, strings, declarations, bodies, fqn) = irData
irTranslationResults.put(srcFile, fileData, symbols, types, strings, declarations, bodies, fqn)
}
}
private fun registerOutputForFile(srcFile: File, name: FqName) {
@@ -118,6 +126,7 @@ open class IncrementalJsCache(
override fun clearCacheForRemovedClasses(changesCollector: ChangesCollector) {
dirtySources.forEach {
translationResults.remove(it, changesCollector)
irTranslationResults.remove(it)
inlineFunctions.remove(it)
}
removeAllFromClassStorage(dirtyOutputClassesMap.getDirtyOutputClasses(), changesCollector)
@@ -140,6 +149,16 @@ open class IncrementalJsCache(
put(fqNameString, packageMetadata[fqNameString]!!)
}
}
fun nonDirtyIrParts(): Map<File, IrTranslationResultValue> =
hashMapOf<File, IrTranslationResultValue>().apply {
for (file in irTranslationResults.keys()) {
if (file !in dirtySources) {
put(file, irTranslationResults[file]!!)
}
}
}
}
private object TranslationResultValueExternalizer : DataExternalizer<TranslationResultValue> {
@@ -202,6 +221,67 @@ private class TranslationResultMap(
}
}
private object IrTranslationResultValueExternalizer : DataExternalizer<IrTranslationResultValue> {
override fun save(output: DataOutput, value: IrTranslationResultValue) {
output.writeArray(value.fileData)
output.writeArray(value.symbols)
output.writeArray(value.types)
output.writeArray(value.strings)
output.writeArray(value.declarations)
output.writeArray(value.bodies)
output.writeArray(value.fqn)
}
private fun DataOutput.writeArray(array: ByteArray) {
writeInt(array.size)
write(array)
}
private fun DataInput.readArray(): ByteArray {
val dataSize = readInt()
val filedata = ByteArray(dataSize)
readFully(filedata)
return filedata
}
override fun read(input: DataInput): IrTranslationResultValue {
val fileData = input.readArray()
val symbols = input.readArray()
val types = input.readArray()
val strings = input.readArray()
val declarations = input.readArray()
val bodies = input.readArray()
val fqn = input.readArray()
return IrTranslationResultValue(fileData, symbols, types, strings, declarations, bodies, fqn)
}
}
private class IrTranslationResultMap(
storageFile: File,
private val pathConverter: FileToPathConverter
) :
BasicStringMap<IrTranslationResultValue>(storageFile, IrTranslationResultValueExternalizer) {
override fun dumpValue(value: IrTranslationResultValue): String =
"Filedata: ${value.fileData.md5()}, Symbols: ${value.symbols.md5()}, Types: ${value.types.md5()}, Strings: ${value.strings.md5()}, Declarations: ${value.declarations.md5()}, Bodies: ${value.bodies.md5()}"
fun put(sourceFile: File, newFiledata: ByteArray, newSymbols: ByteArray, newTypes: ByteArray, newStrings: ByteArray, newDeclarations: ByteArray, newBodies: ByteArray, fqn: ByteArray) {
storage[pathConverter.toPath(sourceFile)] =
IrTranslationResultValue(newFiledata, newSymbols, newTypes, newStrings, newDeclarations, newBodies, fqn)
}
operator fun get(sourceFile: File): IrTranslationResultValue? =
storage[pathConverter.toPath(sourceFile)]
fun keys(): Collection<File> =
storage.keys.map { pathConverter.toFile(it) }
fun remove(sourceFile: File) {
val path = pathConverter.toPath(sourceFile)
storage.remove(path)
}
}
fun getProtoData(sourceFile: File, metadata: ByteArray): Map<ClassId, ProtoData> {
val classes = hashMapOf<ClassId, ProtoData>()
val proto = ProtoBuf.PackageFragment.parseFrom(metadata, JsSerializerProtocol.extensionRegistry)
build-common/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProviderFromCache.kt
+3
View File
@@ -21,4 +21,7 @@ class IncrementalDataProviderFromCache(private val cache: IncrementalJsCache) :
override val packageMetadata: Map<String, ByteArray>
get() = cache.packageMetadata()
override val serializedIrFiles: Map<File, IrTranslationResultValue>
get() = cache.nonDirtyIrParts()
}
compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalDataProvider.kt
+4
View File
@@ -9,11 +9,15 @@ import org.jetbrains.kotlin.daemon.common.CompilerCallbackServicesFacade
import org.jetbrains.kotlin.daemon.common.Profiler
import org.jetbrains.kotlin.daemon.common.withMeasure
import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider
import org.jetbrains.kotlin.incremental.js.IrTranslationResultValue
import org.jetbrains.kotlin.incremental.js.TranslationResultValue
import java.io.File
class RemoteIncrementalDataProvider(val facade: CompilerCallbackServicesFacade, val rpcProfiler: Profiler) :
IncrementalDataProvider {
override val serializedIrFiles: Map<File, IrTranslationResultValue>
get() = TODO("not implemented") //To change initializer of created properties use File | Settings | File Templates.
override val headerMetadata: ByteArray
get() = rpcProfiler.withMeasure(this) {
facade.incrementalDataProvider_getHeaderMetadata()
compiler/daemon/src/org/jetbrains/kotlin/daemon/RemoteIncrementalResultsConsumer.kt
+12
View File
@@ -15,6 +15,18 @@ import java.io.File
class RemoteIncrementalResultsConsumer(val facade: CompilerCallbackServicesFacade, eventManager: EventManager, val rpcProfiler: Profiler) :
IncrementalResultsConsumer {
override fun processIrFile(
sourceFile: File,
fileData: ByteArray,
symbols: ByteArray,
types: ByteArray,
strings: ByteArray,
declarations: ByteArray,
bodies: ByteArray,
fqn: ByteArray
) {
TODO("not implemented") //To change body of created functions use File | Settings | File Templates.
}
init {
eventManager.
jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt
+1 -1
View File
@@ -123,7 +123,7 @@ abstract class AbstractJsLookupTrackerTest : AbstractLookupTrackerTest() {
if (header != null) {
register(
IncrementalDataProvider::class.java,
IncrementalDataProviderImpl(header!!, packageParts, JsMetadataVersion.INSTANCE.toArray(), emptyMap()) // TODO pass correct metadata
IncrementalDataProviderImpl(header!!, packageParts, JsMetadataVersion.INSTANCE.toArray(), emptyMap(), emptyMap()) // TODO pass correct metadata
)
}
jps-plugin/src/org/jetbrains/kotlin/jps/incremental/local.kt
+1 -1
View File
@@ -7,7 +7,7 @@ package org.jetbrains.kotlin.jps.incremental
import java.io.File
private val NORMAL_VERSION = 12
private val NORMAL_VERSION = 13
private val NORMAL_VERSION_FILE_NAME = "format-version.txt"
fun localCacheVersionManager(dataRoot: File, isCachesEnabled: Boolean) =
js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProvider.kt
+4 -1
View File
@@ -31,11 +31,14 @@ interface IncrementalDataProvider {
/** gets non-dirty package metadata from previous compilation */
val packageMetadata: Map<String, ByteArray>
val serializedIrFiles: Map<File, IrTranslationResultValue>
}
class IncrementalDataProviderImpl(
override val headerMetadata: ByteArray,
override val compiledPackageParts: Map<File, TranslationResultValue>,
override val metadataVersion: IntArray,
override val packageMetadata: Map<String, ByteArray>
override val packageMetadata: Map<String, ByteArray>,
override val serializedIrFiles: Map<File, IrTranslationResultValue>
) : IncrementalDataProvider
js/js.frontend/src/org/jetbrains/kotlin/incremental/js/IncrementalResultsConsumer.kt
+29 -1
View File
@@ -16,7 +16,6 @@
package org.jetbrains.kotlin.incremental.js
import org.jetbrains.kotlin.name.FqName
import java.io.File
import java.io.Serializable
import java.security.MessageDigest
@@ -42,6 +41,17 @@ interface IncrementalResultsConsumer {
fun processInlineFunctions(functions: Collection<JsInlineFunctionHash>)
fun processPackageMetadata(packageName: String, metadata: ByteArray)
fun processIrFile(
sourceFile: File,
fileData: ByteArray,
symbols: ByteArray,
types: ByteArray,
strings: ByteArray,
declarations: ByteArray,
bodies: ByteArray,
fqn: ByteArray
)
}
class JsInlineFunctionHash(val sourceFilePath: String, val fqName: String, val inlineFunctionMd5Hash: Long): Serializable
@@ -108,6 +118,24 @@ class IncrementalResultsConsumerImpl : IncrementalResultsConsumer {
override fun processPackageMetadata(packageName: String, metadata: ByteArray) {
_packageMetadata[packageName] = metadata
}
// class IrFileData(fileData: ByteArray, symbols: ByteArray, types: ByteArray, strings: ByteArray, bodies: ByteArray, declarations: ByteArray)
private val _irFileData = hashMapOf<File, IrTranslationResultValue>()
val irFileData: Map<File, IrTranslationResultValue>
get() = _irFileData
override fun processIrFile(
sourceFile: File,
fileData: ByteArray,
symbols: ByteArray,
types: ByteArray,
strings: ByteArray,
declarations: ByteArray,
bodies: ByteArray,
fqn: ByteArray
) {
_irFileData[sourceFile] = IrTranslationResultValue(fileData, symbols, types, strings, declarations, bodies, fqn)
}
}
private fun ByteArray.md5(): Long {
js/js.frontend/src/org/jetbrains/kotlin/incremental/js/TranslationResultValue.kt
+10
View File
@@ -18,3 +18,13 @@ package org.jetbrains.kotlin.incremental.js
data class TranslationResultValue(val metadata: ByteArray, val binaryAst: ByteArray, val inlineData: ByteArray)
data class IrTranslationResultValue(
val fileData: ByteArray,
val symbols: ByteArray,
val types: ByteArray,
val strings: ByteArray,
val declarations: ByteArray,
val bodies: ByteArray,
val fqn: ByteArray
)
js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt
+1 -1
View File
@@ -654,7 +654,7 @@ abstract class BasicBoxTest(
if (header != null) {
configuration.put(
JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER,
IncrementalDataProviderImpl(header, incrementalData.translatedFiles, JsMetadataVersion.INSTANCE.toArray(), incrementalData.packageMetadata)
IncrementalDataProviderImpl(header, incrementalData.translatedFiles, JsMetadataVersion.INSTANCE.toArray(), incrementalData.packageMetadata, emptyMap())
)
}