diff --git a/build-common/build-common.iml b/build-common/build-common.iml index b38d7bfdeca..2f861d68f12 100644 --- a/build-common/build-common.iml +++ b/build-common/build-common.iml @@ -16,5 +16,6 @@ + \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt new file mode 100644 index 00000000000..3d57998a74c --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt @@ -0,0 +1,189 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.incremental + +import com.intellij.util.io.DataExternalizer +import org.jetbrains.kotlin.incremental.js.TranslationResultValue +import org.jetbrains.kotlin.incremental.storage.* +import org.jetbrains.kotlin.name.ClassId +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.serialization.ProtoBuf +import org.jetbrains.kotlin.serialization.deserialization.NameResolverImpl +import org.jetbrains.kotlin.serialization.js.JsProtoBuf +import org.jetbrains.kotlin.serialization.js.JsSerializerProtocol +import java.io.DataInput +import java.io.DataOutput +import java.io.File + +open class IncrementalJsCache(cachesDir: File) : IncrementalCacheCommon(cachesDir) { + companion object { + private val TRANSLATION_RESULT_MAP = "translation-result" + private val SOURCES_TO_CLASSES_FQNS = "sources-to-classes" + private val HEADER_FILE_NAME = "header.meta" + } + + private val dirtySources = arrayListOf() + private val translationResults = registerMap(TranslationResultMap(TRANSLATION_RESULT_MAP.storageFile)) + private val sourcesToClasses = registerMap(SourceToClassesMap(SOURCES_TO_CLASSES_FQNS.storageFile)) + + private val headerFile: File + get() = File(cachesDir, HEADER_FILE_NAME) + + var header: ByteArray + get() = headerFile.readBytes() + set(value) { + cachesDir.mkdirs() + headerFile.writeBytes(value) + } + + override fun markDirty(removedAndCompiledSources: List) { + dirtySources.addAll(removedAndCompiledSources) + } + + fun compareAndUpdate(translatedFiles: Map, changesCollector: ChangesCollector) { + dirtySources.forEach { + if (it !in translatedFiles) { + translationResults.remove(it, changesCollector) + } + + removeAllFromClassStorage(sourcesToClasses[it]) + sourcesToClasses.clearOutputsForSource(it) + } + dirtySources.clear() + + for ((srcFile, data) in translatedFiles) { + val (binaryMetadata, binaryAst) = data + + val oldProtoMap = translationResults[srcFile]?.metadata?.let { getProtoData(srcFile, it) } ?: emptyMap() + val newProtoMap = getProtoData(srcFile, binaryMetadata) + + for (protoData in newProtoMap.values) { + if (protoData is ClassProtoData) { + addToClassStorage(protoData.proto, protoData.nameResolver, srcFile) + } + } + + for (classId in oldProtoMap.keys + newProtoMap.keys) { + changesCollector.collectProtoChanges(oldProtoMap[classId], newProtoMap[classId]) + } + + translationResults.put(srcFile, binaryMetadata, binaryAst) + } + } + + fun nonDirtyPackageParts(): Map = + hashMapOf().apply { + for (path in translationResults.keys()) { + val file = File(path) + if (file !in dirtySources) { + put(file, translationResults[path]!!) + } + } + } +} + +private class SourceToClassesMap(storageFile: File) : BasicStringMap>(storageFile, PathStringDescriptor, StringCollectionExternalizer) { + fun clearOutputsForSource(sourceFile: File) { + remove(sourceFile.canonicalPath) + } + + fun add(sourceFile: File, className: FqName) { + storage.append(sourceFile.canonicalPath, className.asString()) + } + + operator fun get(sourceFile: File): Collection = + storage[sourceFile.canonicalPath].orEmpty().map { FqName(it) } + + override fun dumpValue(value: Collection) = value.dumpCollection() + + private fun remove(path: String) { + storage.remove(path) + } +} + +private object TranslationResultValueExternalizer : DataExternalizer { + override fun save(output: DataOutput, value: TranslationResultValue) { + output.writeInt(value.metadata.size) + output.write(value.metadata) + + output.writeInt(value.binaryAst.size) + output.write(value.binaryAst) + } + + override fun read(input: DataInput): TranslationResultValue { + val metadataSize = input.readInt() + val metadata = ByteArray(metadataSize) + input.readFully(metadata) + + val binaryAstSize = input.readInt() + val binaryAst = ByteArray(binaryAstSize) + input.readFully(binaryAst) + + return TranslationResultValue(metadata = metadata, binaryAst = binaryAst) + } +} + +private class TranslationResultMap(storageFile: File) : BasicStringMap(storageFile, TranslationResultValueExternalizer) { + override fun dumpValue(value: TranslationResultValue): String = + "Metadata: ${value.metadata.md5()}, Binary AST: ${value.binaryAst.md5()}" + + fun put(file: File, newMetadata: ByteArray, newBinaryAst: ByteArray) { + storage[file.canonicalPath] = TranslationResultValue(metadata = newMetadata, binaryAst = newBinaryAst) + } + + operator fun get(file: File): TranslationResultValue? = + storage[file.canonicalPath] + + operator fun get(key: String): TranslationResultValue? = + storage[key] + + fun keys(): Collection = + storage.keys + + fun remove(file: File, changesCollector: ChangesCollector) { + val protoBytes = storage[file.canonicalPath]!!.metadata + val protoMap = getProtoData(file, protoBytes) + + for ((_, protoData) in protoMap) { + changesCollector.collectProtoChanges(oldData = protoData, newData = null) + } + storage.remove(file.canonicalPath) + } +} + +fun getProtoData(sourceFile: File, metadata: ByteArray): Map { + val classes = hashMapOf() + val proto = ProtoBuf.PackageFragment.parseFrom(metadata, JsSerializerProtocol.extensionRegistry) + val nameResolver = NameResolverImpl(proto.strings, proto.qualifiedNames) + + proto.class_List.forEach { + val classId = nameResolver.getClassId(it.fqName) + classes[classId] = ClassProtoData(it, nameResolver) + } + + proto.`package`.apply { + val packageFqName = if (hasExtension(JsProtoBuf.packageFqName)) { + nameResolver.getPackageFqName(getExtension(JsProtoBuf.packageFqName)) + } + else FqName.ROOT + + val packagePartClassId = ClassId(packageFqName, Name.identifier(sourceFile.nameWithoutExtension.capitalize() + "Kt")) + classes[packagePartClassId] = PackagePartProtoData(this, nameResolver, packageFqName) + } + return classes +} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMapsOwner.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMapsOwner.kt index 35b46cbadd2..1f91bbda3a2 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMapsOwner.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMapsOwner.kt @@ -19,7 +19,7 @@ package org.jetbrains.kotlin.incremental.storage import org.jetbrains.annotations.TestOnly import java.io.File -open class BasicMapsOwner(private val baseDir: File) { +open class BasicMapsOwner(val cachesDir: File) { private val maps = arrayListOf>() companion object { @@ -27,7 +27,7 @@ open class BasicMapsOwner(private val baseDir: File) { } protected val String.storageFile: File - get() = File(baseDir, this + "." + CACHE_EXTENSION) + get() = File(cachesDir, this + "." + CACHE_EXTENSION) protected fun > registerMap(map: M): M { maps.add(map) diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java b/compiler/cli/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java index 4ea118b349f..7a32b59d9c8 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java @@ -16,7 +16,6 @@ package org.jetbrains.kotlin.cli.js; -import com.google.common.collect.Lists; import com.intellij.openapi.Disposable; import com.intellij.openapi.project.Project; import com.intellij.openapi.util.io.FileUtil; diff --git a/compiler/incremental-compilation-impl/incremental-compilation-impl.iml b/compiler/incremental-compilation-impl/incremental-compilation-impl.iml index b197f9e7c8d..84461901b20 100644 --- a/compiler/incremental-compilation-impl/incremental-compilation-impl.iml +++ b/compiler/incremental-compilation-impl/incremental-compilation-impl.iml @@ -17,5 +17,6 @@ + \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt index fc9cb1352b4..281a3ebeeec 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt @@ -75,4 +75,13 @@ class IncrementalJvmCachesManager( private val jvmCacheDir = File(cacheDirectory, "jvm").apply { mkdirs() } override val platformCache = IncrementalCacheImpl(jvmCacheDir, outputDir).apply { registerCache() } +} + +class IncrementalJsCachesManager( + cachesRootDir: File, + reporter: ICReporter +) : IncrementalCachesManager(cachesRootDir, reporter) { + + private val jsCacheFile = File(cachesRootDir, "js").apply { mkdirs() } + override val platformCache = IncrementalJsCache(jsCacheFile).apply { registerCache() } } \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt new file mode 100644 index 00000000000..ba1852d1361 --- /dev/null +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt @@ -0,0 +1,136 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.incremental + +import org.jetbrains.kotlin.build.GeneratedFile +import org.jetbrains.kotlin.cli.common.ExitCode +import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments +import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.cli.js.K2JSCompiler +import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler +import org.jetbrains.kotlin.config.IncrementalCompilation +import org.jetbrains.kotlin.config.Services +import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.incremental.js.* +import java.io.File + +fun makeJsIncrementally( + cachesDir: File, + sourceRoots: Iterable, + args: K2JSCompilerArguments, + messageCollector: MessageCollector = MessageCollector.NONE, + reporter: ICReporter = EmptyICReporter +) { + val versions = commonCacheVersions(cachesDir) + standaloneCacheVersion(cachesDir) + val allKotlinFiles = sourceRoots.asSequence().flatMap { it.walk() } + .filter { it.isFile && it.extension.equals("kt", ignoreCase = true) }.toList() + + withJsIC { + val compiler = IncrementalJsCompilerRunner(cachesDir, versions, reporter) + compiler.compile(allKotlinFiles, args, messageCollector) { + it.inputsCache.sourceSnapshotMap.compareAndUpdate(allKotlinFiles) + } + } +} + +inline fun withJsIC(fn: ()->R): R { + val isJsEnabledBackup = IncrementalCompilation.isEnabledForJs() + IncrementalCompilation.setIsEnabledForJs(true) + + try { + return withIC { fn() } + } + finally { + IncrementalCompilation.setIsEnabledForJs(isJsEnabledBackup) + } +} + +class IncrementalJsCompilerRunner( + workingDir: File, + cacheVersions: List, + reporter: ICReporter +) : IncrementalCompilerRunner( + workingDir, + "caches-js", + cacheVersions, + reporter, + artifactChangesProvider = null, + changesRegistry = null +) { + override fun isICEnabled(): Boolean = + IncrementalCompilation.isEnabled() && IncrementalCompilation.isEnabledForJs() + + override fun createCacheManager(args: K2JSCompilerArguments): IncrementalJsCachesManager = + IncrementalJsCachesManager(cacheDirectory, reporter) + + override fun destionationDir(args: K2JSCompilerArguments): File = + File(args.outputFile).parentFile + + override fun calculateSourcesToCompile(caches: IncrementalJsCachesManager, changedFiles: ChangedFiles.Known, args: K2JSCompilerArguments): CompilationMode { + if (BuildInfo.read(lastBuildInfoFile) == null) return CompilationMode.Rebuild { "No information on previous build" } + + return CompilationMode.Incremental(getDirtyFiles(changedFiles)) + } + + override fun makeServices( + args: K2JSCompilerArguments, + lookupTracker: LookupTracker, + caches: IncrementalJsCachesManager, + compilationMode: CompilationMode + ): Services.Builder = + super.makeServices(args, lookupTracker, caches, compilationMode).apply { + register(IncrementalResultsConsumer::class.java, IncrementalResultsConsumerImpl()) + + if (compilationMode is CompilationMode.Incremental) { + register(IncrementalDataProvider::class.java, IncrementalDataProviderFromCache(caches.platformCache)) + } + } + + override fun updateCaches( + services: Services, + caches: IncrementalJsCachesManager, + generatedFiles: List, + changesCollector: ChangesCollector + ) { + val incrementalResults = services.get(IncrementalResultsConsumer::class.java) as IncrementalResultsConsumerImpl + + val jsCache = caches.platformCache + jsCache.header = incrementalResults.headerMetadata + + return jsCache.compareAndUpdate(incrementalResults.packageParts, changesCollector) + } + + override fun runCompiler( + sourcesToCompile: Set, + args: K2JSCompilerArguments, + caches: IncrementalJsCachesManager, + services: Services, + messageCollector: MessageCollector + ): ExitCode { + val freeArgsBackup = args.freeArgs.toMutableList() + + try { + sourcesToCompile.mapTo(args.freeArgs) { it.absolutePath } + val exitCode = K2JSCompiler().exec(messageCollector, services, args) + reporter.reportCompileIteration(sourcesToCompile, exitCode) + return exitCode + } + finally { + args.freeArgs = freeArgsBackup + } + } +} \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt index 7d0da5ec5de..f9fbdedae75 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt @@ -65,7 +65,7 @@ fun makeIncrementally( } } -private object EmptyICReporter : ICReporter { +object EmptyICReporter : ICReporter { override fun report(message: ()->String) { } } @@ -92,7 +92,7 @@ class IncrementalJvmCompilerRunner( changesRegistry: ChangesRegistry? = null ) : IncrementalCompilerRunner( workingDir, - CACHES_DIR_NAME, + "caches-jvm", cacheVersions, reporter, artifactChangesProvider, @@ -309,10 +309,6 @@ class IncrementalJvmCompilerRunner( moduleFile.delete() } } - - companion object { - const val CACHES_DIR_NAME = "caches" - } } var K2JVMCompilerArguments.destinationAsFile: File diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/cacheVersions.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/cacheVersions.kt index 2dcc094c3e5..5c418e5c514 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/cacheVersions.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/cacheVersions.kt @@ -19,7 +19,7 @@ package org.jetbrains.kotlin.incremental import org.jetbrains.kotlin.config.IncrementalCompilation import java.io.File -internal const val STANDALONE_CACHE_VERSION = 1 +internal const val STANDALONE_CACHE_VERSION = 2 internal const val STANDALONE_VERSION_FILE_NAME = "standalone-ic-format-version.txt" fun standaloneCacheVersion(dataRoot: File): CacheVersion = diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProviderFromCache.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProviderFromCache.kt new file mode 100644 index 00000000000..306e967a02a --- /dev/null +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProviderFromCache.kt @@ -0,0 +1,27 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.incremental.js + +import org.jetbrains.kotlin.incremental.IncrementalJsCache +import java.io.File + +internal class IncrementalDataProviderFromCache(private val cache: IncrementalJsCache) : IncrementalDataProvider { + override val headerMetadata: ByteArray + get() = cache.header + override val compiledPackageParts: Map + get() = cache.nonDirtyPackageParts() +} \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunnerTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunnerTest.kt new file mode 100644 index 00000000000..96f20ac0a39 --- /dev/null +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunnerTest.kt @@ -0,0 +1,38 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.incremental + +import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments +import org.jetbrains.kotlin.incremental.utils.TestCompilationResult +import org.jetbrains.kotlin.incremental.utils.TestICReporter +import org.jetbrains.kotlin.incremental.utils.TestMessageCollector +import java.io.File + +class IncrementalJsCompilerRunnerTest : IncrementalCompilerRunnerTestBase() { + override fun make(cacheDir: File, sourceRoots: Iterable, args: K2JSCompilerArguments): TestCompilationResult { + val reporter = TestICReporter() + val messageCollector = TestMessageCollector() + makeJsIncrementally(cacheDir, sourceRoots, args, reporter = reporter, messageCollector = messageCollector) + return TestCompilationResult(reporter, messageCollector) + } + + override fun createCompilerArguments(destinationDir: File, testDir: File): K2JSCompilerArguments = + K2JSCompilerArguments().apply { + outputFile = File(destinationDir, "${testDir.name}.js").path + libraries = File(bootstrapKotlincLib, "kotlin-stdlib-js.jar").path + } +} \ No newline at end of file diff --git a/compiler/util/src/org/jetbrains/kotlin/config/IncrementalCompilation.java b/compiler/util/src/org/jetbrains/kotlin/config/IncrementalCompilation.java index 51d253910c3..a248f8988a9 100644 --- a/compiler/util/src/org/jetbrains/kotlin/config/IncrementalCompilation.java +++ b/compiler/util/src/org/jetbrains/kotlin/config/IncrementalCompilation.java @@ -1,5 +1,5 @@ /* - * Copyright 2010-2015 JetBrains s.r.o. + * Copyright 2010-2017 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,13 +20,23 @@ import org.jetbrains.annotations.TestOnly; public class IncrementalCompilation { private static final String INCREMENTAL_COMPILATION_PROPERTY = "kotlin.incremental.compilation"; + private static final String INCREMENTAL_COMPILATION_JS_PROPERTY = "kotlin.incremental.compilation.js"; public static boolean isEnabled() { return "true".equals(System.getProperty(INCREMENTAL_COMPILATION_PROPERTY)); } + public static boolean isEnabledForJs() { + return "true".equals(System.getProperty(INCREMENTAL_COMPILATION_JS_PROPERTY)); + } + @TestOnly public static void setIsEnabled(boolean value) { System.setProperty(INCREMENTAL_COMPILATION_PROPERTY, String.valueOf(value)); } + + @TestOnly + public static void setIsEnabledForJs(boolean value) { + System.setProperty(INCREMENTAL_COMPILATION_JS_PROPERTY, String.valueOf(value)); + } } \ No newline at end of file