Implement JS IC

build-common/build-common.iml

@@ -16,5 +16,6 @@
     <orderEntry type="library" scope="TEST" name="idea-full" level="project" />
     <orderEntry type="library" name="kotlin-reflect" level="project" />
     <orderEntry type="module" module-name="js.serializer" />
+    <orderEntry type="module" module-name="js.frontend" />
   </component>
 </module>

build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt

@@ -0,0 +1,189 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import com.intellij.util.io.DataExternalizer
+import org.jetbrains.kotlin.incremental.js.TranslationResultValue
+import org.jetbrains.kotlin.incremental.storage.*
+import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.serialization.ProtoBuf
+import org.jetbrains.kotlin.serialization.deserialization.NameResolverImpl
+import org.jetbrains.kotlin.serialization.js.JsProtoBuf
+import org.jetbrains.kotlin.serialization.js.JsSerializerProtocol
+import java.io.DataInput
+import java.io.DataOutput
+import java.io.File
+
+open class IncrementalJsCache(cachesDir: File) : IncrementalCacheCommon(cachesDir) {
+    companion object {
+        private val TRANSLATION_RESULT_MAP = "translation-result"
+        private val SOURCES_TO_CLASSES_FQNS = "sources-to-classes"
+        private val HEADER_FILE_NAME = "header.meta"
+    }
+
+    private val dirtySources = arrayListOf<File>()
+    private val translationResults = registerMap(TranslationResultMap(TRANSLATION_RESULT_MAP.storageFile))
+    private val sourcesToClasses = registerMap(SourceToClassesMap(SOURCES_TO_CLASSES_FQNS.storageFile))
+
+    private val headerFile: File
+        get() = File(cachesDir, HEADER_FILE_NAME)
+
+    var header: ByteArray
+        get() = headerFile.readBytes()
+        set(value) {
+            cachesDir.mkdirs()
+            headerFile.writeBytes(value)
+        }
+
+    override fun markDirty(removedAndCompiledSources: List<File>) {
+        dirtySources.addAll(removedAndCompiledSources)
+    }
+
+    fun compareAndUpdate(translatedFiles: Map<File, TranslationResultValue>, changesCollector: ChangesCollector) {
+        dirtySources.forEach {
+            if (it !in translatedFiles) {
+                translationResults.remove(it, changesCollector)
+            }
+
+            removeAllFromClassStorage(sourcesToClasses[it])
+            sourcesToClasses.clearOutputsForSource(it)
+        }
+        dirtySources.clear()
+
+        for ((srcFile, data) in translatedFiles) {
+            val (binaryMetadata, binaryAst) = data
+
+            val oldProtoMap = translationResults[srcFile]?.metadata?.let { getProtoData(srcFile, it) } ?: emptyMap()
+            val newProtoMap = getProtoData(srcFile, binaryMetadata)
+
+            for (protoData in newProtoMap.values) {
+                if (protoData is ClassProtoData) {
+                    addToClassStorage(protoData.proto, protoData.nameResolver, srcFile)
+                }
+            }
+
+            for (classId in oldProtoMap.keys + newProtoMap.keys) {
+                changesCollector.collectProtoChanges(oldProtoMap[classId], newProtoMap[classId])
+            }
+
+            translationResults.put(srcFile, binaryMetadata, binaryAst)
+        }
+    }
+
+    fun nonDirtyPackageParts(): Map<File, TranslationResultValue> =
+            hashMapOf<File, TranslationResultValue>().apply {
+                for (path in translationResults.keys()) {
+                    val file = File(path)
+                    if (file !in dirtySources) {
+                        put(file, translationResults[path]!!)
+                    }
+                }
+            }
+}
+
+private class SourceToClassesMap(storageFile: File) : BasicStringMap<Collection<String>>(storageFile, PathStringDescriptor, StringCollectionExternalizer) {
+    fun clearOutputsForSource(sourceFile: File) {
+        remove(sourceFile.canonicalPath)
+    }
+
+    fun add(sourceFile: File, className: FqName) {
+        storage.append(sourceFile.canonicalPath, className.asString())
+    }
+
+    operator fun get(sourceFile: File): Collection<FqName> =
+            storage[sourceFile.canonicalPath].orEmpty().map { FqName(it) }
+
+    override fun dumpValue(value: Collection<String>) = value.dumpCollection()
+
+    private fun remove(path: String) {
+        storage.remove(path)
+    }
+}
+
+private object TranslationResultValueExternalizer : DataExternalizer<TranslationResultValue> {
+    override fun save(output: DataOutput, value: TranslationResultValue) {
+        output.writeInt(value.metadata.size)
+        output.write(value.metadata)
+
+        output.writeInt(value.binaryAst.size)
+        output.write(value.binaryAst)
+    }
+
+    override fun read(input: DataInput): TranslationResultValue {
+        val metadataSize = input.readInt()
+        val metadata = ByteArray(metadataSize)
+        input.readFully(metadata)
+
+        val binaryAstSize = input.readInt()
+        val binaryAst = ByteArray(binaryAstSize)
+        input.readFully(binaryAst)
+
+        return TranslationResultValue(metadata = metadata, binaryAst = binaryAst)
+    }
+}
+
+private class TranslationResultMap(storageFile: File) : BasicStringMap<TranslationResultValue>(storageFile, TranslationResultValueExternalizer) {
+    override fun dumpValue(value: TranslationResultValue): String =
+            "Metadata: ${value.metadata.md5()}, Binary AST: ${value.binaryAst.md5()}"
+
+    fun put(file: File, newMetadata: ByteArray, newBinaryAst: ByteArray) {
+        storage[file.canonicalPath] = TranslationResultValue(metadata = newMetadata, binaryAst = newBinaryAst)
+    }
+
+    operator fun get(file: File): TranslationResultValue? =
+            storage[file.canonicalPath]
+
+    operator fun get(key: String): TranslationResultValue? =
+            storage[key]
+
+    fun keys(): Collection<String> =
+            storage.keys
+
+    fun remove(file: File, changesCollector: ChangesCollector) {
+        val protoBytes = storage[file.canonicalPath]!!.metadata
+        val protoMap = getProtoData(file, protoBytes)
+
+        for ((_, protoData) in protoMap) {
+            changesCollector.collectProtoChanges(oldData = protoData, newData = null)
+        }
+        storage.remove(file.canonicalPath)
+    }
+}
+
+fun getProtoData(sourceFile: File, metadata: ByteArray): Map<ClassId, ProtoData>  {
+    val classes = hashMapOf<ClassId, ProtoData>()
+    val proto = ProtoBuf.PackageFragment.parseFrom(metadata, JsSerializerProtocol.extensionRegistry)
+    val nameResolver = NameResolverImpl(proto.strings, proto.qualifiedNames)
+
+    proto.class_List.forEach {
+        val classId = nameResolver.getClassId(it.fqName)
+        classes[classId] = ClassProtoData(it, nameResolver)
+    }
+
+    proto.`package`.apply {
+        val packageFqName = if (hasExtension(JsProtoBuf.packageFqName)) {
+            nameResolver.getPackageFqName(getExtension(JsProtoBuf.packageFqName))
+        }
+        else FqName.ROOT
+
+        val packagePartClassId = ClassId(packageFqName, Name.identifier(sourceFile.nameWithoutExtension.capitalize() + "Kt"))
+        classes[packagePartClassId] = PackagePartProtoData(this, nameResolver, packageFqName)
+    }
+    return classes
+}

build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMapsOwner.kt

@@ -19,7 +19,7 @@ package org.jetbrains.kotlin.incremental.storage
 import org.jetbrains.annotations.TestOnly
 import java.io.File
 
-open class BasicMapsOwner(private val baseDir: File) {
+open class BasicMapsOwner(val cachesDir: File) {
     private val maps = arrayListOf<BasicMap<*, *>>()
 
     companion object {
@@ -27,7 +27,7 @@ open class BasicMapsOwner(private val baseDir: File) {
     }
 
     protected val String.storageFile: File
-        get() = File(baseDir, this + "." + CACHE_EXTENSION)
+        get() = File(cachesDir, this + "." + CACHE_EXTENSION)
 
     protected fun <K, V, M : BasicMap<K, V>> registerMap(map: M): M {
         maps.add(map)

compiler/cli/src/org/jetbrains/kotlin/cli/js/K2JSCompiler.java

@@ -16,7 +16,6 @@
 
 package org.jetbrains.kotlin.cli.js;
 
-import com.google.common.collect.Lists;
 import com.intellij.openapi.Disposable;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.util.io.FileUtil;

compiler/incremental-compilation-impl/incremental-compilation-impl.iml

@@ -17,5 +17,6 @@
     <orderEntry type="module" module-name="frontend.java" />
     <orderEntry type="module" module-name="util" />
     <orderEntry type="library" name="junit-4.12" level="project" />
+    <orderEntry type="module" module-name="js.frontend" />
   </component>
 </module>

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt

@@ -76,3 +76,12 @@ class IncrementalJvmCachesManager(
     private val jvmCacheDir = File(cacheDirectory, "jvm").apply { mkdirs() }
     override val platformCache = IncrementalCacheImpl(jvmCacheDir, outputDir).apply { registerCache() }
 }
+
+class IncrementalJsCachesManager(
+        cachesRootDir: File,
+        reporter: ICReporter
+) : IncrementalCachesManager<IncrementalJsCache>(cachesRootDir, reporter) {
+
+    private val jsCacheFile = File(cachesRootDir, "js").apply { mkdirs() }
+    override val platformCache = IncrementalJsCache(jsCacheFile).apply { registerCache() }
+}

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt

@@ -0,0 +1,136 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import org.jetbrains.kotlin.build.GeneratedFile
+import org.jetbrains.kotlin.cli.common.ExitCode
+import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
+import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import org.jetbrains.kotlin.cli.js.K2JSCompiler
+import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
+import org.jetbrains.kotlin.config.IncrementalCompilation
+import org.jetbrains.kotlin.config.Services
+import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.incremental.js.*
+import java.io.File
+
+fun makeJsIncrementally(
+        cachesDir: File,
+        sourceRoots: Iterable<File>,
+        args: K2JSCompilerArguments,
+        messageCollector: MessageCollector = MessageCollector.NONE,
+        reporter: ICReporter = EmptyICReporter
+) {
+    val versions = commonCacheVersions(cachesDir) + standaloneCacheVersion(cachesDir)
+    val allKotlinFiles = sourceRoots.asSequence().flatMap { it.walk() }
+            .filter { it.isFile && it.extension.equals("kt", ignoreCase = true) }.toList()
+
+    withJsIC {
+        val compiler = IncrementalJsCompilerRunner(cachesDir, versions, reporter)
+        compiler.compile(allKotlinFiles, args, messageCollector) {
+            it.inputsCache.sourceSnapshotMap.compareAndUpdate(allKotlinFiles)
+        }
+    }
+}
+
+inline fun <R> withJsIC(fn: ()->R): R {
+    val isJsEnabledBackup = IncrementalCompilation.isEnabledForJs()
+    IncrementalCompilation.setIsEnabledForJs(true)
+
+    try {
+        return withIC { fn() }
+    }
+    finally {
+        IncrementalCompilation.setIsEnabledForJs(isJsEnabledBackup)
+    }
+}
+
+class IncrementalJsCompilerRunner(
+        workingDir: File,
+        cacheVersions: List<CacheVersion>,
+        reporter: ICReporter
+) : IncrementalCompilerRunner<K2JSCompilerArguments, IncrementalJsCachesManager>(
+        workingDir,
+        "caches-js",
+        cacheVersions,
+        reporter,
+        artifactChangesProvider = null,
+        changesRegistry = null
+) {
+    override fun isICEnabled(): Boolean =
+        IncrementalCompilation.isEnabled() && IncrementalCompilation.isEnabledForJs()
+
+    override fun createCacheManager(args: K2JSCompilerArguments): IncrementalJsCachesManager =
+        IncrementalJsCachesManager(cacheDirectory, reporter)
+
+    override fun destionationDir(args: K2JSCompilerArguments): File =
+        File(args.outputFile).parentFile
+
+    override fun calculateSourcesToCompile(caches: IncrementalJsCachesManager, changedFiles: ChangedFiles.Known, args: K2JSCompilerArguments): CompilationMode {
+        if (BuildInfo.read(lastBuildInfoFile) == null) return CompilationMode.Rebuild { "No information on previous build" }
+
+        return CompilationMode.Incremental(getDirtyFiles(changedFiles))
+    }
+
+    override fun makeServices(
+            args: K2JSCompilerArguments,
+            lookupTracker: LookupTracker,
+            caches: IncrementalJsCachesManager,
+            compilationMode: CompilationMode
+    ): Services.Builder =
+        super.makeServices(args, lookupTracker, caches, compilationMode).apply {
+            register(IncrementalResultsConsumer::class.java, IncrementalResultsConsumerImpl())
+
+            if (compilationMode is CompilationMode.Incremental) {
+                register(IncrementalDataProvider::class.java, IncrementalDataProviderFromCache(caches.platformCache))
+            }
+        }
+
+    override fun updateCaches(
+            services: Services,
+            caches: IncrementalJsCachesManager,
+            generatedFiles: List<GeneratedFile>,
+            changesCollector: ChangesCollector
+    ) {
+        val incrementalResults = services.get(IncrementalResultsConsumer::class.java) as IncrementalResultsConsumerImpl
+
+        val jsCache = caches.platformCache
+        jsCache.header = incrementalResults.headerMetadata
+
+        return jsCache.compareAndUpdate(incrementalResults.packageParts, changesCollector)
+    }
+
+    override fun runCompiler(
+            sourcesToCompile: Set<File>,
+            args: K2JSCompilerArguments,
+            caches: IncrementalJsCachesManager,
+            services: Services,
+            messageCollector: MessageCollector
+    ): ExitCode {
+        val freeArgsBackup = args.freeArgs.toMutableList()
+
+        try {
+            sourcesToCompile.mapTo(args.freeArgs) { it.absolutePath }
+            val exitCode = K2JSCompiler().exec(messageCollector, services, args)
+            reporter.reportCompileIteration(sourcesToCompile, exitCode)
+            return exitCode
+        }
+        finally {
+            args.freeArgs = freeArgsBackup
+        }
+    }
+}

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt

@@ -65,7 +65,7 @@ fun makeIncrementally(
     }
 }
 
-private object EmptyICReporter : ICReporter {
+object EmptyICReporter : ICReporter {
     override fun report(message: ()->String) {
     }
 }
@@ -92,7 +92,7 @@ class IncrementalJvmCompilerRunner(
         changesRegistry: ChangesRegistry? = null
 ) : IncrementalCompilerRunner<K2JVMCompilerArguments, IncrementalJvmCachesManager>(
         workingDir,
-        CACHES_DIR_NAME,
+        "caches-jvm",
         cacheVersions,
         reporter,
         artifactChangesProvider,
@@ -309,10 +309,6 @@ class IncrementalJvmCompilerRunner(
             moduleFile.delete()
         }
     }
-
-    companion object {
-        const val CACHES_DIR_NAME = "caches"
-    }
 }
 
 var K2JVMCompilerArguments.destinationAsFile: File

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/cacheVersions.kt

@@ -19,7 +19,7 @@ package org.jetbrains.kotlin.incremental
 import org.jetbrains.kotlin.config.IncrementalCompilation
 import java.io.File
 
-internal const val STANDALONE_CACHE_VERSION = 1
+internal const val STANDALONE_CACHE_VERSION = 2
 internal const val STANDALONE_VERSION_FILE_NAME = "standalone-ic-format-version.txt"
 
 fun standaloneCacheVersion(dataRoot: File): CacheVersion =

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/js/IncrementalDataProviderFromCache.kt

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental.js
+
+import org.jetbrains.kotlin.incremental.IncrementalJsCache
+import java.io.File
+
+internal class IncrementalDataProviderFromCache(private val cache: IncrementalJsCache) : IncrementalDataProvider {
+    override val headerMetadata: ByteArray
+        get() = cache.header
+    override val compiledPackageParts: Map<File, TranslationResultValue>
+        get() = cache.nonDirtyPackageParts()
+}

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunnerTest.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
+import org.jetbrains.kotlin.incremental.utils.TestCompilationResult
+import org.jetbrains.kotlin.incremental.utils.TestICReporter
+import org.jetbrains.kotlin.incremental.utils.TestMessageCollector
+import java.io.File
+
+class IncrementalJsCompilerRunnerTest : IncrementalCompilerRunnerTestBase<K2JSCompilerArguments>() {
+    override fun make(cacheDir: File, sourceRoots: Iterable<File>, args: K2JSCompilerArguments): TestCompilationResult {
+        val reporter = TestICReporter()
+        val messageCollector = TestMessageCollector()
+        makeJsIncrementally(cacheDir, sourceRoots, args, reporter = reporter, messageCollector = messageCollector)
+        return TestCompilationResult(reporter, messageCollector)
+    }
+
+    override fun createCompilerArguments(destinationDir: File, testDir: File): K2JSCompilerArguments =
+            K2JSCompilerArguments().apply {
+                outputFile = File(destinationDir, "${testDir.name}.js").path
+                libraries = File(bootstrapKotlincLib, "kotlin-stdlib-js.jar").path
+            }
+}

compiler/util/src/org/jetbrains/kotlin/config/IncrementalCompilation.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2010-2015 JetBrains s.r.o.
+ * Copyright 2010-2017 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,13 +20,23 @@ import org.jetbrains.annotations.TestOnly;
 
 public class IncrementalCompilation {
     private static final String INCREMENTAL_COMPILATION_PROPERTY = "kotlin.incremental.compilation";
+    private static final String INCREMENTAL_COMPILATION_JS_PROPERTY = "kotlin.incremental.compilation.js";
 
     public static boolean isEnabled() {
         return "true".equals(System.getProperty(INCREMENTAL_COMPILATION_PROPERTY));
     }
 
+    public static boolean isEnabledForJs() {
+        return "true".equals(System.getProperty(INCREMENTAL_COMPILATION_JS_PROPERTY));
+    }
+
     @TestOnly
     public static void setIsEnabled(boolean value) {
         System.setProperty(INCREMENTAL_COMPILATION_PROPERTY, String.valueOf(value));
     }
+
+    @TestOnly
+    public static void setIsEnabledForJs(boolean value) {
+        System.setProperty(INCREMENTAL_COMPILATION_JS_PROPERTY, String.valueOf(value));
+    }
 }