jps: support multiplatform incremental compilation for jvm and js

- support common modules metadata compilation under flag (it is not required since all common source roots are included transitively for now) - introduce expect actual tracker in jps: move implementation from gradle to build-common - support js incremental compilation: move implementation from gradle to build-common
2018-04-03 13:00:06 +03:00
parent 4a6554a36d
commit 0eee2729cd
89 changed files with 5829 additions and 870 deletions
@@ -1,31 +0,0 @@
-/*
- * Copyright 2010-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.jetbrains.kotlin.incremental
-
-import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
-import java.io.File
-
-internal class ExpectActualTrackerImpl : ExpectActualTracker {
-    private val expectToActual = hashMapOf<File, MutableSet<File>>()
-
-    val expectToActualMap: Map<File, Set<File>>
-        get() = expectToActual
-
-    override fun report(expectedFile: File, actualFile: File) {
-        expectToActual.getOrPut(expectedFile) { hashSetOf() }.add(actualFile)
-    }
-}
@@ -19,7 +19,7 @@ package org.jetbrains.kotlin.incremental
 import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner
 import java.io.File

-abstract class IncrementalCachesManager<PlatformCache : IncrementalCacheCommon<*>>(
+abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache<*>>(
    protected val cachesRootDir: File,
    protected val reporter: ICReporter
 ) {
@@ -200,7 +200,7 @@ abstract class IncrementalCompilerRunner<
        var exitCode = ExitCode.OK

        while (dirtySources.any() || runWithNoDirtyKotlinSources(caches)) {
-            val complementaryFiles = caches.inputsCache.clearComplementaryFilesMapping(dirtySources)
+            val complementaryFiles = caches.platformCache.clearComplementaryFilesMapping(dirtySources)
            dirtySources.addAll(complementaryFiles)
            caches.platformCache.markDirty(dirtySources)
            caches.inputsCache.removeOutputForSourceFiles(dirtySources)
@@ -239,7 +239,7 @@ abstract class IncrementalCompilerRunner<
                }
            }

-            caches.inputsCache.registerComplementaryFiles(expectActualTracker)
+            caches.platformCache.registerComplementaryFiles(expectActualTracker)
            caches.inputsCache.registerOutputForSourceFiles(generatedFiles)
            caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources)
            val changesCollector = ChangesCollector()
@@ -19,10 +19,7 @@ package org.jetbrains.kotlin.incremental
 import com.intellij.util.containers.MultiMap
 import org.jetbrains.kotlin.build.GeneratedFile
 import org.jetbrains.kotlin.incremental.snapshots.FileSnapshotMap
-import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner
-import org.jetbrains.kotlin.incremental.storage.BasicStringMap
-import org.jetbrains.kotlin.incremental.storage.PathStringDescriptor
-import org.jetbrains.kotlin.incremental.storage.StringCollectionExternalizer
+import org.jetbrains.kotlin.incremental.storage.*
 import java.io.File
 import java.util.*
 import kotlin.collections.HashSet
@@ -34,43 +31,10 @@ class InputsCache(
    companion object {
        private val SOURCE_SNAPSHOTS = "source-snapshot"
        private val SOURCE_TO_OUTPUT_FILES = "source-to-output"
-        private val COMPLEMENTARY_FILES = "complementary-files"
    }

    internal val sourceSnapshotMap = registerMap(FileSnapshotMap(SOURCE_SNAPSHOTS.storageFile))
    private val sourceToOutputMap = registerMap(FilesMap(SOURCE_TO_OUTPUT_FILES.storageFile))
-    /**
-     * A file X is a complementary to a file Y if they contain corresponding expect/actual declarations.
-     * Complementary files should be compiled together during IC so the compiler does not complain
-     * about missing parts.
-     * TODO: provide a better solution (maintain an index of expect/actual declarations akin to IncrementalPackagePartProvider)
-     */
-    private val complementaryFilesMap = registerMap(FilesMap(COMPLEMENTARY_FILES.storageFile))
-
-    fun clearComplementaryFilesMapping(dirtyFiles: Collection<File>): Collection<File> {
-        val complementaryFiles = HashSet<File>()
-        val filesQueue = ArrayDeque(dirtyFiles)
-        while (filesQueue.isNotEmpty()) {
-            val file = filesQueue.pollFirst()
-            complementaryFilesMap.remove(file).filterTo(filesQueue) { complementaryFiles.add(it) }
-        }
-        complementaryFiles.removeAll(dirtyFiles)
-        return complementaryFiles
-    }
-
-    internal fun registerComplementaryFiles(expectActualTracker: ExpectActualTrackerImpl) {
-        val actualToExpect = hashMapOf<File, MutableSet<File>>()
-        for ((expect, actuals) in expectActualTracker.expectToActualMap) {
-            for (actual in actuals) {
-                actualToExpect.getOrPut(actual) { hashSetOf() }.add(expect)
-            }
-            complementaryFilesMap[expect] = actuals
-        }
-
-        for ((actual, expects) in actualToExpect) {
-            complementaryFilesMap[actual] = expects
-        }
-    }

    fun removeOutputForSourceFiles(sources: Iterable<File>) {
        for (sourceFile in sources) {
@@ -96,21 +60,4 @@ class InputsCache(
            sourceToOutputMap[source] = outputs
        }
    }
-}
-
-private class FilesMap(storageFile: File)
-    : BasicStringMap<Collection<String>>(storageFile, PathStringDescriptor, StringCollectionExternalizer) {
-
-    operator fun set(sourceFile: File, outputFiles: Collection<File>) {
-        storage[sourceFile.absolutePath] = outputFiles.map { it.absolutePath }
-    }
-
-    operator fun get(sourceFile: File): Collection<File> =
-            storage[sourceFile.absolutePath].orEmpty().map(::File)
-
-    override fun dumpValue(value: Collection<String>) =
-            value.dumpCollection()
-
-    fun remove(file: File): Collection<File> =
-            get(file).also { storage.remove(file.absolutePath) }
 }
@@ -17,7 +17,7 @@ import java.util.regex.Pattern;

 /** This class is generated by {@link org.jetbrains.kotlin.generators.tests.TestsPackage}. DO NOT MODIFY MANUALLY */
@SuppressWarnings("all")
-@TestMetadata("jps-plugin/testData/incremental/multiplatform")
+@TestMetadata("jps-plugin/testData/incremental/multiplatform/singleModule")
@TestDataPath("$PROJECT_ROOT")
@RunWith(JUnit3RunnerWithInners.class)
 public class IncrementalMultiplatformJsCompilerRunnerTestGenerated extends AbstractIncrementalMultiplatformJsCompilerRunnerTest {
@@ -25,21 +25,21 @@ public class IncrementalMultiplatformJsCompilerRunnerTestGenerated extends Abstr
        KotlinTestUtils.runTest(this::doTest, TargetBackend.ANY, testDataFilePath);
    }

-    public void testAllFilesPresentInMultiplatform() throws Exception {
-        KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("jps-plugin/testData/incremental/multiplatform"), Pattern.compile("^([^\\.]+)$"), TargetBackend.ANY, true);
+    public void testAllFilesPresentInSingleModule() throws Exception {
+        KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("jps-plugin/testData/incremental/multiplatform/singleModule"), Pattern.compile("^([^\\.]+)$"), TargetBackend.ANY, true);
    }

    @TestMetadata("touchActual")
    public void testTouchActual() throws Exception {
-        runTest("jps-plugin/testData/incremental/multiplatform/touchActual/");
+        runTest("jps-plugin/testData/incremental/multiplatform/singleModule/touchActual/");
    }

    @TestMetadata("touchExpect")
    public void testTouchExpect() throws Exception {
-        runTest("jps-plugin/testData/incremental/multiplatform/touchExpect/");
+        runTest("jps-plugin/testData/incremental/multiplatform/singleModule/touchExpect/");
    }

-    @TestMetadata("jps-plugin/testData/incremental/multiplatform/touchActual")
+    @TestMetadata("jps-plugin/testData/incremental/multiplatform/singleModule/touchActual")
    @TestDataPath("$PROJECT_ROOT")
    @RunWith(JUnit3RunnerWithInners.class)
    public static class TouchActual extends AbstractIncrementalMultiplatformJsCompilerRunnerTest {
@@ -48,11 +48,11 @@ public class IncrementalMultiplatformJsCompilerRunnerTestGenerated extends Abstr
        }

        public void testAllFilesPresentInTouchActual() throws Exception {
-            KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("jps-plugin/testData/incremental/multiplatform/touchActual"), Pattern.compile("^([^\\.]+)$"), TargetBackend.ANY, true);
+            KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("jps-plugin/testData/incremental/multiplatform/singleModule/touchActual"), Pattern.compile("^([^\\.]+)$"), TargetBackend.ANY, true);
        }
    }

-    @TestMetadata("jps-plugin/testData/incremental/multiplatform/touchExpect")
+    @TestMetadata("jps-plugin/testData/incremental/multiplatform/singleModule/touchExpect")
    @TestDataPath("$PROJECT_ROOT")
    @RunWith(JUnit3RunnerWithInners.class)
    public static class TouchExpect extends AbstractIncrementalMultiplatformJsCompilerRunnerTest {
@@ -61,7 +61,7 @@ public class IncrementalMultiplatformJsCompilerRunnerTestGenerated extends Abstr
        }

        public void testAllFilesPresentInTouchExpect() throws Exception {
-            KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("jps-plugin/testData/incremental/multiplatform/touchExpect"), Pattern.compile("^([^\\.]+)$"), TargetBackend.ANY, true);
+            KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("jps-plugin/testData/incremental/multiplatform/singleModule/touchExpect"), Pattern.compile("^([^\\.]+)$"), TargetBackend.ANY, true);
        }
    }
 }
@@ -17,7 +17,7 @@ import java.util.regex.Pattern;

 /** This class is generated by {@link org.jetbrains.kotlin.generators.tests.TestsPackage}. DO NOT MODIFY MANUALLY */
@SuppressWarnings("all")
-@TestMetadata("jps-plugin/testData/incremental/multiplatform")
+@TestMetadata("jps-plugin/testData/incremental/multiplatform/singleModule")
@TestDataPath("$PROJECT_ROOT")
@RunWith(JUnit3RunnerWithInners.class)
 public class IncrementalMultiplatformJvmCompilerRunnerTestGenerated extends AbstractIncrementalMultiplatformJvmCompilerRunnerTest {
@@ -25,21 +25,21 @@ public class IncrementalMultiplatformJvmCompilerRunnerTestGenerated extends Abst
        KotlinTestUtils.runTest(this::doTest, TargetBackend.ANY, testDataFilePath);
    }

-    public void testAllFilesPresentInMultiplatform() throws Exception {
-        KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("jps-plugin/testData/incremental/multiplatform"), Pattern.compile("^([^\\.]+)$"), TargetBackend.ANY, true);
+    public void testAllFilesPresentInSingleModule() throws Exception {
+        KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("jps-plugin/testData/incremental/multiplatform/singleModule"), Pattern.compile("^([^\\.]+)$"), TargetBackend.ANY, true);
    }

    @TestMetadata("touchActual")
    public void testTouchActual() throws Exception {
-        runTest("jps-plugin/testData/incremental/multiplatform/touchActual/");
+        runTest("jps-plugin/testData/incremental/multiplatform/singleModule/touchActual/");
    }

    @TestMetadata("touchExpect")
    public void testTouchExpect() throws Exception {
-        runTest("jps-plugin/testData/incremental/multiplatform/touchExpect/");
+        runTest("jps-plugin/testData/incremental/multiplatform/singleModule/touchExpect/");
    }

-    @TestMetadata("jps-plugin/testData/incremental/multiplatform/touchActual")
+    @TestMetadata("jps-plugin/testData/incremental/multiplatform/singleModule/touchActual")
    @TestDataPath("$PROJECT_ROOT")
    @RunWith(JUnit3RunnerWithInners.class)
    public static class TouchActual extends AbstractIncrementalMultiplatformJvmCompilerRunnerTest {
@@ -48,11 +48,11 @@ public class IncrementalMultiplatformJvmCompilerRunnerTestGenerated extends Abst
        }

        public void testAllFilesPresentInTouchActual() throws Exception {
-            KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("jps-plugin/testData/incremental/multiplatform/touchActual"), Pattern.compile("^([^\\.]+)$"), TargetBackend.ANY, true);
+            KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("jps-plugin/testData/incremental/multiplatform/singleModule/touchActual"), Pattern.compile("^([^\\.]+)$"), TargetBackend.ANY, true);
        }
    }

-    @TestMetadata("jps-plugin/testData/incremental/multiplatform/touchExpect")
+    @TestMetadata("jps-plugin/testData/incremental/multiplatform/singleModule/touchExpect")
    @TestDataPath("$PROJECT_ROOT")
    @RunWith(JUnit3RunnerWithInners.class)
    public static class TouchExpect extends AbstractIncrementalMultiplatformJvmCompilerRunnerTest {
@@ -61,7 +61,7 @@ public class IncrementalMultiplatformJvmCompilerRunnerTestGenerated extends Abst
        }

        public void testAllFilesPresentInTouchExpect() throws Exception {
-            KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("jps-plugin/testData/incremental/multiplatform/touchExpect"), Pattern.compile("^([^\\.]+)$"), TargetBackend.ANY, true);
+            KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("jps-plugin/testData/incremental/multiplatform/singleModule/touchExpect"), Pattern.compile("^([^\\.]+)$"), TargetBackend.ANY, true);
        }
    }
 }