Compile actual and expected declarations together

#KT-20840 fixed

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ExpectActualTrackerImpl.kt

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
+import java.io.File
+
+internal class ExpectActualTrackerImpl : ExpectActualTracker {
+    private val expectToActual = hashMapOf<File, MutableSet<File>>()
+
+    val expectToActualMap: Map<File, Set<File>>
+        get() = expectToActual
+
+    override fun report(expectedFile: File, actualFile: File) {
+        expectToActual.getOrPut(expectedFile) { hashSetOf() }.add(actualFile)
+    }
+}

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt

@@ -25,6 +25,7 @@ import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl
 import org.jetbrains.kotlin.compilerRunner.SimpleOutputItem
 import org.jetbrains.kotlin.compilerRunner.toGeneratedFile
 import org.jetbrains.kotlin.config.Services
+import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.multiproject.ArtifactChangesProvider
 import org.jetbrains.kotlin.incremental.multiproject.ChangesRegistry
@@ -153,11 +154,13 @@ abstract class IncrementalCompilerRunner<
     protected open fun makeServices(
             args: Args,
             lookupTracker: LookupTracker,
+            expectActualTracker: ExpectActualTracker,
             caches: CacheManager,
             compilationMode: CompilationMode
     ): Services.Builder =
         Services.Builder().apply {
             register(LookupTracker::class.java, lookupTracker)
+            register(ExpectActualTracker::class.java, expectActualTracker)
             register(CompilationCanceledStatus::class.java, EmptyCompilationCanceledStatus)
         }
 
@@ -191,10 +194,13 @@ abstract class IncrementalCompilerRunner<
         var exitCode = ExitCode.OK
 
         while (dirtySources.any() || runWithNoDirtyKotlinSources(caches)) {
+            val complementaryFiles = caches.inputsCache.clearComplementaryFilesMapping(dirtySources)
+            dirtySources.addAll(complementaryFiles)
             caches.platformCache.markDirty(dirtySources)
             caches.inputsCache.removeOutputForSourceFiles(dirtySources)
 
             val lookupTracker = LookupTrackerImpl(LookupTracker.DO_NOTHING)
+            val expectActualTracker = ExpectActualTrackerImpl()
             val (sourcesToCompile, removedKotlinSources) = dirtySources.partition(File::exists)
 
             // todo: more optimal to save only last iteration, but it will require adding standalone-ic specific logs
@@ -203,7 +209,7 @@ abstract class IncrementalCompilerRunner<
             val text = allSourcesToCompile.joinToString(separator = System.getProperty("line.separator")) { it.canonicalPath }
             dirtySourcesSinceLastTimeFile.writeText(text)
 
-            val services = makeServices(args, lookupTracker, caches, compilationMode).build()
+            val services = makeServices(args, lookupTracker, expectActualTracker, caches, compilationMode).build()
 
             args.reportOutputFiles = true
             val outputItemsCollector = OutputItemsCollectorImpl()
@@ -227,6 +233,7 @@ abstract class IncrementalCompilerRunner<
                 }
             }
 
+            caches.inputsCache.registerComplementaryFiles(expectActualTracker)
             caches.inputsCache.registerOutputForSourceFiles(generatedFiles)
             caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources)
             val changesCollector = ChangesCollector()

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt

@@ -24,6 +24,7 @@ import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.cli.js.K2JSCompiler
 import org.jetbrains.kotlin.config.IncrementalCompilation
 import org.jetbrains.kotlin.config.Services
+import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.js.*
 import java.io.File
@@ -97,10 +98,11 @@ class IncrementalJsCompilerRunner(
     override fun makeServices(
             args: K2JSCompilerArguments,
             lookupTracker: LookupTracker,
+            expectActualTracker: ExpectActualTracker,
             caches: IncrementalJsCachesManager,
             compilationMode: CompilationMode
     ): Services.Builder =
-        super.makeServices(args, lookupTracker, caches, compilationMode).apply {
+        super.makeServices(args, lookupTracker, expectActualTracker, caches, compilationMode).apply {
             register(IncrementalResultsConsumer::class.java, IncrementalResultsConsumerImpl())
 
             if (compilationMode is CompilationMode.Incremental) {

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt

@@ -37,6 +37,7 @@ import org.jetbrains.kotlin.compilerRunner.ArgumentUtils
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.IncrementalCompilation
 import org.jetbrains.kotlin.config.Services
+import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.multiproject.ArtifactChangesProvider
 import org.jetbrains.kotlin.incremental.multiproject.ChangesRegistry
@@ -401,10 +402,11 @@ class IncrementalJvmCompilerRunner(
     override fun makeServices(
             args: K2JVMCompilerArguments,
             lookupTracker: LookupTracker,
+            expectActualTracker: ExpectActualTracker,
             caches: IncrementalJvmCachesManager,
             compilationMode: CompilationMode
     ): Services.Builder =
-        super.makeServices(args, lookupTracker, caches, compilationMode).apply {
+        super.makeServices(args, lookupTracker, expectActualTracker, caches, compilationMode).apply {
             val targetId = TargetId(args.moduleName!!, "java-production")
             val targetToCache = mapOf(targetId to caches.platformCache)
             val incrementalComponents = IncrementalCompilationComponentsImpl(targetToCache)

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt

@@ -23,20 +23,63 @@ import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner
 import org.jetbrains.kotlin.incremental.storage.BasicStringMap
 import org.jetbrains.kotlin.incremental.storage.PathStringDescriptor
 import org.jetbrains.kotlin.incremental.storage.StringCollectionExternalizer
-import org.jetbrains.kotlin.modules.TargetId
 import java.io.File
+import java.util.*
+import kotlin.collections.HashSet
 
 class InputsCache(
         workingDir: File,
         private val reporter: ICReporter
 ) : BasicMapsOwner(workingDir) {
     companion object {
-        private val SOURCE_TO_OUTPUT_FILES = "source-to-output"
         private val SOURCE_SNAPSHOTS = "source-snapshot"
+        private val SOURCE_TO_OUTPUT_FILES = "source-to-output"
+        private val COMPLEMENTARY_FILES = "complementary-files"
     }
 
-    internal val sourceToOutputMap = registerMap(SourceToOutputFilesMap(SOURCE_TO_OUTPUT_FILES.storageFile))
     internal val sourceSnapshotMap = registerMap(FileSnapshotMap(SOURCE_SNAPSHOTS.storageFile))
+    private val sourceToOutputMap = registerMap(FilesMap(SOURCE_TO_OUTPUT_FILES.storageFile))
+    /**
+     * A file X is a complementary to a file Y if they contain corresponding expect/actual declarations.
+     * Complementary files should be compiled together during IC so the compiler does not complain
+     * about missing parts.
+     * TODO: provide a better solution (maintain an index of expect/actual declarations akin to IncrementalPackagePartProvider)
+     */
+    private val complementaryFilesMap = registerMap(FilesMap(COMPLEMENTARY_FILES.storageFile))
+
+    fun clearComplementaryFilesMapping(dirtyFiles: Collection<File>): Collection<File> {
+        val complementaryFiles = HashSet<File>()
+        val filesQueue = ArrayDeque(dirtyFiles)
+        while (filesQueue.isNotEmpty()) {
+            val file = filesQueue.pollFirst()
+            complementaryFilesMap.remove(file).filterTo(filesQueue) { complementaryFiles.add(it) }
+        }
+        complementaryFiles.removeAll(dirtyFiles)
+        return complementaryFiles
+    }
+
+    internal fun registerComplementaryFiles(expectActualTracker: ExpectActualTrackerImpl) {
+        val actualToExpect = hashMapOf<File, MutableSet<File>>()
+        for ((expect, actuals) in expectActualTracker.expectToActualMap) {
+            for (actual in actuals) {
+                actualToExpect.getOrPut(actual) { hashSetOf() }.add(expect)
+            }
+            complementaryFilesMap[expect] = actuals
+        }
+
+        for ((actual, expects) in actualToExpect) {
+            complementaryFilesMap[actual] = expects
+        }
+    }
+
+    fun removeOutputForSourceFiles(sources: Iterable<File>) {
+        for (sourceFile in sources) {
+            sourceToOutputMap.remove(sourceFile).forEach { it ->
+                reporter.report { "Deleting $it on clearing cache for $sourceFile" }
+                it.delete()
+            }
+        }
+    }
 
     // generatedFiles can contain multiple entries with the same source file
     // for example Kapt3 IC will generate a .java stub and .class stub for each source file
@@ -53,29 +96,21 @@ class InputsCache(
             sourceToOutputMap[source] = outputs
         }
     }
+}
 
-    fun removeOutputForSourceFiles(sources: Iterable<File>) {
-        sources.forEach { sourceToOutputMap.remove(it) }
+private class FilesMap(storageFile: File)
+    : BasicStringMap<Collection<String>>(storageFile, PathStringDescriptor, StringCollectionExternalizer) {
+
+    operator fun set(sourceFile: File, outputFiles: Collection<File>) {
+        storage[sourceFile.absolutePath] = outputFiles.map { it.absolutePath }
     }
 
-    inner class SourceToOutputFilesMap(storageFile: File) : BasicStringMap<Collection<String>>(storageFile, PathStringDescriptor, StringCollectionExternalizer) {
-        operator fun set(sourceFile: File, outputFiles: Collection<File>) {
-            storage[sourceFile.absolutePath] = outputFiles.map { it.absolutePath }
-        }
+    operator fun get(sourceFile: File): Collection<File> =
+            storage[sourceFile.absolutePath].orEmpty().map(::File)
 
-        operator fun get(sourceFile: File): Collection<File> =
-                storage[sourceFile.absolutePath].orEmpty().map(::File)
+    override fun dumpValue(value: Collection<String>) =
+            value.dumpCollection()
 
-        override fun dumpValue(value: Collection<String>) = value.dumpCollection()
-
-        fun remove(file: File) {
-            // TODO: do it in the code that uses cache, since cache should not generally delete anything outside of it!
-            // but for a moment it is an easiest solution to implement
-            get(file).forEach {
-                reporter.report { "Deleting $it on clearing cache for $file" }
-                it.delete()
-            }
-            storage.remove(file.absolutePath)
-        }
-    }
+    fun remove(file: File): Collection<File> =
+            get(file).also { storage.remove(file.absolutePath) }
 }

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalMultiplatformJsCompilerRunnerTest.kt

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
+import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
+import java.io.File
+
+abstract class AbstractIncrementalMultiplatformJsCompilerRunnerTest : AbstractIncrementalJsCompilerRunnerTest() {
+    override fun createCompilerArguments(destinationDir: File, testDir: File): K2JSCompilerArguments {
+        return super.createCompilerArguments(destinationDir, testDir).apply {
+            multiPlatform = true
+        }
+    }
+}

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/AbstractIncrementalMultiplatformJvmCompilerRunnerTest.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
+import java.io.File
+
+abstract class AbstractIncrementalMultiplatformJvmCompilerRunnerTest : AbstractIncrementalJvmCompilerRunnerTest() {
+    override fun createCompilerArguments(destinationDir: File, testDir: File): K2JVMCompilerArguments {
+        return super.createCompilerArguments(destinationDir, testDir).apply {
+            multiPlatform = true
+        }
+    }
+}

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/IncrementalMultiplatformJsCompilerRunnerTestGenerated.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental;
+
+import com.intellij.testFramework.TestDataPath;
+import org.jetbrains.kotlin.test.JUnit3RunnerWithInners;
+import org.jetbrains.kotlin.test.KotlinTestUtils;
+import org.jetbrains.kotlin.test.TargetBackend;
+import org.jetbrains.kotlin.test.TestMetadata;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.util.regex.Pattern;
+
+/** This class is generated by {@link org.jetbrains.kotlin.generators.tests.TestsPackage}. DO NOT MODIFY MANUALLY */
+@SuppressWarnings("all")
+@TestMetadata("jps-plugin/testData/incremental/multiplatform")
+@TestDataPath("$PROJECT_ROOT")
+@RunWith(JUnit3RunnerWithInners.class)
+public class IncrementalMultiplatformJsCompilerRunnerTestGenerated extends AbstractIncrementalMultiplatformJsCompilerRunnerTest {
+    public void testAllFilesPresentInMultiplatform() throws Exception {
+        KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("jps-plugin/testData/incremental/multiplatform"), Pattern.compile("^([^\\.]+)$"), TargetBackend.ANY, true);
+    }
+
+    @TestMetadata("touchActual")
+    public void testTouchActual() throws Exception {
+        String fileName = KotlinTestUtils.navigationMetadata("jps-plugin/testData/incremental/multiplatform/touchActual/");
+        doTest(fileName);
+    }
+
+    @TestMetadata("touchExpect")
+    public void testTouchExpect() throws Exception {
+        String fileName = KotlinTestUtils.navigationMetadata("jps-plugin/testData/incremental/multiplatform/touchExpect/");
+        doTest(fileName);
+    }
+}

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/IncrementalMultiplatformJvmCompilerRunnerTestGenerated.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental;
+
+import com.intellij.testFramework.TestDataPath;
+import org.jetbrains.kotlin.test.JUnit3RunnerWithInners;
+import org.jetbrains.kotlin.test.KotlinTestUtils;
+import org.jetbrains.kotlin.test.TargetBackend;
+import org.jetbrains.kotlin.test.TestMetadata;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.util.regex.Pattern;
+
+/** This class is generated by {@link org.jetbrains.kotlin.generators.tests.TestsPackage}. DO NOT MODIFY MANUALLY */
+@SuppressWarnings("all")
+@TestMetadata("jps-plugin/testData/incremental/multiplatform")
+@TestDataPath("$PROJECT_ROOT")
+@RunWith(JUnit3RunnerWithInners.class)
+public class IncrementalMultiplatformJvmCompilerRunnerTestGenerated extends AbstractIncrementalMultiplatformJvmCompilerRunnerTest {
+    public void testAllFilesPresentInMultiplatform() throws Exception {
+        KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("jps-plugin/testData/incremental/multiplatform"), Pattern.compile("^([^\\.]+)$"), TargetBackend.ANY, true);
+    }
+
+    @TestMetadata("touchActual")
+    public void testTouchActual() throws Exception {
+        String fileName = KotlinTestUtils.navigationMetadata("jps-plugin/testData/incremental/multiplatform/touchActual/");
+        doTest(fileName);
+    }
+
+    @TestMetadata("touchExpect")
+    public void testTouchExpect() throws Exception {
+        String fileName = KotlinTestUtils.navigationMetadata("jps-plugin/testData/incremental/multiplatform/touchExpect/");
+        doTest(fileName);
+    }
+}