Move IC from Gradle to compiler

Reasons for moving:
1. Integration with Kotlin daemon for Gradle.
Gradle bundles Kotlin compiler (kotlin-compiler-embeddable) since Gradle 3.2.
There is no plugin isolation in Gradle, so strange compilation errors/exceptions may happen.
We decided to fix this problem by compiling out-of-process using Kotlin daemon.

Reasons for moving IC to the compiler:
* Better isolation from Gradle process.
* Incremental compilation logic is not scattered across two process and multiple modules.
* Makes it possible to implement standalone CLI IC

.idea/modules.xml

@@ -56,6 +56,7 @@
       <module fileurl="file://$PROJECT_DIR$/idea/idea-repl/idea-repl.iml" filepath="$PROJECT_DIR$/idea/idea-repl/idea-repl.iml" group="ide" />
       <module fileurl="file://$PROJECT_DIR$/idea-runner/idea-runner.iml" filepath="$PROJECT_DIR$/idea-runner/idea-runner.iml" group="ide" />
       <module fileurl="file://$PROJECT_DIR$/idea/idea-test-framework/idea-test-framework.iml" filepath="$PROJECT_DIR$/idea/idea-test-framework/idea-test-framework.iml" group="ide" />
+      <module fileurl="file://$PROJECT_DIR$/compiler/incremental-compilation-impl/incremental-compilation-impl.iml" filepath="$PROJECT_DIR$/compiler/incremental-compilation-impl/incremental-compilation-impl.iml" />
       <module fileurl="file://$PROJECT_DIR$/compiler/preloader/instrumentation/instrumentation.iml" filepath="$PROJECT_DIR$/compiler/preloader/instrumentation/instrumentation.iml" group="compiler/cli" />
       <module fileurl="file://$PROJECT_DIR$/compiler/ir/ir.ir2cfg/ir.ir2cfg.iml" filepath="$PROJECT_DIR$/compiler/ir/ir.ir2cfg/ir.ir2cfg.iml" group="compiler/ir" />
       <module fileurl="file://$PROJECT_DIR$/compiler/ir/ir.psi2ir/ir.psi2ir.iml" filepath="$PROJECT_DIR$/compiler/ir/ir.psi2ir/ir.psi2ir.iml" group="compiler/ir" />

build.xml

@@ -107,6 +107,8 @@
         <include name="compiler/plugin-api/src"/>
         <include name="compiler/daemon/src"/>
         <include name="compiler/daemon/daemon-common/src"/>
+        <include name="build-common/src"/>
+        <include name="compiler/incremental-compilation-impl/src"/>
         <include name="compiler/serialization/src"/>
         <include name="compiler/util/src"/>
         <include name="js/js.dart-ast/src"/>
@@ -139,6 +141,8 @@
         <include name="conditional-preprocessor/**"/>
         <include name="daemon/**"/>
         <include name="daemon-common/**"/>
+        <include name="build-common/**"/>
+        <include name="incremental-compilation-impl/**"/>
         <include name="util/**"/>
         <include name="util.runtime/**"/>
         <include name="light-classes/**"/>
@@ -245,6 +249,8 @@
             <fileset dir="compiler/conditional-preprocessor/src"/>
             <fileset dir="compiler/daemon/src"/>
             <fileset dir="compiler/daemon/daemon-common/src"/>
+            <fileset dir="build-common/src"/>
+            <fileset dir="compiler/incremental-compilation-impl/src"/>
             <fileset dir="compiler/container/src"/>
             <fileset dir="compiler/frontend/src"/>
             <fileset dir="compiler/resolution/src"/>
@@ -684,54 +690,6 @@
         <pack-compiler-for-maven/>
     </target>
 
-    <target name="kotlin-build-common">
-        <cleandir dir="${output}/classes/kotlin-build-common"/>
-
-        <javac2 destdir="${output}/classes/kotlin-build-common" debug="true" debuglevel="lines,vars,source" includeAntRuntime="false"
-                source="${java.target}" target="${java.target}">
-            <withKotlin modulename="kotlin-build-common">
-                <compilerarg value="-version"/>
-            </withKotlin>
-            <skip pattern="kotlin/Metadata"/>
-            <src>
-                <pathelement path="build-common/src"/>
-            </src>
-            <classpath>
-                <pathelement path="${bootstrap.runtime}"/>
-                <pathelement path="${bootstrap.reflect}"/>
-                <pathelement path="${bootstrap.script.runtime}"/>
-                <pathelement path="${idea.sdk}/lib/util.jar"/>
-                <pathelement path="${kotlin-home}/lib/kotlin-compiler.jar"/>
-            </classpath>
-        </javac2>
-
-        <jar destfile="${kotlin-home}/lib/kotlin-build-common.jar">
-            <fileset dir="${output}/classes/kotlin-build-common"/>
-            <zipfileset file="${kotlin-home}/build.txt" prefix="META-INF"/>
-
-            <manifest>
-                <attribute name="Built-By" value="${manifest.impl.vendor}"/>
-
-                <attribute name="Implementation-Vendor" value="${manifest.impl.vendor}"/>
-                <attribute name="Implementation-Title" value="${manifest.impl.title.kotlin.build.common}"/>
-                <attribute name="Implementation-Version" value="${build.number}"/>
-            </manifest>
-        </jar>
-
-        <jar jarfile="${output}/kotlin-build-common-sources.jar">
-            <fileset dir="build-common/src"/>
-            <zipfileset file="${kotlin-home}/build.txt" prefix="META-INF"/>
-
-            <manifest>
-                <attribute name="Built-By" value="${manifest.impl.vendor}"/>
-
-                <attribute name="Implementation-Vendor" value="${manifest.impl.vendor}"/>
-                <attribute name="Implementation-Title" value="${manifest.impl.title.kotlin.build.common.sources}"/>
-                <attribute name="Implementation-Version" value="${build.number}"/>
-            </manifest>
-        </jar>
-    </target>
-
     <target name="kotlin-build-common-test">
         <cleandir dir="${output}/classes/kotlin-build-common-test"/>
 
@@ -739,7 +697,10 @@
                 source="${java.target}" target="${java.target}">
             <withKotlin modulename="kotlin-build-common"/>
             <skip pattern="kotlin/Metadata"/>
-            <src path="build-common/test"/>
+            <src>
+                <pathelement path="build-common/test"/>
+                <pathelement path="compiler/incremental-compilation-impl/test"/>
+            </src>
             <classpath>
                 <pathelement path="${bootstrap.runtime}"/>
                 <pathelement path="${bootstrap.reflect}"/>
@@ -747,7 +708,6 @@
                 <pathelement path="${bootstrap.kotlin.test}"/>
                 <pathelement path="${protobuf.jar}"/>
                 <pathelement path="${idea.sdk}/lib/junit-4.12.jar"/>
-                <pathelement path="${kotlin-home}/lib/kotlin-build-common.jar"/>
                 <pathelement path="${kotlin-home}/lib/kotlin-compiler.jar"/>
             </classpath>
         </javac2>
@@ -902,7 +862,7 @@
             <fileset dir="${basedir}/plugins/sam-with-receiver/sam-with-receiver-cli/src" includes="META-INF/services/**"/>
         </jar>
     </target>
-    
+
     <target name="annotation-processing-under-jdk8">
         <property environment="env"/>
 
@@ -1313,7 +1273,7 @@
             depends="builtins,stdlib,kotlin-test,core,reflection,pack-runtime,pack-runtime-sources,script-runtime,mock-runtime-for-test"/>
 
     <target name="dist"
-            depends="clean,init,prepare-dist,preloader,runner,serialize-builtins,compiler,compiler-sources,kotlin-build-common,ant-tools,runtime,kotlin-js-stdlib,android-extensions-compiler,allopen-compiler-plugin,noarg-compiler-plugin,sam-with-receiver-compiler-plugin,annotation-processing-under-jdk8,daemon-client,kotlin-build-common-test"
+            depends="clean,init,prepare-dist,preloader,runner,serialize-builtins,compiler,compiler-sources,ant-tools,runtime,kotlin-js-stdlib,android-extensions-compiler,allopen-compiler-plugin,noarg-compiler-plugin,sam-with-receiver-compiler-plugin,annotation-processing-under-jdk8,daemon-client,kotlin-build-common-test"
             description="Builds redistributables from sources"/>
 
     <target name="dist-quick"

compiler/incremental-compilation-impl/incremental-compilation-impl.iml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="intellij-core" level="project" />
+    <orderEntry type="library" name="kotlin-runtime" level="project" />
+    <orderEntry type="module" module-name="build-common" />
+    <orderEntry type="module" module-name="cli" />
+    <orderEntry type="module" module-name="descriptor.loader.java" />
+    <orderEntry type="module" module-name="frontend" />
+    <orderEntry type="module" module-name="frontend.java" />
+    <orderEntry type="module" module-name="util" />
+  </component>
+</module>

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/annotation/AnnotationFileUpdater.kt

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.annotation
+
+import org.jetbrains.kotlin.resolve.jvm.JvmClassName
+
+/**
+ * Annotation file is generated by collecting annotated elements of generated files.
+ * When compiling incrementally, the compiler generates only subset of all classes,
+ * so after compilation annotation file contains only a subset of annotated elements,
+ * which breaks the build.
+ *
+ * The workaround is to:
+ * 1. backup old file before incremental compilation;
+ * 2. after each iteration of IC:
+ *  2.1 remove classes corresponding to dirty source files
+ *  2.2 add annotations from newly generated annotations file
+ */
+interface AnnotationFileUpdater {
+    fun updateAnnotations(outdatedClasses: Iterable<JvmClassName>)
+    fun revert()
+}
+

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedFiles.kt

@@ -14,12 +14,13 @@
  * limitations under the License.
  */
 
-package org.jetbrains.kotlin.annotation
+package org.jetbrains.kotlin.incremental
 
-import org.jetbrains.kotlin.resolve.jvm.JvmClassName
+import org.gradle.api.tasks.incremental.IncrementalTaskInputs
+import java.io.File
+import java.util.*
 
-interface AnnotationFileUpdater {
+internal sealed class ChangedFiles {
-    fun updateAnnotations(outdatedClasses: Iterable<JvmClassName>)
+    class Known(val modified: List<File>, val removed: List<File>) : ChangedFiles()
-    fun revert()
+    class Unknown : ChangedFiles()
 }
-

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/fileUtils.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import java.io.File
+
+internal fun File.isJavaFile() =
+        extension.equals("java", ignoreCase = true)
+
+internal fun File.isKotlinFile(): Boolean =
+    extension.let {
+        "kt".equals(it, ignoreCase = true) ||
+        "kts".equals(it, ignoreCase = true)
+    }
+
+internal fun File.isClassFile(): Boolean =
+        extension.equals("class", ignoreCase = true)
+
+internal fun listClassFiles(path: String): Sequence<File> =
+        File(path).walk().filter { it.isFile && it.isClassFile() }
+
+internal fun File.relativeOrCanonical(base: File): String =
+        relativeToOrNull(base)?.path ?: canonicalPath
+
+internal fun Iterable<File>.pathsAsStringRelativeTo(base: File): String =
+        map { it.relativeOrCanonical(base) }.sorted().joinToString()

libraries/pom.xml

@@ -77,7 +77,6 @@
         <module>tools/native-platform</module>
         <module>tools/kotlin-compiler-runner</module>
         <module>tools/kotlin-daemon-client</module>
-        <module>tools/kotlin-build-common</module>
         <module>tools/kotlin-build-common-test</module>
         <module>tools/kotlin-maven-plugin</module>
 

libraries/tools/kotlin-build-common/pom.xml

@@ -1,80 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project
-        xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
-        xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
-
-    <modelVersion>4.0.0</modelVersion>
-    <properties>
-        <maven-plugin-anno.version>1.4.1</maven-plugin-anno.version>
-        <maven.version>3.0.4</maven.version>
-    </properties>
-
-    <parent>
-        <groupId>org.jetbrains.kotlin</groupId>
-        <artifactId>kotlin-project</artifactId>
-        <version>1.1-SNAPSHOT</version>
-        <relativePath>../../pom.xml</relativePath>
-    </parent>
-
-    <artifactId>kotlin-build-common</artifactId>
-    <packaging>jar</packaging>
-
-    <description>the Kotlin builders</description>
-
-    <build>
-        <sourceDirectory>../../../compiler/compiler-runner/src</sourceDirectory>
-
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-source-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>attach-sources</id>
-                        <goals>
-                            <goal>jar</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-compiler-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>skip-compile</id>
-                        <phase>compile</phase>
-                        <goals>
-                            <goal>compile</goal>
-                        </goals>
-                        <configuration>
-                            <skipMain>true</skipMain>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-antrun-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>copy-jar</id>
-                        <phase>package</phase>
-                        <configuration>
-                            <tasks>
-                                <copy file="${kotlin-dist}/kotlinc/lib/kotlin-build-common.jar"
-                                      tofile="${basedir}/target/${project.artifactId}-${project.version}.jar"
-                                      overwrite="true" verbose="true"/>
-                            </tasks>
-                        </configuration>
-                        <goals>
-                            <goal>run</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-</project>

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/annotation/AnnotationFileUpdaterImpl.kt

@@ -5,18 +5,6 @@ import org.jetbrains.kotlin.gradle.plugin.kotlinDebug
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName
 import java.io.File
 
-/**
- * Annotation file is generated by collecting annotated elements of generated files.
- * When compiling incrementally, the compiler generates only subset of all classes,
- * so after compilation annotation file contains only a subset of annotated elements,
- * which breaks the build.
- *
- * The workaround is to:
- * 1. backup old file before incremental compilation;
- * 2. after each iteration of IC:
- *  2.1 remove classes corresponding to dirty source files
- *  2.2 add annotations from newly generated annotations file
- */
 internal class AnnotationFileUpdaterImpl(private val generatedAnnotationFile: File) : AnnotationFileUpdater {
     private val logger = Logging.getLogger(this.javaClass)
     private val lastSuccessfullyUpdatedFile = File.createTempFile("kapt-annotations-copy", "tmp")

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/ChangedFiles.kt

@@ -20,11 +20,6 @@ import org.gradle.api.tasks.incremental.IncrementalTaskInputs
 import java.io.File
 import java.util.*
 
-internal sealed class ChangedFiles {
-    class Known(val modified: List<File>, val removed: List<File>) : ChangedFiles()
-    class Unknown : ChangedFiles()
-}
-
 internal fun ChangedFiles(taskInputs: IncrementalTaskInputs): ChangedFiles {
     if (!taskInputs.isIncremental) return ChangedFiles.Unknown()