Baby steps for incremental compilation.

Original commit: 5acd054c72

jps/jps-plugin/kannotator-jps-plugin-test/test/org/jetbrains/jet/jps/build/kannotator/KAnnotatorJpsBuildTestCase.java

@@ -90,11 +90,6 @@ public class KAnnotatorJpsBuildTestCase extends AbstractKotlinJpsBuildTestCase {
         return result;
     }
 
-    @Override
-    protected File doGetProjectDir() throws IOException {
-        return workDir;
-    }
-
     private void initProject() {
         addJdk(JDK_NAME);
         loadProject(workDir.getAbsolutePath());

jps/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.java

@@ -16,9 +16,12 @@
 
 package org.jetbrains.jet.jps.build;
 
+import com.intellij.openapi.util.Key;
+import com.intellij.openapi.util.io.FileUtil;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.util.Function;
 import com.intellij.util.containers.ContainerUtil;
+import gnu.trove.THashSet;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.jet.cli.common.KotlinVersion;
 import org.jetbrains.jet.cli.common.arguments.CommonCompilerArguments;
@@ -46,6 +49,7 @@ import org.jetbrains.jps.model.JpsProject;
 import org.jetbrains.jps.model.module.JpsModule;
 
 import java.io.File;
+import java.io.FileFilter;
 import java.io.IOException;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
@@ -58,8 +62,9 @@ import static org.jetbrains.jet.compiler.runner.KotlinCompilerRunner.runK2JsComp
 import static org.jetbrains.jet.compiler.runner.KotlinCompilerRunner.runK2JvmCompiler;
 
 public class KotlinBuilder extends ModuleLevelBuilder {
+    private static final Key<Set<File>> ALL_COMPILED_FILES_KEY = Key.create("_all_kotlin_compiled_files_");
 
-    private static final String KOTLIN_BUILDER_NAME = "Kotlin Builder";
+    public static final String KOTLIN_BUILDER_NAME = "Kotlin Builder";
     private static final List<String> COMPILABLE_FILE_EXTENSIONS = Collections.singletonList("kt");
 
     private static final Function<JpsModule,String> MODULE_NAME = new Function<JpsModule, String>() {
@@ -122,6 +127,8 @@ public class KotlinBuilder extends ModuleLevelBuilder {
         CommonCompilerArguments commonArguments = JpsKotlinCompilerSettings.getCommonCompilerArguments(project);
         CompilerSettings compilerSettings = JpsKotlinCompilerSettings.getCompilerSettings(project);
 
+        final Set<File> allCompiledFiles = getAllCompiledFilesContainer(context);
+
         if (JpsUtils.isJsKotlinModule(representativeTarget)) {
             if (chunk.getModules().size() > 1) {
                 // We do not support circular dependencies, but if they are present, we do our best should not break the build,
@@ -134,7 +141,7 @@ public class KotlinBuilder extends ModuleLevelBuilder {
                 return ExitCode.NOTHING_DONE;
             }
 
-            List<File> sourceFiles = KotlinSourceFileCollector.getAllKotlinSourceFiles(representativeTarget);
+            Collection<File> sourceFiles = KotlinSourceFileCollector.getAllKotlinSourceFiles(representativeTarget);
             //List<File> sourceFiles = KotlinSourceFileCollector.getDirtySourceFiles(dirtyFilesHolder);
 
             if (sourceFiles.isEmpty()) {
@@ -157,7 +164,11 @@ public class KotlinBuilder extends ModuleLevelBuilder {
                         NO_LOCATION);
             }
 
-            File moduleFile = KotlinBuilderModuleScriptGenerator.generateModuleDescription(context, chunk);
+            List<File> filesToCompile = KotlinSourceFileCollector.getDirtySourceFiles(dirtyFilesHolder);
+            filesToCompile.removeAll(allCompiledFiles);
+            allCompiledFiles.addAll(filesToCompile);
+
+            File moduleFile = KotlinBuilderModuleScriptGenerator.generateModuleDescription(context, chunk, filesToCompile);
             if (moduleFile == null) {
                 // No Kotlin sources found
                 return ExitCode.NOTHING_DONE;
@@ -192,13 +203,29 @@ public class KotlinBuilder extends ModuleLevelBuilder {
             outputConsumer.registerOutputFile(target != null ? target : representativeTarget, outputItem.getOutputFile(), paths(sourceFiles));
         }
 
-        return ExitCode.OK;
+        // TODO should mark dependencies as dirty, as well
+        FSOperations.markDirty(context, chunk, new FileFilter() {
+            @Override
+            public boolean accept(@NotNull File file) {
+                return !allCompiledFiles.contains(file);
+            }
+        });
+        return ExitCode.ADDITIONAL_PASS_REQUIRED;
+    }
+
+    private static Set<File> getAllCompiledFilesContainer(CompileContext context) {
+        Set<File> allCompiledFiles = ALL_COMPILED_FILES_KEY.get(context);
+        if (allCompiledFiles == null) {
+            allCompiledFiles = new THashSet<File>(FileUtil.FILE_HASHING_STRATEGY);
+            ALL_COMPILED_FILES_KEY.set(context, allCompiledFiles);
+        }
+        return allCompiledFiles;
     }
 
     private static boolean hasKotlinFiles(@NotNull ModuleChunk chunk) {
         boolean hasKotlinFiles = false;
         for (ModuleBuildTarget target : chunk.getTargets()) {
-            List<File> sourceFiles = KotlinSourceFileCollector.getAllKotlinSourceFiles(target);
+            Collection<File> sourceFiles = KotlinSourceFileCollector.getAllKotlinSourceFiles(target);
             if (!sourceFiles.isEmpty()) {
                 hasKotlinFiles = true;
                 break;

jps/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilderModuleScriptGenerator.java

@@ -25,6 +25,7 @@ import org.jetbrains.jet.compiler.runner.KotlinModuleDescriptionBuilderFactory;
 import org.jetbrains.jet.compiler.runner.KotlinModuleXmlBuilderFactory;
 import org.jetbrains.jps.ModuleChunk;
 import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor;
+import org.jetbrains.jps.builders.logging.ProjectBuilderLogger;
 import org.jetbrains.jps.incremental.CompileContext;
 import org.jetbrains.jps.incremental.ModuleBuildTarget;
 import org.jetbrains.jps.incremental.messages.BuildMessage;
@@ -52,7 +53,11 @@ public class KotlinBuilderModuleScriptGenerator {
     public static final KotlinModuleDescriptionBuilderFactory FACTORY = KotlinModuleXmlBuilderFactory.INSTANCE;
 
     @Nullable
-    public static File generateModuleDescription(CompileContext context, ModuleChunk chunk)
+    public static File generateModuleDescription(
+            CompileContext context,
+            ModuleChunk chunk,
+            List<File> sourceFiles
+    )
             throws IOException
     {
         KotlinModuleDescriptionBuilder builder = FACTORY.create();
@@ -63,11 +68,17 @@ public class KotlinBuilderModuleScriptGenerator {
         for (ModuleBuildTarget target : chunk.getTargets()) {
             outputDirs.add(getOutputDir(target));
         }
+        ProjectBuilderLogger logger = context.getLoggingManager().getProjectBuilderLogger();
         for (ModuleBuildTarget target : chunk.getTargets()) {
             File outputDir = getOutputDir(target);
 
-            List<File> sourceFiles = KotlinSourceFileCollector.getAllKotlinSourceFiles(target);
+            if (sourceFiles.size() > 0) {
-            noSources &= sourceFiles.isEmpty();
+                noSources = false;
+
+                if (logger.isEnabled()) {
+                    logger.logCompiledFiles(sourceFiles, KotlinBuilder.KOTLIN_BUILDER_NAME, "Compiling files:");
+                }
+            }
 
             builder.addModule(
                     target.getId(),

jps/jps-plugin/test/org/jetbrains/jet/jps/build/AbstractKotlinJpsBuildTestCase.java

@@ -37,7 +37,7 @@ import java.io.IOException;
 import java.util.Collection;
 
 public abstract class AbstractKotlinJpsBuildTestCase extends JpsBuildTestCase {
-    protected static final String TEST_DATA_PATH = "jps-plugin/testData/";
+    public static final String TEST_DATA_PATH = "jps-plugin/testData/";
 
     protected File workDir;
 

jps/jps-plugin/test/org/jetbrains/jet/jps/build/IncrementalJpsTest.kt

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2010-2014 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.jet.jps.build
+
+import org.jetbrains.ether.IncrementalTestCase
+import org.jetbrains.jps.builders.JpsBuildTestCase
+import kotlin.properties.Delegates
+import com.intellij.openapi.util.io.FileUtil
+import java.io.File
+import org.jetbrains.jps.builders.CompileScopeTestBuilder
+import org.jetbrains.jps.builders.BuildResult
+import org.jetbrains.jps.builders.impl.logging.ProjectBuilderLoggerBase
+import org.jetbrains.jps.builders.logging.BuildLoggingManager
+import org.jetbrains.jps.model.java.JpsJavaExtensionService
+import org.jetbrains.jps.util.JpsPathUtil
+import com.intellij.testFramework.UsefulTestCase
+
+public class IncrementalJpsTest : JpsBuildTestCase() {
+    private val testDataDir: File
+        get() = File(AbstractKotlinJpsBuildTestCase.TEST_DATA_PATH + "incremental/" + getTestName(true))
+
+    var workDir: File by Delegates.notNull()
+
+    override fun setUp() {
+        super.setUp()
+        System.setProperty("kotlin.jps.tests", "true")
+
+        workDir = FileUtil.createTempDirectory("jps-build", null)
+
+        FileUtil.copyDir(testDataDir, File(workDir, "src"), { it.getName().endsWith(".kt") || it.getName().endsWith(".java") })
+
+        JpsJavaExtensionService.getInstance().getOrCreateProjectExtension(myProject!!)
+                .setOutputUrl(JpsPathUtil.pathToUrl(getAbsolutePath("out")))
+    }
+
+    override fun tearDown() {
+        System.clearProperty("kotlin.jps.tests")
+        super.tearDown()
+    }
+
+    fun makeAllGetLog(): String {
+        val scope = CompileScopeTestBuilder.make().all()
+        val logger = MyLogger(FileUtil.toSystemIndependentName(workDir.getAbsolutePath()))
+        val descriptor = createProjectDescriptor(BuildLoggingManager(logger))
+        try {
+            doBuild(descriptor, scope)!!.assertSuccessful()
+            return logger.log
+        } finally {
+            descriptor.release()
+        }
+    }
+
+    private fun doTest() {
+        addModule("module", array<String>(getAbsolutePath("src")), null, null, addJdk("my jdk"))
+
+        makeAllGetLog()
+
+        FileUtil.processFilesRecursively(testDataDir, {
+            if (it!!.getName().endsWith(".new")) {
+                it.copyTo(File(workDir, "src/" + it.getName().trimTrailing(".new")))
+            }
+
+            true
+        })
+
+        val log = makeAllGetLog()
+        UsefulTestCase.assertSameLinesWithFile(File(testDataDir, "build.log").getAbsolutePath(), log)
+    }
+
+    override fun doGetProjectDir(): File? = workDir
+
+    fun testIndependentClasses() {
+        doTest()
+    }
+
+    fun testSimpleClassDependency() {
+        doTest()
+    }
+
+    private class MyLogger(val rootPath: String) : ProjectBuilderLoggerBase() {
+        private val logBuf = StringBuilder()
+        public val log: String
+            get() = logBuf.toString()
+
+        override fun isEnabled(): Boolean = true
+
+        override fun logLine(message: String?) {
+
+            fun String.replaceHashWithStar(): String {
+                val lastHyphen = this.lastIndexOf('-')
+                if (lastHyphen != -1 && substring(lastHyphen + 1).matches("[0-9a-f]{8}\\.class")) {
+                    return substring(0, lastHyphen) + "-*.class"
+                }
+                return this
+            }
+
+            logBuf.append(message!!.trimLeading(rootPath + "/").replaceHashWithStar()).append('\n')
+        }
+    }
+}

jps/jps-plugin/testData/incremental/independentClasses/Bar.kt

@@ -0,0 +1,2 @@
+package test
+class Bar

jps/jps-plugin/testData/incremental/independentClasses/Foo.kt

@@ -0,0 +1,2 @@
+package test
+class Foo

jps/jps-plugin/testData/incremental/independentClasses/Foo.kt.new

@@ -0,0 +1,2 @@
+package test
+class Foo

jps/jps-plugin/testData/incremental/independentClasses/build.log

@@ -0,0 +1,12 @@
+Cleaning output files:
+out/production/module/test/Foo.class
+End of files
+Compiling files:
+src/Foo.kt
+End of files
+Cleaning output files:
+out/production/module/test/Bar.class
+End of files
+Compiling files:
+src/Bar.kt
+End of files

jps/jps-plugin/testData/incremental/simpleClassDependency/Bar.kt

@@ -0,0 +1,2 @@
+package test
+class Bar

jps/jps-plugin/testData/incremental/simpleClassDependency/Foo.kt

@@ -0,0 +1,8 @@
+package test
+class Foo {
+    fun f() {
+        Bar()
+    }
+
+    class Boo
+}

jps/jps-plugin/testData/incremental/simpleClassDependency/Foo.kt.new

@@ -0,0 +1,8 @@
+package test
+class Foo {
+    fun f() {
+        Bar()
+    }
+
+    class Boo
+}

jps/jps-plugin/testData/incremental/simpleClassDependency/build.log

@@ -0,0 +1,13 @@
+Cleaning output files:
+out/production/module/test/Foo$Boo.class
+out/production/module/test/Foo.class
+End of files
+Compiling files:
+src/Foo.kt
+End of files
+Cleaning output files:
+out/production/module/test/Bar.class
+End of files
+Compiling files:
+src/Bar.kt
+End of files