From 259d03198d59fdf3290b12908f75e040cd765582 Mon Sep 17 00:00:00 2001 From: Evgeny Gerashchenko Date: Wed, 22 Jan 2014 23:38:57 +0400 Subject: [PATCH] Baby steps for incremental compilation. Original commit: 5acd054c72bae50bc61fce045fa867e78cf68994 --- .../KAnnotatorJpsBuildTestCase.java | 5 - .../jet/jps/build/KotlinBuilder.java | 37 +++++- .../KotlinBuilderModuleScriptGenerator.java | 17 ++- .../build/AbstractKotlinJpsBuildTestCase.java | 2 +- .../jet/jps/build/IncrementalJpsTest.kt | 114 ++++++++++++++++++ .../incremental/independentClasses/Bar.kt | 2 + .../incremental/independentClasses/Foo.kt | 2 + .../incremental/independentClasses/Foo.kt.new | 2 + .../incremental/independentClasses/build.log | 12 ++ .../incremental/simpleClassDependency/Bar.kt | 2 + .../incremental/simpleClassDependency/Foo.kt | 8 ++ .../simpleClassDependency/Foo.kt.new | 8 ++ .../simpleClassDependency/build.log | 13 ++ 13 files changed, 210 insertions(+), 14 deletions(-) create mode 100644 jps/jps-plugin/test/org/jetbrains/jet/jps/build/IncrementalJpsTest.kt create mode 100644 jps/jps-plugin/testData/incremental/independentClasses/Bar.kt create mode 100644 jps/jps-plugin/testData/incremental/independentClasses/Foo.kt create mode 100644 jps/jps-plugin/testData/incremental/independentClasses/Foo.kt.new create mode 100644 jps/jps-plugin/testData/incremental/independentClasses/build.log create mode 100644 jps/jps-plugin/testData/incremental/simpleClassDependency/Bar.kt create mode 100644 jps/jps-plugin/testData/incremental/simpleClassDependency/Foo.kt create mode 100644 jps/jps-plugin/testData/incremental/simpleClassDependency/Foo.kt.new create mode 100644 jps/jps-plugin/testData/incremental/simpleClassDependency/build.log diff --git a/jps/jps-plugin/kannotator-jps-plugin-test/test/org/jetbrains/jet/jps/build/kannotator/KAnnotatorJpsBuildTestCase.java b/jps/jps-plugin/kannotator-jps-plugin-test/test/org/jetbrains/jet/jps/build/kannotator/KAnnotatorJpsBuildTestCase.java index 467d6976a59..d6c95360413 100644 --- a/jps/jps-plugin/kannotator-jps-plugin-test/test/org/jetbrains/jet/jps/build/kannotator/KAnnotatorJpsBuildTestCase.java +++ b/jps/jps-plugin/kannotator-jps-plugin-test/test/org/jetbrains/jet/jps/build/kannotator/KAnnotatorJpsBuildTestCase.java @@ -90,11 +90,6 @@ public class KAnnotatorJpsBuildTestCase extends AbstractKotlinJpsBuildTestCase { return result; } - @Override - protected File doGetProjectDir() throws IOException { - return workDir; - } - private void initProject() { addJdk(JDK_NAME); loadProject(workDir.getAbsolutePath()); diff --git a/jps/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.java b/jps/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.java index 3b21e8d228d..dfa5db0a8ae 100644 --- a/jps/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.java +++ b/jps/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilder.java @@ -16,9 +16,12 @@ package org.jetbrains.jet.jps.build; +import com.intellij.openapi.util.Key; +import com.intellij.openapi.util.io.FileUtil; import com.intellij.openapi.util.text.StringUtil; import com.intellij.util.Function; import com.intellij.util.containers.ContainerUtil; +import gnu.trove.THashSet; import org.jetbrains.annotations.NotNull; import org.jetbrains.jet.cli.common.KotlinVersion; import org.jetbrains.jet.cli.common.arguments.CommonCompilerArguments; @@ -46,6 +49,7 @@ import org.jetbrains.jps.model.JpsProject; import org.jetbrains.jps.model.module.JpsModule; import java.io.File; +import java.io.FileFilter; import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Modifier; @@ -58,8 +62,9 @@ import static org.jetbrains.jet.compiler.runner.KotlinCompilerRunner.runK2JsComp import static org.jetbrains.jet.compiler.runner.KotlinCompilerRunner.runK2JvmCompiler; public class KotlinBuilder extends ModuleLevelBuilder { + private static final Key> ALL_COMPILED_FILES_KEY = Key.create("_all_kotlin_compiled_files_"); - private static final String KOTLIN_BUILDER_NAME = "Kotlin Builder"; + public static final String KOTLIN_BUILDER_NAME = "Kotlin Builder"; private static final List COMPILABLE_FILE_EXTENSIONS = Collections.singletonList("kt"); private static final Function MODULE_NAME = new Function() { @@ -122,6 +127,8 @@ public class KotlinBuilder extends ModuleLevelBuilder { CommonCompilerArguments commonArguments = JpsKotlinCompilerSettings.getCommonCompilerArguments(project); CompilerSettings compilerSettings = JpsKotlinCompilerSettings.getCompilerSettings(project); + final Set allCompiledFiles = getAllCompiledFilesContainer(context); + if (JpsUtils.isJsKotlinModule(representativeTarget)) { if (chunk.getModules().size() > 1) { // We do not support circular dependencies, but if they are present, we do our best should not break the build, @@ -134,7 +141,7 @@ public class KotlinBuilder extends ModuleLevelBuilder { return ExitCode.NOTHING_DONE; } - List sourceFiles = KotlinSourceFileCollector.getAllKotlinSourceFiles(representativeTarget); + Collection sourceFiles = KotlinSourceFileCollector.getAllKotlinSourceFiles(representativeTarget); //List sourceFiles = KotlinSourceFileCollector.getDirtySourceFiles(dirtyFilesHolder); if (sourceFiles.isEmpty()) { @@ -157,7 +164,11 @@ public class KotlinBuilder extends ModuleLevelBuilder { NO_LOCATION); } - File moduleFile = KotlinBuilderModuleScriptGenerator.generateModuleDescription(context, chunk); + List filesToCompile = KotlinSourceFileCollector.getDirtySourceFiles(dirtyFilesHolder); + filesToCompile.removeAll(allCompiledFiles); + allCompiledFiles.addAll(filesToCompile); + + File moduleFile = KotlinBuilderModuleScriptGenerator.generateModuleDescription(context, chunk, filesToCompile); if (moduleFile == null) { // No Kotlin sources found return ExitCode.NOTHING_DONE; @@ -192,13 +203,29 @@ public class KotlinBuilder extends ModuleLevelBuilder { outputConsumer.registerOutputFile(target != null ? target : representativeTarget, outputItem.getOutputFile(), paths(sourceFiles)); } - return ExitCode.OK; + // TODO should mark dependencies as dirty, as well + FSOperations.markDirty(context, chunk, new FileFilter() { + @Override + public boolean accept(@NotNull File file) { + return !allCompiledFiles.contains(file); + } + }); + return ExitCode.ADDITIONAL_PASS_REQUIRED; + } + + private static Set getAllCompiledFilesContainer(CompileContext context) { + Set allCompiledFiles = ALL_COMPILED_FILES_KEY.get(context); + if (allCompiledFiles == null) { + allCompiledFiles = new THashSet(FileUtil.FILE_HASHING_STRATEGY); + ALL_COMPILED_FILES_KEY.set(context, allCompiledFiles); + } + return allCompiledFiles; } private static boolean hasKotlinFiles(@NotNull ModuleChunk chunk) { boolean hasKotlinFiles = false; for (ModuleBuildTarget target : chunk.getTargets()) { - List sourceFiles = KotlinSourceFileCollector.getAllKotlinSourceFiles(target); + Collection sourceFiles = KotlinSourceFileCollector.getAllKotlinSourceFiles(target); if (!sourceFiles.isEmpty()) { hasKotlinFiles = true; break; diff --git a/jps/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilderModuleScriptGenerator.java b/jps/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilderModuleScriptGenerator.java index f105f36047c..b77ce9de3d5 100644 --- a/jps/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilderModuleScriptGenerator.java +++ b/jps/jps-plugin/src/org/jetbrains/jet/jps/build/KotlinBuilderModuleScriptGenerator.java @@ -25,6 +25,7 @@ import org.jetbrains.jet.compiler.runner.KotlinModuleDescriptionBuilderFactory; import org.jetbrains.jet.compiler.runner.KotlinModuleXmlBuilderFactory; import org.jetbrains.jps.ModuleChunk; import org.jetbrains.jps.builders.java.JavaSourceRootDescriptor; +import org.jetbrains.jps.builders.logging.ProjectBuilderLogger; import org.jetbrains.jps.incremental.CompileContext; import org.jetbrains.jps.incremental.ModuleBuildTarget; import org.jetbrains.jps.incremental.messages.BuildMessage; @@ -52,7 +53,11 @@ public class KotlinBuilderModuleScriptGenerator { public static final KotlinModuleDescriptionBuilderFactory FACTORY = KotlinModuleXmlBuilderFactory.INSTANCE; @Nullable - public static File generateModuleDescription(CompileContext context, ModuleChunk chunk) + public static File generateModuleDescription( + CompileContext context, + ModuleChunk chunk, + List sourceFiles + ) throws IOException { KotlinModuleDescriptionBuilder builder = FACTORY.create(); @@ -63,11 +68,17 @@ public class KotlinBuilderModuleScriptGenerator { for (ModuleBuildTarget target : chunk.getTargets()) { outputDirs.add(getOutputDir(target)); } + ProjectBuilderLogger logger = context.getLoggingManager().getProjectBuilderLogger(); for (ModuleBuildTarget target : chunk.getTargets()) { File outputDir = getOutputDir(target); - List sourceFiles = KotlinSourceFileCollector.getAllKotlinSourceFiles(target); - noSources &= sourceFiles.isEmpty(); + if (sourceFiles.size() > 0) { + noSources = false; + + if (logger.isEnabled()) { + logger.logCompiledFiles(sourceFiles, KotlinBuilder.KOTLIN_BUILDER_NAME, "Compiling files:"); + } + } builder.addModule( target.getId(), diff --git a/jps/jps-plugin/test/org/jetbrains/jet/jps/build/AbstractKotlinJpsBuildTestCase.java b/jps/jps-plugin/test/org/jetbrains/jet/jps/build/AbstractKotlinJpsBuildTestCase.java index 7dd4d207f22..eb7ddb5a7f6 100644 --- a/jps/jps-plugin/test/org/jetbrains/jet/jps/build/AbstractKotlinJpsBuildTestCase.java +++ b/jps/jps-plugin/test/org/jetbrains/jet/jps/build/AbstractKotlinJpsBuildTestCase.java @@ -37,7 +37,7 @@ import java.io.IOException; import java.util.Collection; public abstract class AbstractKotlinJpsBuildTestCase extends JpsBuildTestCase { - protected static final String TEST_DATA_PATH = "jps-plugin/testData/"; + public static final String TEST_DATA_PATH = "jps-plugin/testData/"; protected File workDir; diff --git a/jps/jps-plugin/test/org/jetbrains/jet/jps/build/IncrementalJpsTest.kt b/jps/jps-plugin/test/org/jetbrains/jet/jps/build/IncrementalJpsTest.kt new file mode 100644 index 00000000000..aca6d23a63d --- /dev/null +++ b/jps/jps-plugin/test/org/jetbrains/jet/jps/build/IncrementalJpsTest.kt @@ -0,0 +1,114 @@ +/* + * Copyright 2010-2014 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.jet.jps.build + +import org.jetbrains.ether.IncrementalTestCase +import org.jetbrains.jps.builders.JpsBuildTestCase +import kotlin.properties.Delegates +import com.intellij.openapi.util.io.FileUtil +import java.io.File +import org.jetbrains.jps.builders.CompileScopeTestBuilder +import org.jetbrains.jps.builders.BuildResult +import org.jetbrains.jps.builders.impl.logging.ProjectBuilderLoggerBase +import org.jetbrains.jps.builders.logging.BuildLoggingManager +import org.jetbrains.jps.model.java.JpsJavaExtensionService +import org.jetbrains.jps.util.JpsPathUtil +import com.intellij.testFramework.UsefulTestCase + +public class IncrementalJpsTest : JpsBuildTestCase() { + private val testDataDir: File + get() = File(AbstractKotlinJpsBuildTestCase.TEST_DATA_PATH + "incremental/" + getTestName(true)) + + var workDir: File by Delegates.notNull() + + override fun setUp() { + super.setUp() + System.setProperty("kotlin.jps.tests", "true") + + workDir = FileUtil.createTempDirectory("jps-build", null) + + FileUtil.copyDir(testDataDir, File(workDir, "src"), { it.getName().endsWith(".kt") || it.getName().endsWith(".java") }) + + JpsJavaExtensionService.getInstance().getOrCreateProjectExtension(myProject!!) + .setOutputUrl(JpsPathUtil.pathToUrl(getAbsolutePath("out"))) + } + + override fun tearDown() { + System.clearProperty("kotlin.jps.tests") + super.tearDown() + } + + fun makeAllGetLog(): String { + val scope = CompileScopeTestBuilder.make().all() + val logger = MyLogger(FileUtil.toSystemIndependentName(workDir.getAbsolutePath())) + val descriptor = createProjectDescriptor(BuildLoggingManager(logger)) + try { + doBuild(descriptor, scope)!!.assertSuccessful() + return logger.log + } finally { + descriptor.release() + } + } + + private fun doTest() { + addModule("module", array(getAbsolutePath("src")), null, null, addJdk("my jdk")) + + makeAllGetLog() + + FileUtil.processFilesRecursively(testDataDir, { + if (it!!.getName().endsWith(".new")) { + it.copyTo(File(workDir, "src/" + it.getName().trimTrailing(".new"))) + } + + true + }) + + val log = makeAllGetLog() + UsefulTestCase.assertSameLinesWithFile(File(testDataDir, "build.log").getAbsolutePath(), log) + } + + override fun doGetProjectDir(): File? = workDir + + fun testIndependentClasses() { + doTest() + } + + fun testSimpleClassDependency() { + doTest() + } + + private class MyLogger(val rootPath: String) : ProjectBuilderLoggerBase() { + private val logBuf = StringBuilder() + public val log: String + get() = logBuf.toString() + + override fun isEnabled(): Boolean = true + + override fun logLine(message: String?) { + + fun String.replaceHashWithStar(): String { + val lastHyphen = this.lastIndexOf('-') + if (lastHyphen != -1 && substring(lastHyphen + 1).matches("[0-9a-f]{8}\\.class")) { + return substring(0, lastHyphen) + "-*.class" + } + return this + } + + logBuf.append(message!!.trimLeading(rootPath + "/").replaceHashWithStar()).append('\n') + } + } +} diff --git a/jps/jps-plugin/testData/incremental/independentClasses/Bar.kt b/jps/jps-plugin/testData/incremental/independentClasses/Bar.kt new file mode 100644 index 00000000000..d10dda29c09 --- /dev/null +++ b/jps/jps-plugin/testData/incremental/independentClasses/Bar.kt @@ -0,0 +1,2 @@ +package test +class Bar \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/independentClasses/Foo.kt b/jps/jps-plugin/testData/incremental/independentClasses/Foo.kt new file mode 100644 index 00000000000..de4f811f0b1 --- /dev/null +++ b/jps/jps-plugin/testData/incremental/independentClasses/Foo.kt @@ -0,0 +1,2 @@ +package test +class Foo \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/independentClasses/Foo.kt.new b/jps/jps-plugin/testData/incremental/independentClasses/Foo.kt.new new file mode 100644 index 00000000000..de4f811f0b1 --- /dev/null +++ b/jps/jps-plugin/testData/incremental/independentClasses/Foo.kt.new @@ -0,0 +1,2 @@ +package test +class Foo \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/independentClasses/build.log b/jps/jps-plugin/testData/incremental/independentClasses/build.log new file mode 100644 index 00000000000..0d52aa1b838 --- /dev/null +++ b/jps/jps-plugin/testData/incremental/independentClasses/build.log @@ -0,0 +1,12 @@ +Cleaning output files: +out/production/module/test/Foo.class +End of files +Compiling files: +src/Foo.kt +End of files +Cleaning output files: +out/production/module/test/Bar.class +End of files +Compiling files: +src/Bar.kt +End of files diff --git a/jps/jps-plugin/testData/incremental/simpleClassDependency/Bar.kt b/jps/jps-plugin/testData/incremental/simpleClassDependency/Bar.kt new file mode 100644 index 00000000000..862629bf67c --- /dev/null +++ b/jps/jps-plugin/testData/incremental/simpleClassDependency/Bar.kt @@ -0,0 +1,2 @@ +package test +class Bar diff --git a/jps/jps-plugin/testData/incremental/simpleClassDependency/Foo.kt b/jps/jps-plugin/testData/incremental/simpleClassDependency/Foo.kt new file mode 100644 index 00000000000..d40f04709d3 --- /dev/null +++ b/jps/jps-plugin/testData/incremental/simpleClassDependency/Foo.kt @@ -0,0 +1,8 @@ +package test +class Foo { + fun f() { + Bar() + } + + class Boo +} diff --git a/jps/jps-plugin/testData/incremental/simpleClassDependency/Foo.kt.new b/jps/jps-plugin/testData/incremental/simpleClassDependency/Foo.kt.new new file mode 100644 index 00000000000..a9fb2ea18ed --- /dev/null +++ b/jps/jps-plugin/testData/incremental/simpleClassDependency/Foo.kt.new @@ -0,0 +1,8 @@ +package test +class Foo { + fun f() { + Bar() + } + + class Boo +} \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/simpleClassDependency/build.log b/jps/jps-plugin/testData/incremental/simpleClassDependency/build.log new file mode 100644 index 00000000000..6c95f57482f --- /dev/null +++ b/jps/jps-plugin/testData/incremental/simpleClassDependency/build.log @@ -0,0 +1,13 @@ +Cleaning output files: +out/production/module/test/Foo$Boo.class +out/production/module/test/Foo.class +End of files +Compiling files: +src/Foo.kt +End of files +Cleaning output files: +out/production/module/test/Bar.class +End of files +Compiling files: +src/Bar.kt +End of files \ No newline at end of file