Provide incremental compilation for Maven

#KT-11916 fixed

To use the IC either:
1. set the `kotlin.compiler.incremental` property to `true` in a pom.xml:
```
<properties>
    <kotlin.compiler.incremental>true</kotlin.compiler.incremental>
</properties>
```
2. pass the `kotlin.compiler.incremental` property in a command line:
```
mvn install -Dkotlin.compiler.incremental=true
```

When IC is on Kotlin plugin is expected to print the warning in the log:
```
Using experimental Kotlin incremental compilation
```

After each call an incremental compiler will also log how many files it has compiled:
```
Compiled %SOME_NUMBER% Kotlin files using incremental compiler
```

Note that the first build will be non-incremental.

For more diagnostic information (such as an exact list of compiled files) use the `kotlin.compiler.incremental.log.level` system property:
```
mvn install -Dkotlin.compiler.incremental=true -Dkotlin.compiler.incremental.log.level=info
```

To force the rebuild just run the 'clean' goal:
```
mvn clean install
```
This commit is contained in:
Alexey Tsvetkov
2017-03-30 16:25:31 +03:00
parent 21da11fe18
commit 30d6af7aae
14 changed files with 665 additions and 21 deletions
@@ -12,8 +12,45 @@
<artifactId>kotlin-maven-plugin-test</artifactId>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-maven-plugin</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<testSourceDirectory>${project.basedir}/src/test/java</testSourceDirectory>
<resources>
<resource>
<directory>${project.basedir}/src/test/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<executions>
<execution>
<id>test-compile</id>
<phase>test-compile</phase>
<goals><goal>testCompile</goal></goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
@@ -35,6 +72,33 @@
</activation>
<build>
<plugins>
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.6</version>
<executions>
<execution>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
<configuration>
<jvm>${env.JAVA_HOME}/bin/java</jvm>
<forkMode>once</forkMode>
<systemProperties>
<property>
<name>maven.home</name>
<value>${maven.home}</value>
</property>
<property>
<name>kotlin.version</name>
<value>${project.version}</value>
</property>
</systemProperties>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-invoker-plugin</artifactId>
<version>1.9</version>
@@ -0,0 +1,5 @@
package org.jetbrains.kotlin.maven;
interface Action<T> {
void run(T param) throws Exception;
}
@@ -0,0 +1,61 @@
package org.jetbrains.kotlin.maven;
import org.junit.Test;
import java.io.File;
public class IncrementalCompilationIT {
@Test
public void testSimpleCompile() throws Exception {
MavenProject project = new MavenProject("kotlinSimple");
project.exec("package")
.succeeded()
.compiledKotlin("src/A.kt", "src/useA.kt", "src/Dummy.kt");
}
@Test
public void testNoChanges() throws Exception {
MavenProject project = new MavenProject("kotlinSimple");
project.exec("package");
project.exec("package")
.succeeded()
.compiledKotlin();
}
@Test
public void testCompileError() throws Exception {
MavenProject project = new MavenProject("kotlinSimple");
project.exec("package");
File aKt = project.file("src/A.kt");
String original = "class A";
String replacement = "private class A";
MavenTestUtils.replaceFirstInFile(aKt, original, replacement);
project.exec("package")
.failed()
.contains("Cannot access 'A': it is private in file");
MavenTestUtils.replaceFirstInFile(aKt, replacement, original);
project.exec("package")
.succeeded()
.compiledKotlin("src/A.kt", "src/useA.kt");
}
@Test
public void testFunctionVisibilityChanged() throws Exception {
MavenProject project = new MavenProject("kotlinSimple");
project.exec("package");
File aKt = project.file("src/A.kt");
MavenTestUtils.replaceFirstInFile(aKt, "fun foo", "internal fun foo");
project.exec("package")
.succeeded()
.compiledKotlin("src/A.kt", "src/useA.kt");
// todo rebuild and compare output
}
}
@@ -0,0 +1,113 @@
package org.jetbrains.kotlin.maven;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import org.jetbrains.annotations.NotNull;
import org.junit.Assert;
import java.io.File;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
class MavenExecutionResult {
@NotNull
private final String stdout;
@NotNull
private final File workingDir;
private int exitCode;
MavenExecutionResult(
@NotNull String output,
@NotNull File workingDir,
int exitCode
) {
this.stdout = output;
this.workingDir = workingDir;
this.exitCode = exitCode;
}
MavenExecutionResult check(@NotNull Action<MavenExecutionResult> fn) throws Exception {
try {
fn.run(this);
}
catch (Throwable t) {
System.out.println(stdout);
throw new RuntimeException(t);
}
return this;
}
MavenExecutionResult succeeded() throws Exception {
return check(new Action<MavenExecutionResult>() {
@Override
public void run(MavenExecutionResult execResult) {
Assert.assertEquals("Maven process was expected to succeeded", 0, exitCode);
}
});
}
MavenExecutionResult failed() throws Exception {
return check(new Action<MavenExecutionResult>() {
@Override
public void run(MavenExecutionResult execResult) {
Assert.assertNotEquals("Maven process was expected to fail", 0, exitCode);
}
});
}
MavenExecutionResult contains(@NotNull final String str) throws Exception {
return check(new Action<MavenExecutionResult>() {
@Override
public void run(MavenExecutionResult execResult) {
if (!stdout.contains(str)) {
throw new AssertionError("Maven output should contain '" + str + "'");
}
}
});
}
MavenExecutionResult notContains(@NotNull final String str) throws Exception {
return check(new Action<MavenExecutionResult>() {
@Override
public void run(MavenExecutionResult execResult) {
if (stdout.contains(str)) {
throw new AssertionError("Maven output should not contain '" + str + "'");
}
}
});
}
MavenExecutionResult compiledKotlin(@NotNull final String... expectedPaths) throws Exception {
return check(new Action<MavenExecutionResult>() {
@Override
public void run(MavenExecutionResult execResult) {
Pattern kotlinCompileIteration = Pattern.compile("(?m)Kotlin compile iteration: (.*)$");
Matcher m = kotlinCompileIteration.matcher(stdout);
Set<String> normalizedActualPaths = new HashSet<String>();
while (m.find()) {
String[] compiledFiles = m.group(1).split(",");
for (String path : compiledFiles) {
File file = new File(path.trim());
String relativePath = FileUtil.getRelativePath(workingDir, file);
normalizedActualPaths.add(FileUtil.normalize(relativePath));
}
}
String[] actualPaths = normalizedActualPaths.toArray(new String[normalizedActualPaths.size()]);
Arrays.sort(actualPaths);
for (int i = 0; i < expectedPaths.length; i++) {
expectedPaths[i] = FileUtil.normalize(expectedPaths[i]);
}
Arrays.sort(expectedPaths);
String expected = StringUtil.join(expectedPaths, "\n");
String actual = StringUtil.join(actualPaths, "\n");
Assert.assertEquals("Compiled files differ", expected, actual);
}
});
}
}
@@ -0,0 +1,77 @@
package org.jetbrains.kotlin.maven;
import com.intellij.openapi.util.io.FileUtil;
import kotlin.io.TextStreamsKt;
import org.jetbrains.annotations.NotNull;
import java.io.*;
import java.util.*;
import static org.jetbrains.kotlin.maven.MavenTestUtils.getNotNullSystemProperty;
class MavenProject {
@NotNull
private final File workingDir;
MavenProject(@NotNull String name) throws IOException {
File originalProjectDir = new File("src/test/resources/" + name);
workingDir = FileUtil.createTempDirectory("maven-test-" + name, null);
File[] filesToCopy = originalProjectDir.listFiles();
for (File from : filesToCopy) {
File to = new File(workingDir, from.getName());
FileUtil.copyFileOrDir(from, to);
}
}
@NotNull
File file(@NotNull String path) {
return new File(workingDir, path);
}
MavenExecutionResult exec(String... targets) throws Exception {
List<String> cmd = buildCmd(targets);
ProcessBuilder processBuilder = new ProcessBuilder(cmd);
setUpEnvVars(processBuilder.environment());
processBuilder.directory(workingDir);
processBuilder.redirectErrorStream(true);
Process process = processBuilder.start();
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String stdout = TextStreamsKt.readText(reader);
int exitCode = process.waitFor();
return new MavenExecutionResult(stdout, workingDir, exitCode);
}
private void setUpEnvVars(Map<String, String> env) throws IOException {
String mavenHome = getNotNullSystemProperty("maven.home");
env.put("M2_HOME", mavenHome);
String mavenPath = mavenHome + File.separator + "bin";
env.put("PATH", env.get("PATH") + File.pathSeparator + mavenPath);
}
private List<String> buildCmd(String... args) {
List<String> cmd = new ArrayList<String>();
String osName = getNotNullSystemProperty("os.name");
if (osName.contains("Windows")) {
cmd.addAll(Arrays.asList("cmd", "/C"));
}
else {
cmd.add("/bin/bash");
}
cmd.add("mvn");
cmd.add("-Dkotlin.compiler.incremental.log.level=info");
String kotlinVersion = getNotNullSystemProperty("kotlin.version");
cmd.add("-Dkotlin.version=" + kotlinVersion);
cmd.addAll(Arrays.asList(args));
return cmd;
}
}
@@ -0,0 +1,33 @@
package org.jetbrains.kotlin.maven;
import kotlin.io.FilesKt;
import kotlin.text.Charsets;
import org.jetbrains.annotations.NotNull;
import java.io.*;
class MavenTestUtils {
@NotNull
static String readText(@NotNull File file) throws IOException {
return FilesKt.readText(file, Charsets.UTF_8);
}
static void writeText(@NotNull File file, @NotNull String text) throws IOException {
FilesKt.writeText(file, text, Charsets.UTF_8);
}
static void replaceFirstInFile(@NotNull File file, @NotNull String regex, @NotNull String replacement) throws IOException {
String text = readText(file);
String processedText = text.replaceFirst(regex, replacement);
writeText(file, processedText);
}
@NotNull
static String getNotNullSystemProperty(@NotNull String propertyName) {
String value = System.getProperty(propertyName);
if (value == null) {
throw new IllegalStateException("A system property '" + propertyName + "' is not set");
}
return value;
}
}
@@ -0,0 +1,69 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>test-kotlin-incremental</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<kotlin.compiler.incremental>true</kotlin.compiler.incremental>
</properties>
<dependencies>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
<version>${kotlin.version}</version>
</dependency>
</dependencies>
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.6</source>
<target>1.6</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>2.1.2</version>
</plugin>
</plugins>
</pluginManagement>
<sourceDirectory>${project.basedir}/src</sourceDirectory>
<plugins>
<plugin>
<artifactId>kotlin-maven-plugin</artifactId>
<groupId>org.jetbrains.kotlin</groupId>
<version>${kotlin.version}</version>
<executions>
<execution>
<id>compile</id>
<phase>process-sources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
<execution>
<id>test-compile</id>
<phase>process-test-sources</phase>
<goals>
<goal>test-compile</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
@@ -0,0 +1,3 @@
class A {
fun foo(s: String) = s + s
}
@@ -0,0 +1,3 @@
fun useA() {
A().foo("Hello, world")
}
@@ -20,13 +20,17 @@ import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.*;
import org.apache.maven.project.MavenProject;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.kotlin.cli.common.CLICompiler;
import org.jetbrains.kotlin.cli.common.ExitCode;
import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments;
import org.jetbrains.kotlin.cli.common.messages.MessageCollector;
import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler;
import org.jetbrains.kotlin.incremental.IncrementalJvmCompilerRunnerKt;
import org.jetbrains.kotlin.maven.incremental.MavenICReporter;
import org.jetbrains.kotlin.maven.kapt.AnnotationProcessingManager;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.*;
import static com.intellij.openapi.util.text.StringUtil.join;
import static org.jetbrains.kotlin.maven.Util.filterClassPath;
@@ -65,6 +69,26 @@ public class K2JVMCompileMojo extends KotlinCompileMojoBase<K2JVMCompilerArgumen
@Parameter(property = "kotlin.compiler.scriptTemplates", required = false, readonly = false)
protected List<String> scriptTemplates;
@Parameter(property = "kotlin.compiler.incremental", defaultValue = "false", required = false, readonly = false)
private boolean myIncremental;
@Parameter(property = "kotlin.compiler.incremental.cache.root", defaultValue = "${project.build.directory}/kotlin-ic", required = false, readonly = false)
public String incrementalCachesRoot;
@NotNull
private File getCachesDir() {
return new File(incrementalCachesRoot, getSourceSetName());
}
protected boolean isIncremental() {
return myIncremental;
}
private boolean isIncrementalSystemProperty() {
String value = System.getProperty("kotlin.incremental");
return value != null && value.equals("true");
}
@Override
protected List<String> getRelatedSourceRoots(MavenProject project) {
return project.getCompileSourceRoots();
@@ -141,4 +165,50 @@ public class K2JVMCompileMojo extends KotlinCompileMojoBase<K2JVMCompilerArgumen
arguments.scriptTemplates = scriptTemplates.toArray(new String[0]);
}
}
@Override
protected ExitCode execCompiler(
CLICompiler<K2JVMCompilerArguments> compiler,
MessageCollector messageCollector,
K2JVMCompilerArguments arguments,
List<File> sourceRoots
) throws MojoExecutionException {
if (isIncremental()) {
return runIncrementalCompiler(messageCollector, arguments, sourceRoots);
}
return super.execCompiler(compiler, messageCollector, arguments, sourceRoots);
}
@NotNull
private ExitCode runIncrementalCompiler(
MessageCollector messageCollector,
K2JVMCompilerArguments arguments,
List<File> sourceRoots
) throws MojoExecutionException {
getLog().warn("Using experimental Kotlin incremental compilation");
File cachesDir = getCachesDir();
//noinspection ResultOfMethodCallIgnored
cachesDir.mkdirs();
MavenICReporter icReporter = MavenICReporter.get(getLog());
try {
IncrementalJvmCompilerRunnerKt.makeIncrementally(cachesDir, sourceRoots, arguments, messageCollector, icReporter);
int compiledKtFilesCount = icReporter.getCompiledKotlinFiles().size();
getLog().info("Compiled " + icReporter.getCompiledKotlinFiles().size() + " Kotlin files using incremental compiler");
}
catch (Throwable t) {
t.printStackTrace();
return ExitCode.INTERNAL_ERROR;
}
if (messageCollector.hasErrors()) {
return ExitCode.COMPILATION_ERROR;
}
else {
return ExitCode.OK;
}
}
}
@@ -38,6 +38,7 @@ import org.jetbrains.annotations.Nullable;
import org.jetbrains.kotlin.cli.common.CLICompiler;
import org.jetbrains.kotlin.cli.common.ExitCode;
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments;
import org.jetbrains.kotlin.cli.common.messages.MessageCollector;
import org.jetbrains.kotlin.config.KotlinCompilerVersion;
import org.jetbrains.kotlin.config.Services;
@@ -192,18 +193,33 @@ public abstract class KotlinCompileMojoBase<A extends CommonCompilerArguments> e
A arguments = createCompilerArguments();
CLICompiler<A> compiler = createCompiler();
List<File> sourceRoots = getSourceRoots();
configureCompilerArguments(arguments, compiler);
printCompilerArgumentsIfDebugEnabled(arguments, compiler);
MavenPluginLogMessageCollector messageCollector = new MavenPluginLogMessageCollector(getLog());
ExitCode exitCode = compiler.exec(messageCollector, Services.EMPTY, arguments);
ExitCode exitCode = execCompiler(compiler, messageCollector, arguments, sourceRoots);
if (exitCode != ExitCode.OK) {
messageCollector.throwKotlinCompilerException();
}
}
@NotNull
protected ExitCode execCompiler(
CLICompiler<A> compiler,
MessageCollector messageCollector,
A arguments,
List<File> sourceRoots
) throws MojoExecutionException {
for (File sourceRoot : sourceRoots) {
arguments.freeArgs.add(sourceRoot.getPath());
}
return compiler.exec(messageCollector, Services.EMPTY, arguments);
}
private boolean hasKotlinFilesInSources() throws MojoExecutionException {
List<File> sources = getSourceDirs();
if (sources == null || sources.isEmpty()) return false;
@@ -400,32 +416,34 @@ public abstract class KotlinCompileMojoBase<A extends CommonCompilerArguments> e
return pluginOptions;
}
@NotNull
private List<File> getSourceRoots() throws MojoExecutionException {
List<File> sourceRoots = new ArrayList<File>();
for (File sourceDir : getSourceDirs()) {
if (sourceDir.exists()) {
sourceRoots.add(sourceDir);
}
else {
getLog().warn("Source root doesn't exist: " + sourceDir);
}
}
if (sourceRoots.isEmpty()) {
throw new MojoExecutionException("No source roots to compile");
}
getLog().info("Compiling Kotlin sources from " + sourceRoots);
return sourceRoots;
}
private void configureCompilerArguments(@NotNull A arguments, @NotNull CLICompiler<A> compiler) throws MojoExecutionException {
if (getLog().isDebugEnabled()) {
arguments.verbose = true;
}
List<String> sources = new ArrayList<String>();
for (File source : getSourceDirs()) {
if (source.exists()) {
sources.add(source.getPath());
}
else {
getLog().warn("Source root doesn't exist: " + source);
}
}
if (sources.isEmpty()) {
throw new MojoExecutionException("No source roots to compile");
}
arguments.suppressWarnings = nowarn;
arguments.languageVersion = languageVersion;
arguments.apiVersion = apiVersion;
arguments.multiPlatform = multiPlatform;
getLog().info("Compiling Kotlin sources from " + sources);
configureSpecificCompilerArguments(arguments);
try {
@@ -435,8 +453,6 @@ public abstract class KotlinCompileMojoBase<A extends CommonCompilerArguments> e
throw new MojoExecutionException(e.getMessage());
}
arguments.freeArgs.addAll(sources);
if (arguments.noInline) {
getLog().info("Method inlining is turned off");
}
@@ -0,0 +1,124 @@
package org.jetbrains.kotlin.maven.incremental;
import kotlin.jvm.functions.Function0;
import org.apache.maven.plugin.logging.Log;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.kotlin.cli.common.ExitCode;
import org.jetbrains.kotlin.incremental.ICReporter;
import java.io.File;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
public class MavenICReporter implements ICReporter {
private static final String IC_LOG_LEVEL_PROPERTY_NAME = "kotlin.compiler.incremental.log.level";
private static final String IC_LOG_LEVEL_NONE = "none";
private static final String IC_LOG_LEVEL_INFO = "info";
private static final String IC_LOG_LEVEL_DEBUG = "debug";
public static MavenICReporter get(@NotNull final Log log) {
String logLevel = System.getProperty(IC_LOG_LEVEL_PROPERTY_NAME);
if (logLevel == null) {
if (log.isDebugEnabled()) {
logLevel = IC_LOG_LEVEL_DEBUG;
}
else {
logLevel = IC_LOG_LEVEL_NONE;
}
}
if (logLevel.equalsIgnoreCase(IC_LOG_LEVEL_INFO)) {
return MavenICReporter.info(log);
}
else if (logLevel.equalsIgnoreCase(IC_LOG_LEVEL_DEBUG)) {
return MavenICReporter.debug(log);
}
else {
if (!logLevel.equalsIgnoreCase(IC_LOG_LEVEL_NONE)) {
log.warn("Unknown incremental compilation log level '" + logLevel + "'," +
"possible values: " + IC_LOG_LEVEL_NONE + ", " + IC_LOG_LEVEL_INFO + ", " + IC_LOG_LEVEL_DEBUG);
}
return MavenICReporter.noLog();
}
}
private static MavenICReporter info(@NotNull final Log log) {
return new MavenICReporter() {
@Override
protected boolean isLogEnabled() {
return log.isInfoEnabled();
}
@Override
protected void log(String str) {
log.info(str);
}
};
}
private static MavenICReporter debug(@NotNull final Log log) {
return new MavenICReporter() {
@Override
protected boolean isLogEnabled() {
return log.isDebugEnabled();
}
@Override
protected void log(String str) {
log.debug(str);
}
};
}
private static MavenICReporter noLog() {
return new MavenICReporter();
}
@NotNull
private final Set<File> compiledKotlinFiles = new HashSet<File>();
protected boolean isLogEnabled() {
return false;
}
protected void log(String str) {
}
private MavenICReporter() {
}
@Override
public void report(Function0<String> getMessage) {
if (isLogEnabled()) {
log(getMessage.invoke());
}
}
@Override
public void reportCompileIteration(Collection<? extends File> sourceFiles, ExitCode exitCode) {
compiledKotlinFiles.addAll(sourceFiles);
if (isLogEnabled()) {
log("Kotlin compile iteration: " + pathsAsString(sourceFiles));
log("Exit code: " + exitCode.toString());
}
}
@NotNull
@Override
public String pathsAsString(Iterable<? extends File> files) {
return ICReporter.DefaultImpls.pathsAsString(this, files);
}
@NotNull
@Override
public String pathsAsString(File... files) {
return ICReporter.DefaultImpls.pathsAsString(this, files);
}
@NotNull
public Set<File> getCompiledKotlinFiles() {
return compiledKotlinFiles;
}
}
@@ -180,4 +180,9 @@ public class KaptJVMCompilerMojo extends K2JVMCompileMojo {
return result;
}
@Override
protected boolean isIncremental() {
return false;
}
}