From f40a3eff20f463659be66041d78d255a80d12322 Mon Sep 17 00:00:00 2001 From: Alexey Tsvetkov Date: Thu, 24 Nov 2016 18:07:07 +0300 Subject: [PATCH] Move IC from Gradle to compiler Reasons for moving: 1. Integration with Kotlin daemon for Gradle. Gradle bundles Kotlin compiler (kotlin-compiler-embeddable) since Gradle 3.2. There is no plugin isolation in Gradle, so strange compilation errors/exceptions may happen. We decided to fix this problem by compiling out-of-process using Kotlin daemon. Reasons for moving IC to the compiler: * Better isolation from Gradle process. * Incremental compilation logic is not scattered across two process and multiple modules. * Makes it possible to implement standalone CLI IC --- .idea/modules.xml | 1 + build.xml | 64 +++------------ .../incremental-compilation-impl.iml | 19 +++++ .../annotation/AnnotationFileUpdater.kt | 37 +++++++++ .../jetbrains/kotlin/incremental/BuildInfo.kt | 0 .../kotlin/incremental/ChangedFiles.kt | 13 +-- .../incremental/ChangedJavaFilesProcessor.kt | 0 .../kotlin/incremental/ChangesEither.kt | 0 .../incremental/GradleIncrementalCacheImpl.kt | 0 .../incremental/IncrementalCachesManager.kt | 0 .../IncrementalJvmCompilerRunner.kt | 0 .../jetbrains/kotlin/incremental/fileUtils.kt | 40 ++++++++++ .../ArtifactDifferenceRegistry.kt | 0 .../incremental/snapshots/FileSnapshot.kt | 0 .../snapshots/FileSnapshotExternalizer.kt | 0 .../incremental/snapshots/FileSnapshotMap.kt | 0 .../snapshots/FileSnapshotProvider.kt | 0 .../kotlin/incremental/snapshots/hashUtil.kt | 0 .../incremental/standaloneCacheVersion.kt | 0 libraries/pom.xml | 1 - libraries/tools/kotlin-build-common/pom.xml | 80 ------------------- .../annotation/AnnotationFileUpdaterImpl.kt | 12 --- .../kotlin/incremental/ChangedFiles.kt | 5 -- 23 files changed, 116 insertions(+), 156 deletions(-) create mode 100644 compiler/incremental-compilation-impl/incremental-compilation-impl.iml create mode 100644 compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/annotation/AnnotationFileUpdater.kt rename {libraries/tools/kotlin-gradle-plugin/src/main/kotlin => compiler/incremental-compilation-impl/src}/org/jetbrains/kotlin/incremental/BuildInfo.kt (100%) rename libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/annotation/AnnotationFileUpdater.kt => compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedFiles.kt (67%) rename {libraries/tools/kotlin-gradle-plugin/src/main/kotlin => compiler/incremental-compilation-impl/src}/org/jetbrains/kotlin/incremental/ChangedJavaFilesProcessor.kt (100%) rename {libraries/tools/kotlin-gradle-plugin/src/main/kotlin => compiler/incremental-compilation-impl/src}/org/jetbrains/kotlin/incremental/ChangesEither.kt (100%) rename {libraries/tools/kotlin-gradle-plugin/src/main/kotlin => compiler/incremental-compilation-impl/src}/org/jetbrains/kotlin/incremental/GradleIncrementalCacheImpl.kt (100%) rename {libraries/tools/kotlin-gradle-plugin/src/main/kotlin => compiler/incremental-compilation-impl/src}/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt (100%) rename {libraries/tools/kotlin-gradle-plugin/src/main/kotlin => compiler/incremental-compilation-impl/src}/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt (100%) create mode 100644 compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/fileUtils.kt rename {libraries/tools/kotlin-gradle-plugin/src/main/kotlin => compiler/incremental-compilation-impl/src}/org/jetbrains/kotlin/incremental/multiproject/ArtifactDifferenceRegistry.kt (100%) rename {libraries/tools/kotlin-gradle-plugin/src/main/kotlin => compiler/incremental-compilation-impl/src}/org/jetbrains/kotlin/incremental/snapshots/FileSnapshot.kt (100%) rename {libraries/tools/kotlin-gradle-plugin/src/main/kotlin => compiler/incremental-compilation-impl/src}/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotExternalizer.kt (100%) rename {libraries/tools/kotlin-gradle-plugin/src/main/kotlin => compiler/incremental-compilation-impl/src}/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt (100%) rename {libraries/tools/kotlin-gradle-plugin/src/main/kotlin => compiler/incremental-compilation-impl/src}/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotProvider.kt (100%) rename {libraries/tools/kotlin-gradle-plugin/src/main/kotlin => compiler/incremental-compilation-impl/src}/org/jetbrains/kotlin/incremental/snapshots/hashUtil.kt (100%) rename {libraries/tools/kotlin-gradle-plugin/src/main/kotlin => compiler/incremental-compilation-impl/src}/org/jetbrains/kotlin/incremental/standaloneCacheVersion.kt (100%) delete mode 100644 libraries/tools/kotlin-build-common/pom.xml diff --git a/.idea/modules.xml b/.idea/modules.xml index 0abc0ef6b64..d68d76f1106 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -56,6 +56,7 @@ + diff --git a/build.xml b/build.xml index 5d80f49a632..b09bbef90dd 100644 --- a/build.xml +++ b/build.xml @@ -107,6 +107,8 @@ + + @@ -139,6 +141,8 @@ + + @@ -245,6 +249,8 @@ + + @@ -684,54 +690,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -739,7 +697,10 @@ source="${java.target}" target="${java.target}"> - + + + + @@ -747,7 +708,6 @@ - @@ -902,7 +862,7 @@ - + @@ -1313,7 +1273,7 @@ depends="builtins,stdlib,kotlin-test,core,reflection,pack-runtime,pack-runtime-sources,script-runtime,mock-runtime-for-test"/> + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/annotation/AnnotationFileUpdater.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/annotation/AnnotationFileUpdater.kt new file mode 100644 index 00000000000..1220348d750 --- /dev/null +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/annotation/AnnotationFileUpdater.kt @@ -0,0 +1,37 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.annotation + +import org.jetbrains.kotlin.resolve.jvm.JvmClassName + +/** + * Annotation file is generated by collecting annotated elements of generated files. + * When compiling incrementally, the compiler generates only subset of all classes, + * so after compilation annotation file contains only a subset of annotated elements, + * which breaks the build. + * + * The workaround is to: + * 1. backup old file before incremental compilation; + * 2. after each iteration of IC: + * 2.1 remove classes corresponding to dirty source files + * 2.2 add annotations from newly generated annotations file + */ +interface AnnotationFileUpdater { + fun updateAnnotations(outdatedClasses: Iterable) + fun revert() +} + diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/BuildInfo.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildInfo.kt similarity index 100% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/BuildInfo.kt rename to compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/BuildInfo.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/annotation/AnnotationFileUpdater.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedFiles.kt similarity index 67% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/annotation/AnnotationFileUpdater.kt rename to compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedFiles.kt index b536e6f94ad..df94e49abd1 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/annotation/AnnotationFileUpdater.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedFiles.kt @@ -14,12 +14,13 @@ * limitations under the License. */ -package org.jetbrains.kotlin.annotation +package org.jetbrains.kotlin.incremental -import org.jetbrains.kotlin.resolve.jvm.JvmClassName +import org.gradle.api.tasks.incremental.IncrementalTaskInputs +import java.io.File +import java.util.* -interface AnnotationFileUpdater { - fun updateAnnotations(outdatedClasses: Iterable) - fun revert() +internal sealed class ChangedFiles { + class Known(val modified: List, val removed: List) : ChangedFiles() + class Unknown : ChangedFiles() } - diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/ChangedJavaFilesProcessor.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedJavaFilesProcessor.kt similarity index 100% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/ChangedJavaFilesProcessor.kt rename to compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangedJavaFilesProcessor.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/ChangesEither.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangesEither.kt similarity index 100% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/ChangesEither.kt rename to compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/ChangesEither.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/GradleIncrementalCacheImpl.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/GradleIncrementalCacheImpl.kt similarity index 100% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/GradleIncrementalCacheImpl.kt rename to compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/GradleIncrementalCacheImpl.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt similarity index 100% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt rename to compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt similarity index 100% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt rename to compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/fileUtils.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/fileUtils.kt new file mode 100644 index 00000000000..0129e427e03 --- /dev/null +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/fileUtils.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.incremental + +import java.io.File + +internal fun File.isJavaFile() = + extension.equals("java", ignoreCase = true) + +internal fun File.isKotlinFile(): Boolean = + extension.let { + "kt".equals(it, ignoreCase = true) || + "kts".equals(it, ignoreCase = true) + } + +internal fun File.isClassFile(): Boolean = + extension.equals("class", ignoreCase = true) + +internal fun listClassFiles(path: String): Sequence = + File(path).walk().filter { it.isFile && it.isClassFile() } + +internal fun File.relativeOrCanonical(base: File): String = + relativeToOrNull(base)?.path ?: canonicalPath + +internal fun Iterable.pathsAsStringRelativeTo(base: File): String = + map { it.relativeOrCanonical(base) }.sorted().joinToString() diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/multiproject/ArtifactDifferenceRegistry.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/multiproject/ArtifactDifferenceRegistry.kt similarity index 100% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/multiproject/ArtifactDifferenceRegistry.kt rename to compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/multiproject/ArtifactDifferenceRegistry.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshot.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshot.kt similarity index 100% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshot.kt rename to compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshot.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotExternalizer.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotExternalizer.kt similarity index 100% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotExternalizer.kt rename to compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotExternalizer.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt similarity index 100% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt rename to compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotProvider.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotProvider.kt similarity index 100% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotProvider.kt rename to compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotProvider.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/hashUtil.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/hashUtil.kt similarity index 100% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/hashUtil.kt rename to compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/hashUtil.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/standaloneCacheVersion.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/standaloneCacheVersion.kt similarity index 100% rename from libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/standaloneCacheVersion.kt rename to compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/standaloneCacheVersion.kt diff --git a/libraries/pom.xml b/libraries/pom.xml index 04b4c3b361e..35fbdc850b2 100644 --- a/libraries/pom.xml +++ b/libraries/pom.xml @@ -77,7 +77,6 @@ tools/native-platform tools/kotlin-compiler-runner tools/kotlin-daemon-client - tools/kotlin-build-common tools/kotlin-build-common-test tools/kotlin-maven-plugin diff --git a/libraries/tools/kotlin-build-common/pom.xml b/libraries/tools/kotlin-build-common/pom.xml deleted file mode 100644 index 8b8a1486d04..00000000000 --- a/libraries/tools/kotlin-build-common/pom.xml +++ /dev/null @@ -1,80 +0,0 @@ - - - - 4.0.0 - - 1.4.1 - 3.0.4 - - - - org.jetbrains.kotlin - kotlin-project - 1.1-SNAPSHOT - ../../pom.xml - - - kotlin-build-common - jar - - the Kotlin builders - - - ../../../compiler/compiler-runner/src - - - - org.apache.maven.plugins - maven-source-plugin - - - attach-sources - - jar - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - skip-compile - compile - - compile - - - true - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - copy-jar - package - - - - - - - run - - - - - - - diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/annotation/AnnotationFileUpdaterImpl.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/annotation/AnnotationFileUpdaterImpl.kt index 2a8362e118b..034f4f3e1e9 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/annotation/AnnotationFileUpdaterImpl.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/annotation/AnnotationFileUpdaterImpl.kt @@ -5,18 +5,6 @@ import org.jetbrains.kotlin.gradle.plugin.kotlinDebug import org.jetbrains.kotlin.resolve.jvm.JvmClassName import java.io.File -/** - * Annotation file is generated by collecting annotated elements of generated files. - * When compiling incrementally, the compiler generates only subset of all classes, - * so after compilation annotation file contains only a subset of annotated elements, - * which breaks the build. - * - * The workaround is to: - * 1. backup old file before incremental compilation; - * 2. after each iteration of IC: - * 2.1 remove classes corresponding to dirty source files - * 2.2 add annotations from newly generated annotations file - */ internal class AnnotationFileUpdaterImpl(private val generatedAnnotationFile: File) : AnnotationFileUpdater { private val logger = Logging.getLogger(this.javaClass) private val lastSuccessfullyUpdatedFile = File.createTempFile("kapt-annotations-copy", "tmp") diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/ChangedFiles.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/ChangedFiles.kt index 7d7c84fcd9d..835876b04bf 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/ChangedFiles.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/ChangedFiles.kt @@ -20,11 +20,6 @@ import org.gradle.api.tasks.incremental.IncrementalTaskInputs import java.io.File import java.util.* -internal sealed class ChangedFiles { - class Known(val modified: List, val removed: List) : ChangedFiles() - class Unknown : ChangedFiles() -} - internal fun ChangedFiles(taskInputs: IncrementalTaskInputs): ChangedFiles { if (!taskInputs.isIncremental) return ChangedFiles.Unknown()