From 7c16d086aa669407f3a50019a1762076cf7d1689 Mon Sep 17 00:00:00 2001 From: Alexey Tsvetkov Date: Thu, 13 Oct 2016 16:22:06 +0300 Subject: [PATCH] Groundwork for using IC without Gradle --- .../kotlin/incremental/IncReporter.kt | 9 +- .../kotlin/jps/build/KotlinBuilder.kt | 3 - libraries/tools/kotlin-gradle-plugin/pom.xml | 6 + .../jetbrains/kotlin/gradle/tasks/Tasks.kt | 2 +- .../incremental/ChangedJavaFilesProcessor.kt | 12 +- .../incremental/GradleIncrementalCacheImpl.kt | 13 +- .../IncrementalJvmCompilerRunner.kt | 77 ++++-- .../kotlin/incremental/gradleCacheVersion.kt | 2 +- .../incremental/snapshots/FileSnapshotMap.kt | 10 +- .../incremental/standaloneCacheVersion.kt | 31 +++ ...linStandaloneIncrementalCompilationTest.kt | 223 ++++++++++++++++++ .../snapshots/FileSnapshotMapTest.kt | 9 +- .../incremental/testLogsParsingUtil.kt | 2 + 13 files changed, 354 insertions(+), 45 deletions(-) create mode 100644 libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/standaloneCacheVersion.kt create mode 100644 libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/incremental/KotlinStandaloneIncrementalCompilationTest.kt rename libraries/tools/{kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle => kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin}/incremental/testLogsParsingUtil.kt (98%) diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncReporter.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncReporter.kt index db51626aaad..b6d1dc42cdb 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/IncReporter.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncReporter.kt @@ -16,11 +16,18 @@ package org.jetbrains.kotlin.incremental +import org.jetbrains.kotlin.cli.common.ExitCode import java.io.File abstract class IncReporter { abstract fun report(message: ()->String) - abstract fun pathsAsString(files: Iterable): String + + // used in Gradle plugin + @Suppress("unused") + open fun reportCompileIteration(sourceFiles: Iterable, exitCode: ExitCode) {} + + open fun pathsAsString(files: Iterable): String = + files.map { it.canonicalPath }.joinToString() fun pathsAsString(vararg files: File): String = pathsAsString(files.toList()) diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt index 1c9d32d930c..45d378514d6 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt @@ -794,9 +794,6 @@ private class JpsIncReporter : IncReporter() { KotlinBuilder.LOG.debug(message()) } } - - override fun pathsAsString(files: Iterable): String = - files.map { it.canonicalPath }.joinToString() } private fun CompilationResult.doProcessChangesUsingLookups( diff --git a/libraries/tools/kotlin-gradle-plugin/pom.xml b/libraries/tools/kotlin-gradle-plugin/pom.xml index 38ae3103496..09dca5f04be 100644 --- a/libraries/tools/kotlin-gradle-plugin/pom.xml +++ b/libraries/tools/kotlin-gradle-plugin/pom.xml @@ -55,6 +55,12 @@ kotlin-annotation-processing ${project.version} + + org.jetbrains.kotlin + kotlin-build-common-test + ${project.version} + test + com.android.tools.build gradle diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt index a73baa8fec4..497f7af77c7 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt @@ -195,7 +195,7 @@ open class KotlinCompile : AbstractKotlinCompile(), Kotl ) args.classpathAsList = classpath.toList() args.destinationAsFile = destinationDir - compiler.compile(allKotlinSources, changedFiles, args, messageCollector) + compiler.compile(allKotlinSources, args, messageCollector, { changedFiles }) anyClassesCompiled = compiler.anyClassesCompiled } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/ChangedJavaFilesProcessor.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/ChangedJavaFilesProcessor.kt index 96f6e335923..f07b2cd6e16 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/ChangedJavaFilesProcessor.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/ChangedJavaFilesProcessor.kt @@ -27,7 +27,6 @@ import org.jetbrains.kotlin.com.intellij.psi.PsiFileFactory import org.jetbrains.kotlin.com.intellij.psi.PsiJavaFile import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.gradle.plugin.kotlinDebug -import org.jetbrains.kotlin.incremental.snapshots.FileCollectionDiff import java.io.File import java.util.* @@ -46,14 +45,17 @@ internal class ChangedJavaFilesProcessor { val allChangedSymbols: Collection get() = allSymbols - fun process(filesDiff: FileCollectionDiff): ChangesEither { - if (filesDiff.removed.any()) { - log.kotlinDebug { "Some java files are removed: [${filesDiff.removed.joinToString()}]" } + fun process(filesDiff: ChangedFiles.Known): ChangesEither { + val modifiedJava = filesDiff.modified.filter(File::isJavaFile) + val removedJava = filesDiff.removed.filter(File::isJavaFile) + + if (removedJava.any()) { + log.kotlinDebug { "Some java files are removed: [${removedJava.joinToString()}]" } return ChangesEither.Unknown() } val symbols = HashSet() - for (javaFile in filesDiff.newOrModified) { + for (javaFile in modifiedJava) { assert(javaFile.extension.equals("java", ignoreCase = true)) val psiFile = javaFile.psiFile() diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/GradleIncrementalCacheImpl.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/GradleIncrementalCacheImpl.kt index 5b40ce385f2..7e4133a981e 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/GradleIncrementalCacheImpl.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/GradleIncrementalCacheImpl.kt @@ -18,9 +18,7 @@ package org.jetbrains.kotlin.incremental import org.jetbrains.kotlin.build.GeneratedJvmClass import org.jetbrains.kotlin.gradle.plugin.kotlinDebug -import org.jetbrains.kotlin.incremental.snapshots.FileCollectionDiff import org.jetbrains.kotlin.incremental.snapshots.FileSnapshotMap -import org.jetbrains.kotlin.incremental.snapshots.SimpleFileSnapshotProviderImpl import org.jetbrains.kotlin.incremental.storage.BasicStringMap import org.jetbrains.kotlin.incremental.storage.PathStringDescriptor import org.jetbrains.kotlin.incremental.storage.StringCollectionExternalizer @@ -30,16 +28,15 @@ import java.io.File internal class GradleIncrementalCacheImpl(targetDataRoot: File, targetOutputDir: File?, target: TargetId) : IncrementalCacheImpl(targetDataRoot, targetOutputDir, target) { companion object { private val SOURCES_TO_CLASSFILES = "sources-to-classfiles" - private val FILE_SNAPSHOT = "file-snapshot" + private val GENERATED_SOURCE_SNAPSHOTS = "generated-source-snapshot" + private val SOURCE_SNAPSHOTS = "source-snapshot" } private val log = org.gradle.api.logging.Logging.getLogger(this.javaClass) - private val sourceToClassfilesMap = registerMap(SourceToClassfilesMap(GradleIncrementalCacheImpl.Companion.SOURCES_TO_CLASSFILES.storageFile)) - private val fileSnapshotMap = registerMap(FileSnapshotMap(FILE_SNAPSHOT.storageFile)) - - fun compareAndUpdateFileSnapshots(files: Iterable): FileCollectionDiff = - fileSnapshotMap.compareAndUpdate(files, SimpleFileSnapshotProviderImpl()) + internal val sourceToClassfilesMap = registerMap(SourceToClassfilesMap(SOURCES_TO_CLASSFILES.storageFile)) + internal val generatedSourceSnapshotMap = registerMap(FileSnapshotMap(GENERATED_SOURCE_SNAPSHOTS.storageFile)) + internal val sourceSnapshotMap = registerMap(FileSnapshotMap(SOURCE_SNAPSHOTS.storageFile)) fun removeClassfilesBySources(sources: Iterable): Unit = sources.forEach { sourceToClassfilesMap.remove(it) } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt index c04c9238bae..2d7479c60af 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt @@ -30,16 +30,63 @@ import org.jetbrains.kotlin.com.intellij.util.io.PersistentEnumeratorBase import org.jetbrains.kotlin.compilerRunner.ArgumentUtils import org.jetbrains.kotlin.compilerRunner.OutputItemsCollector import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl +import org.jetbrains.kotlin.config.IncrementalCompilation import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.multiproject.ArtifactDifference import org.jetbrains.kotlin.incremental.multiproject.ArtifactDifferenceRegistryProvider -import org.jetbrains.kotlin.incremental.snapshots.FileCollectionDiff import org.jetbrains.kotlin.modules.TargetId import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.progress.CompilationCanceledStatus import java.io.File import java.util.* +internal fun makeIncrementally( + cachesDir: File, + sourceRoots: Iterable, + args: K2JVMCompilerArguments, + messageCollector: MessageCollector = MessageCollector.NONE, + reporter: IncReporter = EmptyIncReporter +) { + val versions = listOf(normalCacheVersion(cachesDir), + experimentalCacheVersion(cachesDir), + dataContainerCacheVersion(cachesDir), + standaloneCacheVersion(cachesDir)) + + val kotlinExtensions = listOf("kt", "kts") + val allExtensions = kotlinExtensions + listOf("java") + val rootsWalk = sourceRoots.asSequence().map { it.walk() }.flatten() + val files = rootsWalk.filter(File::isFile) + val sourceFiles = files.filter { it.extension.toLowerCase() in allExtensions }.toList() + val kotlinFiles = sourceFiles.filter { it.extension.toLowerCase() in kotlinExtensions } + + enableIC { + val compiler = IncrementalJvmCompilerRunner(cachesDir, /* javaSourceRoots = */sourceRoots.toSet(), versions, reporter) + compiler.compile(kotlinFiles, args, messageCollector) { + it.incrementalCache.sourceSnapshotMap.compareAndUpdate(sourceFiles) + } + } +} + +private object EmptyIncReporter : IncReporter() { + override fun report(message: ()->String) { + } +} + +private inline fun enableIC(fn: ()->Unit) { + val isEnabledBackup = IncrementalCompilation.isEnabled() + val isExperimentalBackup = IncrementalCompilation.isExperimental() + IncrementalCompilation.setIsEnabled(true) + IncrementalCompilation.setIsExperimental(true) + + try { + fn() + } + finally { + IncrementalCompilation.setIsEnabled(isEnabledBackup) + IncrementalCompilation.setIsExperimental(isExperimentalBackup) + } +} + internal class IncrementalJvmCompilerRunner( workingDir: File, private val javaSourceRoots: Set, @@ -59,15 +106,16 @@ internal class IncrementalJvmCompilerRunner( fun compile( allKotlinSources: List, - changedFiles: ChangedFiles, args: K2JVMCompilerArguments, - messageCollector: MessageCollector + messageCollector: MessageCollector, + getChangedFiles: (IncrementalCachesManager)->ChangedFiles ): ExitCode { val targetId = TargetId(name = args.moduleName, type = "java-production") var caches = IncrementalCachesManager(targetId, cacheDirectory, File(args.destination)) return try { val javaFilesProcessor = ChangedJavaFilesProcessor() + val changedFiles = getChangedFiles(caches) val compilationMode = calculateSourcesToCompile(javaFilesProcessor, caches, changedFiles, args.classpathAsList) compileIncrementally(args, caches, javaFilesProcessor, allKotlinSources, targetId, compilationMode, messageCollector) } @@ -119,10 +167,8 @@ internal class IncrementalJvmCompilerRunner( if (classpathChanges !is ChangesEither.Known) { return rebuild {"could not get changes from modified classpath entries: ${reporter.pathsAsString(modifiedClasspathEntries)}"} } - val javaFilesDiff = FileCollectionDiff( - newOrModified = changedFiles.modified.filter(File::isJavaFile), - removed = changedFiles.removed.filter(File::isJavaFile)) - val javaFilesChanges = javaFilesProcessor.process(javaFilesDiff) + + val javaFilesChanges = javaFilesProcessor.process(changedFiles) val affectedJavaSymbols = when (javaFilesChanges) { is ChangesEither.Known -> javaFilesChanges.lookupSymbols is ChangesEither.Unknown -> return rebuild {"Could not get changes for java files"} @@ -214,12 +260,15 @@ internal class IncrementalJvmCompilerRunner( compilationMode: CompilationMode, messageCollector: MessageCollector ): ExitCode { + assert(IncrementalCompilation.isEnabled()) { "Incremental compilation is not enabled" } + assert(IncrementalCompilation.isExperimental()) { "Experimental incremental compilation is not enabled" } + val allGeneratedFiles = hashSetOf>() val dirtySources: MutableList when (compilationMode) { is CompilationMode.Incremental -> { - dirtySources = allKotlinSources.filterTo(ArrayList()) { it in compilationMode.dirtyFiles } + dirtySources = ArrayList(compilationMode.dirtyFiles) args.classpathAsList += args.destinationAsFile } is CompilationMode.Rebuild -> { @@ -274,7 +323,7 @@ internal class IncrementalJvmCompilerRunner( caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources) val generatedJavaFiles = (kapt2GeneratedSourcesDir?.walk() ?: emptySequence()).filter(File::isJavaFile).toList() - val generatedJavaFilesDiff = caches.incrementalCache.compareAndUpdateFileSnapshots(generatedJavaFiles) + val generatedJavaFilesDiff = caches.incrementalCache.generatedSourceSnapshotMap.compareAndUpdate(generatedJavaFiles) if (compilationMode is CompilationMode.Rebuild) { artifactFile?.let { artifactFile -> @@ -378,13 +427,9 @@ internal class IncrementalJvmCompilerRunner( reporter.report { "compiling with classpath: ${classpath.toList().sorted().joinToString()}" } val compileServices = makeCompileServices(incrementalCaches, lookupTracker, compilationCanceledStatus, sourceAnnotationsRegistry) val exitCode = compiler.exec(messageCollector, compileServices, args) - return CompileChangedResults( - exitCode, - outputItemCollector.generatedFiles( - targets = targets, - representativeTarget = targets.first(), - getSources = {sourcesToCompile}, - getOutputDir = {outputDir})) + val generatedFiles = outputItemCollector.generatedFiles(targets, targets.first(), {sourcesToCompile}, {outputDir}) + reporter.reportCompileIteration(sourcesToCompile, exitCode) + return CompileChangedResults(exitCode, generatedFiles) } finally { moduleFile.delete() diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/gradleCacheVersion.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/gradleCacheVersion.kt index fdbabb8d088..7eac0969a14 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/gradleCacheVersion.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/gradleCacheVersion.kt @@ -19,7 +19,7 @@ package org.jetbrains.kotlin.incremental import org.jetbrains.kotlin.config.IncrementalCompilation import java.io.File -internal const val GRADLE_CACHE_VERSION = 2 +internal const val GRADLE_CACHE_VERSION = 3 internal const val GRADLE_CACHE_VERSION_FILE_NAME = "gradle-format-version.txt" internal fun gradleCacheVersion(dataRoot: File): CacheVersion = diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt index 06448b23c15..50f3f23c127 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt @@ -16,6 +16,7 @@ package org.jetbrains.kotlin.incremental.snapshots +import org.jetbrains.kotlin.incremental.ChangedFiles import org.jetbrains.kotlin.incremental.storage.BasicStringMap import org.jetbrains.kotlin.incremental.storage.PathStringDescriptor import java.io.File @@ -25,7 +26,8 @@ internal class FileSnapshotMap(storageFile: File) : BasicStringMap override fun dumpValue(value: FileSnapshot): String = value.toString() - fun compareAndUpdate(newFiles: Iterable, snapshotProvider: FileSnapshotProvider): FileCollectionDiff { + fun compareAndUpdate(newFiles: Iterable): ChangedFiles.Known { + val snapshotProvider = SimpleFileSnapshotProviderImpl() val newOrModified = ArrayList() val removed = ArrayList() @@ -48,8 +50,6 @@ internal class FileSnapshotMap(storageFile: File) : BasicStringMap } } - return FileCollectionDiff(newOrModified, removed) + return ChangedFiles.Known(newOrModified, removed) } -} - -internal class FileCollectionDiff(val newOrModified: Iterable, val removed: Iterable) \ No newline at end of file +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/standaloneCacheVersion.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/standaloneCacheVersion.kt new file mode 100644 index 00000000000..60297754e7c --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/incremental/standaloneCacheVersion.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.incremental + +import org.jetbrains.kotlin.config.IncrementalCompilation +import java.io.File + +internal const val STANDALONE_CACHE_VERSION = 0 +internal const val STANDALONE_VERSION_FILE_NAME = "standalone-ic-format-version.txt" + +internal fun standaloneCacheVersion(dataRoot: File): CacheVersion = + CacheVersion(ownVersion = STANDALONE_CACHE_VERSION, + versionFile = File(dataRoot, STANDALONE_VERSION_FILE_NAME), + whenVersionChanged = CacheVersion.Action.REBUILD_ALL_KOTLIN, + whenTurnedOn = CacheVersion.Action.REBUILD_ALL_KOTLIN, + whenTurnedOff = CacheVersion.Action.REBUILD_ALL_KOTLIN, + isEnabled = { IncrementalCompilation.isExperimental() }) diff --git a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/incremental/KotlinStandaloneIncrementalCompilationTest.kt b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/incremental/KotlinStandaloneIncrementalCompilationTest.kt new file mode 100644 index 00000000000..f1803407edb --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/incremental/KotlinStandaloneIncrementalCompilationTest.kt @@ -0,0 +1,223 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.incremental + +import org.jetbrains.kotlin.TestWithWorkingDir +import org.jetbrains.kotlin.cli.common.ExitCode +import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity +import org.jetbrains.kotlin.cli.common.messages.MessageCollector +import org.jetbrains.kotlin.com.intellij.util.containers.HashMap +import org.jetbrains.kotlin.gradle.incremental.parseTestBuildLog +import org.jetbrains.kotlin.incremental.testingUtils.* +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.Parameterized +import java.io.File +import java.util.* +import kotlin.test.assertEquals + +@RunWith(Parameterized::class) +class KotlinStandaloneIncrementalCompilationTest : TestWithWorkingDir() { + @Parameterized.Parameter + lateinit var testDir: File + + @Suppress("unused") + @Parameterized.Parameter(value = 1) + lateinit var readableName: String + + @Test + fun testFromJps() { + fun Iterable.relativePaths() = + map { it.relativeTo(workingDir).path.replace('\\', '/') } + + val srcDir = File(workingDir, "src").apply { mkdirs() } + val cacheDir = File(workingDir, "incremental-data").apply { mkdirs() } + val outDir = File(workingDir, "out").apply { mkdirs() } + + val mapWorkingToOriginalFile = HashMap(copyTestSources(testDir, srcDir, filePrefix = "")) + val sourceRoots = listOf(srcDir) + val args = K2JVMCompilerArguments() + args.destination = outDir.path + args.moduleName = testDir.name + args.classpath = compileClasspath() + // initial build + make(cacheDir, sourceRoots, args) + + // modifications + val buildLogFile = buildLogFinder.findBuildLog(testDir) ?: throw IllegalStateException("build log file not found in $workingDir") + val buildLogSteps = parseTestBuildLog(buildLogFile) + val modifications = getModificationsToPerform(testDir, + moduleNames = null, + allowNoFilesWithSuffixInTestData = false, + touchPolicy = TouchPolicy.CHECKSUM) + + assert(modifications.size == buildLogSteps.size) { + "Modifications count (${modifications.size}) != expected build log steps count (${buildLogSteps.size})" + } + + // Sometimes error messages differ. + // This needs to be fixed, but it does not really matter much (e.g extra lines), + // The workaround is to compare logs without errors, then logs with errors. + // (if logs without errors differ then either compiled files differ or exit codes differ) + val expectedSB = StringBuilder() + val actualSB = StringBuilder() + val expectedSBWithoutErrors = StringBuilder() + val actualSBWithoutErrors = StringBuilder() + var step = 1 + for ((modificationStep, buildLogStep) in modifications.zip(buildLogSteps)) { + modificationStep.forEach { it.perform(workingDir, mapWorkingToOriginalFile) } + val (exitCode, compiledSources, compileErrors) = make(cacheDir, sourceRoots, args) + + expectedSB.appendLine(stepLogAsString(step, buildLogStep.compiledKotlinFiles, buildLogStep.compileErrors)) + expectedSBWithoutErrors.appendLine(stepLogAsString(step, buildLogStep.compiledKotlinFiles, buildLogStep.compileErrors, includeErrors = false)) + actualSB.appendLine(stepLogAsString(step, compiledSources.relativePaths(), compileErrors)) + actualSBWithoutErrors.appendLine(stepLogAsString(step, compiledSources.relativePaths(), compileErrors, includeErrors = false)) + step++ + } + + if (expectedSBWithoutErrors.toString() != actualSBWithoutErrors.toString()) { + assertEquals(expectedSB.toString(), actualSB.toString()) + } + + // todo: also compare caches + run rebuildAndCompareOutput@ { + val rebuildOutDir = File(workingDir, "rebuild-out").apply { mkdirs() } + val rebuildCacheDir = File(workingDir, "rebuild-cache").apply { mkdirs() } + args.destination = rebuildOutDir.path + val rebuildResult = make(rebuildCacheDir, sourceRoots, args) + + val rebuildExpectedToSucceed = buildLogSteps.last().compileSucceeded + val rebuildSucceeded = rebuildResult.exitCode == ExitCode.OK + assertEquals(rebuildExpectedToSucceed, rebuildSucceeded, "Rebuild exit code differs from incremental exit code") + + if (rebuildSucceeded) { + assertEqualDirectories(outDir, rebuildOutDir, forgiveExtraFiles = rebuildSucceeded) + } + } + } + + private fun compileClasspath(): String { + val currentClasspath = System.getProperty("java.class.path").split(File.pathSeparator) + val stdlib = currentClasspath.find { it.contains("kotlin-stdlib") } + val runtime = currentClasspath.find { it.contains("kotlin-runtime") } + return listOf(stdlib, runtime).joinToString(File.pathSeparator) + } + + data class CompilationResult(val exitCode: ExitCode, val compiledSources: Iterable, val compileErrors: Collection) + + private fun make(cacheDir: File, sourceRoots: Iterable, args: K2JVMCompilerArguments): CompilationResult { + val compiledSources = arrayListOf() + var resultExitCode = ExitCode.OK + + val reporter = object : IncReporter() { + override fun report(message: ()->String) { + } + + override fun reportCompileIteration(sourceFiles: Iterable, exitCode: ExitCode) { + compiledSources.addAll(sourceFiles) + resultExitCode = exitCode + } + } + + val messageCollector = ErrorMessageCollector() + makeIncrementally(cacheDir, sourceRoots, args, reporter = reporter, messageCollector = messageCollector) + return CompilationResult(resultExitCode, compiledSources, messageCollector.errors) + } + + private fun stepLogAsString(step: Int, ktSources: Iterable, errors: Collection, includeErrors: Boolean = true): String { + val sb = StringBuilder() + + sb.appendLine("<======= STEP $step =======>") + sb.appendLine() + sb.appendLine("Compiled kotlin sources:") + ktSources.toSet().toTypedArray().sortedArray().forEach { sb.appendLine(it) } + sb.appendLine() + + if (errors.isEmpty()) { + sb.appendLine("SUCCESS") + } + else { + sb.appendLine("FAILURE") + if (includeErrors) { + errors.filter(String::isNotEmpty).forEach { sb.appendLine(it) } + } + } + + return sb.toString() + } + + private fun StringBuilder.appendLine(line: String = "") { + append(line) + append('\n') + } + + private class ErrorMessageCollector : MessageCollector { + val errors = ArrayList() + + override fun report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation) { + if (severity.isError) { + errors.add(message) + } + } + + override fun clear() { + errors.clear() + } + + override fun hasErrors(): Boolean = + errors.isNotEmpty() + } + + companion object { + private val jpsResourcesPath = File("../../../jps-plugin/testData/incremental") + private val ignoredDirs = setOf(File(jpsResourcesPath, "cacheVersionChanged"), + File(jpsResourcesPath, "changeIncrementalOption"), + File(jpsResourcesPath, "custom"), + File(jpsResourcesPath, "lookupTracker")) + private val buildLogFinder = BuildLogFinder(isExperimentalEnabled = true, isGradleEnabled = true) + + @Suppress("unused") + @Parameterized.Parameters(name = "{1}") + @JvmStatic + fun data(): List> { + fun File.isValidTestDir(): Boolean { + if (!isDirectory) return false + + // multi-module tests + val files = list() + if ("dependencies.txt" in files) return false + + val logFile = buildLogFinder.findBuildLog(this) ?: return false + val parsedLog = parseTestBuildLog(logFile) + // tests with java may be expected to fail in javac + return files.none { it.endsWith(".java") } && parsedLog.all { it.compiledJavaFiles.isEmpty() } + } + + fun File.relativeToGrandfather() = + relativeTo(parentFile.parentFile).path + + return jpsResourcesPath.walk() + .onEnter { it !in ignoredDirs } + .filter(File::isValidTestDir) + .map { arrayOf(it, it.relativeToGrandfather()) } + .toList() + } + } +} + diff --git a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt index b3812a1b317..4b7cbcac351 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt @@ -35,14 +35,13 @@ class FileSnapshotMapTest : TestWithWorkingDir() { val unchangedTxt = File(foo, "unchanged.txt").apply { writeText("unchanged") } val changedTxt = File(foo, "changed.txt").apply { writeText("changed") } - val snapshotProvider = SimpleFileSnapshotProviderImpl() - val diff1 = snapshotMap.compareAndUpdate(src.filesWithExt("txt"), snapshotProvider) + val diff1 = snapshotMap.compareAndUpdate(src.filesWithExt("txt")) assertArrayEquals("diff1.removed", diff1.removed.toSortedPaths(), emptyArray()) assertArrayEquals("diff1.newOrModified", - diff1.newOrModified.toSortedPaths(), + diff1.modified.toSortedPaths(), listOf(removedTxt, unchangedTxt, changedTxt).toSortedPaths()) removedTxt.delete() @@ -50,12 +49,12 @@ class FileSnapshotMapTest : TestWithWorkingDir() { changedTxt.writeText("degnahc") val newTxt = File(foo, "new.txt").apply { writeText("new") } - val diff2 = snapshotMap.compareAndUpdate(src.filesWithExt("txt"), snapshotProvider) + val diff2 = snapshotMap.compareAndUpdate(src.filesWithExt("txt")) assertArrayEquals("diff2.removed", diff2.removed.toSortedPaths(), listOf(removedTxt).toSortedPaths()) assertArrayEquals("diff2.newOrModified", - diff2.newOrModified.toSortedPaths(), + diff2.modified.toSortedPaths(), listOf(newTxt, changedTxt).toSortedPaths()) } diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/incremental/testLogsParsingUtil.kt b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/incremental/testLogsParsingUtil.kt similarity index 98% rename from libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/incremental/testLogsParsingUtil.kt rename to libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/incremental/testLogsParsingUtil.kt index c5f20b97c11..9c00cd34ef4 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/incremental/testLogsParsingUtil.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/incremental/testLogsParsingUtil.kt @@ -83,6 +83,8 @@ fun parseTestBuildLog(file: File): List { } } +// used in integration tests +@Suppress("unused") fun dumpBuildLog(buildSteps: Iterable): String { val sb = StringBuilder()