diff --git a/jps/jps-plugin/jps-tests/jps-tests.iml b/jps/jps-plugin/jps-tests/jps-tests.iml index b4cb296a6a6..6debe8f5988 100644 --- a/jps/jps-plugin/jps-tests/jps-tests.iml +++ b/jps/jps-plugin/jps-tests/jps-tests.iml @@ -24,5 +24,6 @@ + \ No newline at end of file diff --git a/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt b/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt index f3d2effd435..d173af166f7 100644 --- a/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt +++ b/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt @@ -74,6 +74,7 @@ abstract class AbstractIncrementalJpsTest( protected lateinit var testDataDir: File protected lateinit var workDir: File protected lateinit var projectDescriptor: ProjectDescriptor + // is used to compare lookup dumps in a human readable way (lookup symbols are hashed in an actual lookup storage) protected lateinit var lookupsDuringTest: MutableSet private var isICEnabledBackup: Boolean = false @@ -132,17 +133,12 @@ abstract class AbstractIncrementalJpsTest( protected open val mockConstantSearch: Callbacks.ConstantAffectionResolver? get() = null - private fun createLookupTracker(): TestLookupTracker = TestLookupTracker() - - protected open fun checkLookups(@Suppress("UNUSED_PARAMETER") lookupTracker: LookupTracker, compiledFiles: Set) { - } - - private fun build(scope: CompileScopeTestBuilder = CompileScopeTestBuilder.make().allModules(), checkLookups: Boolean = true): MakeResult { + private fun build(scope: CompileScopeTestBuilder = CompileScopeTestBuilder.make().allModules()): MakeResult { val workDirPath = FileUtil.toSystemIndependentName(workDir.absolutePath) val logger = MyLogger(workDirPath) projectDescriptor = createProjectDescriptor(BuildLoggingManager(logger)) - val lookupTracker = createLookupTracker() + val lookupTracker = TestLookupTracker() projectDescriptor.project.setTestingContext(TestingContext(lookupTracker, logger)) try { @@ -151,12 +147,7 @@ abstract class AbstractIncrementalJpsTest( builder.addMessageHandler(buildResult) builder.build(scope.build(), false) - if (checkLookups) { - checkLookups(lookupTracker, logger.compiledFiles) - } - - val lookups = lookupTracker.lookups.map { LookupSymbol(it.name, it.scopeFqName) } - lookupsDuringTest.addAll(lookups) + lookupTracker.lookups.mapTo(lookupsDuringTest) { LookupSymbol(it.name, it.scopeFqName) } if (!buildResult.isSuccessful) { val errorMessages = @@ -196,7 +187,7 @@ abstract class AbstractIncrementalJpsTest( } private fun rebuild(): MakeResult { - return build(CompileScopeTestBuilder.rebuild().allModules(), checkLookups = false) + return build(CompileScopeTestBuilder.rebuild().allModules()) } private fun rebuildAndCheckOutput(makeOverallResult: MakeResult) { diff --git a/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt b/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt index 0040e9e183f..fca62b6dba2 100644 --- a/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt +++ b/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractLookupTrackerTest.kt @@ -17,40 +17,178 @@ package org.jetbrains.kotlin.jps.build import com.intellij.openapi.util.io.FileUtil +import com.intellij.testFramework.UsefulTestCase +import com.intellij.util.containers.HashMap import com.intellij.util.containers.StringInterner +import org.jetbrains.kotlin.TestWithWorkingDir +import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler +import org.jetbrains.kotlin.compilerRunner.* +import org.jetbrains.kotlin.config.IncrementalCompilation +import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.incremental.components.LookupInfo import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.components.Position import org.jetbrains.kotlin.incremental.components.ScopeKind +import org.jetbrains.kotlin.incremental.isKotlinFile +import org.jetbrains.kotlin.incremental.makeModuleFile +import org.jetbrains.kotlin.incremental.testingUtils.TouchPolicy +import org.jetbrains.kotlin.incremental.testingUtils.copyTestSources +import org.jetbrains.kotlin.incremental.testingUtils.getModificationsToPerform +import org.jetbrains.kotlin.incremental.utils.TestMessageCollector +import org.jetbrains.kotlin.jps.incremental.createTestingCompilerEnvironment import org.jetbrains.kotlin.test.KotlinTestUtils -import java.io.File +import java.io.* import java.util.* -private val DECLARATION_KEYWORDS = listOf("interface", "class", "enum class", "object", "fun", "operator fun", "val", "var") -private val DECLARATION_STARTS_WITH = DECLARATION_KEYWORDS.map { it + " " } -abstract class AbstractLookupTrackerTest : AbstractIncrementalJpsTest( - allowNoFilesWithSuffixInTestData = true, - allowNoBuildLogFileInTestData = true -) { +abstract class AbstractLookupTrackerTest : TestWithWorkingDir() { + private val DECLARATION_KEYWORDS = listOf("interface", "class", "enum class", "object", "fun", "operator fun", "val", "var") + private val DECLARATION_STARTS_WITH = DECLARATION_KEYWORDS.map { it + " " } // ignore KDoc like comments which starts with `/**`, example: /** text */ - val COMMENT_WITH_LOOKUP_INFO = "/\\*[^*]+\\*/".toRegex() + private val COMMENT_WITH_LOOKUP_INFO = "/\\*[^*]+\\*/".toRegex() - override fun checkLookups(lookupTracker: LookupTracker, compiledFiles: Set) { - if (lookupTracker !is TestLookupTracker) throw AssertionError("Expected TestLookupTracker, but: ${lookupTracker::class.java}") + private fun removeCommentsCommentsWithLookupInfo(files: Iterable) { + files.forEach { + val content = it.readText() + it.writeText(content.replace(COMMENT_WITH_LOOKUP_INFO, "")) + } + } + protected lateinit var srcDir: File + protected lateinit var outDir: File + private var isICEnabledBackup: Boolean = false + + override fun setUp() { + super.setUp() + srcDir = File(workingDir, "src").apply { mkdirs() } + outDir = File(workingDir, "out") + isICEnabledBackup = IncrementalCompilation.isEnabled() + IncrementalCompilation.setIsEnabled(true) + } + + override fun tearDown() { + IncrementalCompilation.setIsEnabled(isICEnabledBackup) + super.tearDown() + } + + fun doTest(path: String) { + val sb = StringBuilder() + fun CompilerOutput.logOutput(stepName: String) { + sb.appendln("==== $stepName ====") + + sb.appendln("Compiling files:") + compiledFiles.map { it.toRelativeString(workingDir) }.sorted().forEach { sb.appendln(" " + it) } + + sb.appendln("Exit code: $exitCode") + errors.forEach { sb.appendln(" " + it) } + + sb.appendln() + } + + val testDir = File(path) + val workToOriginalFileMap = HashMap(copyTestSources(testDir, srcDir, filePrefix = "")) + var dirtyFiles = srcDir.walk().filterTo(HashSet()) { it.isKotlinFile() } + val incrementalData = IncrementalData() + val steps = getModificationsToPerform(testDir, moduleNames = null, allowNoFilesWithSuffixInTestData = true, touchPolicy = TouchPolicy.CHECKSUM) + .filter { it.isNotEmpty() } + + makeAndCheckLookups(dirtyFiles, workToOriginalFileMap, incrementalData).logOutput("INITIAL BUILD") + for ((i, modifications) in steps.withIndex()) { + dirtyFiles = modifications.mapNotNullTo(HashSet()) { it.perform(workingDir, workToOriginalFileMap) } + makeAndCheckLookups(dirtyFiles, workToOriginalFileMap, incrementalData).logOutput("STEP ${i + 1}") + } + + val expectedBuildLog = File(testDir, "build.log") + UsefulTestCase.assertSameLinesWithFile(expectedBuildLog.canonicalPath, sb.toString()) + } + + private class CompilerOutput( + val exitCode: String, + val errors: List, + val compiledFiles: Iterable + ) + private class IncrementalData(val sourceToOutput: MutableMap> = hashMapOf()) + + private fun makeAndCheckLookups( + filesToCompile: Iterable, + workingToOriginalFileMap: Map, + incrementalData: IncrementalData + ): CompilerOutput { + removeCommentsCommentsWithLookupInfo(filesToCompile) + + for (dirtyFile in filesToCompile) { + incrementalData.sourceToOutput.remove(dirtyFile)?.forEach { + it.delete() + } + } + + val lookupTracker = TestLookupTracker() + val messageCollector = TestMessageCollector() + val outputItemsCollector = OutputItemsCollectorImpl() + val services = Services.Builder().run { + register(LookupTracker::class.java, lookupTracker) + build() + } + val environment = createTestingCompilerEnvironment(messageCollector, outputItemsCollector, services) + val exitCode = runCompiler(filesToCompile, environment) + + checkLookups(filesToCompile, lookupTracker, workingToOriginalFileMap) + + for (output in outputItemsCollector.outputs) { + val outputFile = output.outputFile + if (outputFile.extension == "kotlin_module") continue + + for (sourceFile in output.sourceFiles) { + val outputsForSource = incrementalData.sourceToOutput.getOrPut(sourceFile) { hashSetOf() } + outputsForSource.add(outputFile) + } + } + + return CompilerOutput(exitCode.toString(), messageCollector.errors, filesToCompile) + } + + + private fun runCompiler(filesToCompile: Iterable, env: JpsCompilerEnvironment): Any? { + val module = makeModuleFile(name = "test", + isTest = true, + outputDir = outDir, + sourcesToCompile = filesToCompile.toList(), + javaSourceRoots = listOf(srcDir), + classpath = listOf(outDir).filter { it.exists() }, + friendDirs = emptyList()) + outDir.mkdirs() + val args = arrayOf("-module", module.canonicalPath, "-Xreport-output-files") + + try { + val stream = ByteArrayOutputStream() + val out = PrintStream(stream) + val exitCode = CompilerRunnerUtil.invokeExecMethod(K2JVMCompiler::class.java.name, args, env, out) + val reader = BufferedReader(StringReader(stream.toString())) + CompilerOutputParser.parseCompilerMessagesFromReader(env.messageCollector, reader, env.outputItemsCollector) + + return exitCode + } + finally { + module.delete() + } + } + + private fun checkLookups( + compiledFiles: Iterable, + lookupTracker: TestLookupTracker, + workingToOriginalFileMap: Map + ) { val fileToLookups = lookupTracker.lookups.groupBy { it.filePath } fun checkLookupsInFile(expectedFile: File, actualFile: File) { - val independentFilePath = FileUtil.toSystemIndependentName(actualFile.path) - val lookupsFromFile = fileToLookups[independentFilePath] ?: return + val lookupsFromFile = fileToLookups[independentFilePath] ?: error("No lookups from compiled file: $actualFile") val text = actualFile.readText() val matchResult = COMMENT_WITH_LOOKUP_INFO.find(text) if (matchResult != null) { - fail("File $actualFile unexpectedly contains multiline comments. In range ${matchResult.range} found: ${matchResult.value} in $text") + throw AssertionError("File $actualFile contains multiline comment in range ${matchResult.range}") } val lines = text.lines().toMutableList() @@ -95,25 +233,8 @@ abstract class AbstractLookupTrackerTest : AbstractIncrementalJpsTest( KotlinTestUtils.assertEqualsToFile(expectedFile, actual) } - for (actualFile in compiledFiles) { - val expectedFile = mapWorkingToOriginalFile[actualFile]!! - - checkLookupsInFile(expectedFile, actualFile) - } - } - - override fun preProcessSources(srcDir: File) = dropBlockComments(srcDir) - - private fun dropBlockComments(workSrcDir: File) { - for (file in workSrcDir.walkTopDown()) { - if (!file.isFile) continue - - val original = file.readText() - val modified = original.replace(COMMENT_WITH_LOOKUP_INFO, "") - - if (original != modified) { - file.writeText(modified) - } + for (file in compiledFiles) { + checkLookupsInFile(workingToOriginalFileMap[file]!!, file) } } } diff --git a/jps/jps-plugin/testData/incremental/lookupTracker/classifierMembers/build.log b/jps/jps-plugin/testData/incremental/lookupTracker/classifierMembers/build.log new file mode 100644 index 00000000000..27f99960b9e --- /dev/null +++ b/jps/jps-plugin/testData/incremental/lookupTracker/classifierMembers/build.log @@ -0,0 +1,11 @@ +==== INITIAL BUILD ==== +Compiling files: + src/bar.kt + src/constraints.kt + src/foo.kt + src/usages.kt +Exit code: COMPILATION_ERROR + Unresolved reference: vala + Unresolved reference: vara + Unresolved reference: foo + Unresolved reference: bar \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/lookupTracker/classifierMembers/init-build.log b/jps/jps-plugin/testData/incremental/lookupTracker/classifierMembers/init-build.log deleted file mode 100644 index 0b64c97b5c5..00000000000 --- a/jps/jps-plugin/testData/incremental/lookupTracker/classifierMembers/init-build.log +++ /dev/null @@ -1,13 +0,0 @@ -Compiling files: - src/bar.kt - src/constraints.kt - src/foo.kt - src/usages.kt -End of files -Exit code: ABORT ------------------------------------------- -COMPILATION FAILED -Unresolved reference: vala -Unresolved reference: vara -Unresolved reference: foo -Unresolved reference: bar diff --git a/jps/jps-plugin/testData/incremental/lookupTracker/conventions/build.log b/jps/jps-plugin/testData/incremental/lookupTracker/conventions/build.log new file mode 100644 index 00000000000..a03aebe0d5f --- /dev/null +++ b/jps/jps-plugin/testData/incremental/lookupTracker/conventions/build.log @@ -0,0 +1,16 @@ +==== INITIAL BUILD ==== +Compiling files: + src/comparison.kt + src/declarations.kt + src/delegateProperty.kt + src/mathematicalLike.kt + src/other.kt +Exit code: OK + +==== STEP 1 ==== +Compiling files: + src/comparison.kt + src/delegateProperty.kt + src/mathematicalLike.kt + src/other.kt +Exit code: OK \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/lookupTracker/expressionType/build.log b/jps/jps-plugin/testData/incremental/lookupTracker/expressionType/build.log new file mode 100644 index 00000000000..74d3d30ab54 --- /dev/null +++ b/jps/jps-plugin/testData/incremental/lookupTracker/expressionType/build.log @@ -0,0 +1,7 @@ +==== INITIAL BUILD ==== +Compiling files: + src/genericType.kt + src/inferredType.kt + src/lambdaParameterType.kt + src/localInnerClassType.kt +Exit code: OK \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/lookupTracker/java/build.log b/jps/jps-plugin/testData/incremental/lookupTracker/java/build.log new file mode 100644 index 00000000000..1c3639643d7 --- /dev/null +++ b/jps/jps-plugin/testData/incremental/lookupTracker/java/build.log @@ -0,0 +1,6 @@ +==== INITIAL BUILD ==== +Compiling files: + src/usages.kt +Exit code: COMPILATION_ERROR + Unresolved reference: IS + Unresolved reference: IS \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/lookupTracker/java/init-build.log b/jps/jps-plugin/testData/incremental/lookupTracker/java/init-build.log deleted file mode 100644 index 2392cb36ba3..00000000000 --- a/jps/jps-plugin/testData/incremental/lookupTracker/java/init-build.log +++ /dev/null @@ -1,8 +0,0 @@ -Compiling files: - src/usages.kt -End of files -Exit code: ABORT ------------------------------------------- -COMPILATION FAILED -Unresolved reference: IS -Unresolved reference: IS diff --git a/jps/jps-plugin/testData/incremental/lookupTracker/localDeclarations/build.log b/jps/jps-plugin/testData/incremental/lookupTracker/localDeclarations/build.log new file mode 100644 index 00000000000..875fc020612 --- /dev/null +++ b/jps/jps-plugin/testData/incremental/lookupTracker/localDeclarations/build.log @@ -0,0 +1,5 @@ +==== INITIAL BUILD ==== +Compiling files: + src/bar.kt + src/locals.kt +Exit code: OK \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/lookupTracker/packageDeclarations/build.log b/jps/jps-plugin/testData/incremental/lookupTracker/packageDeclarations/build.log new file mode 100644 index 00000000000..6c6710fba2c --- /dev/null +++ b/jps/jps-plugin/testData/incremental/lookupTracker/packageDeclarations/build.log @@ -0,0 +1,13 @@ +==== INITIAL BUILD ==== +Compiling files: + src/bar.kt + src/baz.kt + src/foo1.kt + src/foo2.kt +Exit code: OK + +==== STEP 1 ==== +Compiling files: + src/foo1.kt + src/foo2.kt +Exit code: OK \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/lookupTracker/simple/build.log b/jps/jps-plugin/testData/incremental/lookupTracker/simple/build.log new file mode 100644 index 00000000000..5aaa84efb3d --- /dev/null +++ b/jps/jps-plugin/testData/incremental/lookupTracker/simple/build.log @@ -0,0 +1,4 @@ +==== INITIAL BUILD ==== +Compiling files: + src/main.kt +Exit code: OK \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/lookupTracker/syntheticProperties/build.log b/jps/jps-plugin/testData/incremental/lookupTracker/syntheticProperties/build.log new file mode 100644 index 00000000000..28132674392 --- /dev/null +++ b/jps/jps-plugin/testData/incremental/lookupTracker/syntheticProperties/build.log @@ -0,0 +1,13 @@ +==== INITIAL BUILD ==== +Compiling files: + src/KotlinClass.kt + src/usages.kt +Exit code: COMPILATION_ERROR + Unresolved reference: setFoo + Val cannot be reassigned + Unresolved reference: bazBaz + Unresolved reference: bazBaz + Unresolved reference: boo + Unresolved reference: bazBaz + Unresolved reference: bazBaz + Unresolved reference: boo \ No newline at end of file diff --git a/jps/jps-plugin/testData/incremental/lookupTracker/syntheticProperties/init-build.log b/jps/jps-plugin/testData/incremental/lookupTracker/syntheticProperties/init-build.log deleted file mode 100644 index d91bf1dcf21..00000000000 --- a/jps/jps-plugin/testData/incremental/lookupTracker/syntheticProperties/init-build.log +++ /dev/null @@ -1,15 +0,0 @@ -Compiling files: - src/KotlinClass.kt - src/usages.kt -End of files -Exit code: ABORT ------------------------------------------- -COMPILATION FAILED -Unresolved reference: setFoo -Val cannot be reassigned -Unresolved reference: bazBaz -Unresolved reference: bazBaz -Unresolved reference: boo -Unresolved reference: bazBaz -Unresolved reference: bazBaz -Unresolved reference: boo