diff --git a/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt.202 b/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt.202 new file mode 100644 index 00000000000..ae59ed0dab8 --- /dev/null +++ b/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/AbstractIncrementalJpsTest.kt.202 @@ -0,0 +1,660 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.jps.build + +import com.intellij.openapi.Disposable +import com.intellij.openapi.util.Disposer +import com.intellij.openapi.util.io.FileUtil +import com.intellij.openapi.util.io.FileUtilRt +import com.intellij.testFramework.TestLoggerFactory +import com.intellij.testFramework.UsefulTestCase +import junit.framework.TestCase +import org.apache.log4j.ConsoleAppender +import org.apache.log4j.Level +import org.apache.log4j.Logger +import org.apache.log4j.PatternLayout +import org.jetbrains.jps.ModuleChunk +import org.jetbrains.jps.api.CanceledStatus +import org.jetbrains.jps.builders.BuildResult +import org.jetbrains.jps.builders.CompileScopeTestBuilder +import org.jetbrains.jps.builders.impl.BuildDataPathsImpl +import org.jetbrains.jps.builders.impl.logging.ProjectBuilderLoggerBase +import org.jetbrains.jps.builders.java.dependencyView.Callbacks +import org.jetbrains.jps.builders.logging.BuildLoggingManager +import org.jetbrains.jps.cmdline.ProjectDescriptor +import org.jetbrains.jps.incremental.* +import org.jetbrains.jps.incremental.messages.BuildMessage +import org.jetbrains.jps.model.JpsDummyElement +import org.jetbrains.jps.model.JpsModuleRootModificationUtil +import org.jetbrains.jps.model.java.JpsJavaExtensionService +import org.jetbrains.jps.model.library.sdk.JpsSdk +import org.jetbrains.jps.util.JpsPathUtil +import org.jetbrains.kotlin.cli.common.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY +import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments +import org.jetbrains.kotlin.cli.common.arguments.K2MetadataCompilerArguments +import org.jetbrains.kotlin.incremental.LookupSymbol +import org.jetbrains.kotlin.incremental.testingUtils.* +import org.jetbrains.kotlin.jps.build.dependeciestxt.ModulesTxt +import org.jetbrains.kotlin.jps.build.dependeciestxt.ModulesTxtBuilder +import org.jetbrains.kotlin.jps.build.fixtures.EnableICFixture +import org.jetbrains.kotlin.jps.incremental.* +import org.jetbrains.kotlin.jps.model.JpsKotlinFacetModuleExtension +import org.jetbrains.kotlin.jps.model.kotlinFacet +import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget +import org.jetbrains.kotlin.platform.idePlatformKind +import org.jetbrains.kotlin.platform.impl.isJavaScript +import org.jetbrains.kotlin.platform.impl.isJvm +import org.jetbrains.kotlin.platform.orDefault +import org.jetbrains.kotlin.utils.Printer +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.io.File +import java.io.PrintStream +import java.util.* +import kotlin.reflect.jvm.javaField + +abstract class AbstractIncrementalJpsTest( + private val allowNoFilesWithSuffixInTestData: Boolean = false, + private val checkDumpsCaseInsensitively: Boolean = false +) : BaseKotlinJpsBuildTestCase() { + companion object { + private val COMPILATION_FAILED = "COMPILATION FAILED" + + // change to "/tmp" or anything when default is too long (for easier debugging) + private val TEMP_DIRECTORY_TO_USE = File(FileUtilRt.getTempDirectory()) + + private val DEBUG_LOGGING_ENABLED = System.getProperty("debug.logging.enabled") == "true" + } + + protected lateinit var testDataDir: File + protected lateinit var workDir: File + protected lateinit var projectDescriptor: ProjectDescriptor + // is used to compare lookup dumps in a human readable way (lookup symbols are hashed in an actual lookup storage) + protected lateinit var lookupsDuringTest: MutableSet + private var isJvmICEnabledBackup: Boolean = false + private var isJsICEnabledBackup: Boolean = false + + protected var mapWorkingToOriginalFile: MutableMap = hashMapOf() + + lateinit var kotlinCompileContext: KotlinCompileContext + + protected open val buildLogFinder: BuildLogFinder + get() = BuildLogFinder() + + private fun enableDebugLogging() { + com.intellij.openapi.diagnostic.Logger.setFactory(TestLoggerFactory::class.java) + TestLoggerFactory.dumpLogToStdout("") + TestLoggerFactory.enableDebugLogging(testRootDisposable, "#org") + + val console = ConsoleAppender() + console.layout = PatternLayout("%d [%p|%c|%C{1}] %m%n") + console.threshold = Level.ALL + console.activateOptions() + Logger.getRootLogger().addAppender(console) + } + + private var systemPropertiesBackup = run { + val props = System.getProperties() + val output = ByteArrayOutputStream() + props.store(output, "System properties backup") + output.toByteArray() + } + + private fun restoreSystemProperties() { + val input = ByteArrayInputStream(systemPropertiesBackup) + val props = Properties() + props.load(input) + System.setProperties(props) + } + + private val enableICFixture = EnableICFixture() + + override fun setUp() { + super.setUp() + + enableICFixture.setUp() + lookupsDuringTest = hashSetOf() + System.setProperty(KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY, "true") + + if (DEBUG_LOGGING_ENABLED) { + enableDebugLogging() + } + } + + override fun tearDown() { + restoreSystemProperties() + + (AbstractIncrementalJpsTest::myProject).javaField!![this] = null + (AbstractIncrementalJpsTest::projectDescriptor).javaField!![this] = null + (AbstractIncrementalJpsTest::systemPropertiesBackup).javaField!![this] = null + + lookupsDuringTest.clear() + enableICFixture.tearDown() + super.tearDown() + } + + // JPS forces rebuild of all files when JVM constant has been changed and Callbacks.ConstantAffectionResolver + // is not provided, so ConstantAffectionResolver is mocked with empty implementation + // Usages in Kotlin files are expected to be found by KotlinLookupConstantSearch + protected open val mockConstantSearch: Callbacks.ConstantAffectionResolver? + get() = MockJavaConstantSearch(workDir) + + private fun build( + name: String?, + scope: CompileScopeTestBuilder = CompileScopeTestBuilder.make().allModules() + ): MakeResult { + val workDirPath = FileUtil.toSystemIndependentName(workDir.absolutePath) + + val logger = MyLogger(workDirPath) + projectDescriptor = createProjectDescriptor(BuildLoggingManager(logger)) + + val lookupTracker = TestLookupTracker() + val testingContext = TestingContext(lookupTracker, logger) + projectDescriptor.project.setTestingContext(testingContext) + + try { + val builder = IncProjectBuilder( + projectDescriptor, + BuilderRegistry.getInstance(), + myBuildParams, + CanceledStatus.NULL, + true + ) + val buildResult = BuildResult() + builder.addMessageHandler(buildResult) + val finalScope = scope.build() + + builder.build(finalScope, false) + + // testingContext.kotlinCompileContext is initialized in KotlinBuilder.initializeKotlinContext + kotlinCompileContext = testingContext.kotlinCompileContext!! + + lookupTracker.lookups.mapTo(lookupsDuringTest) { LookupSymbol(it.name, it.scopeFqName) } + + if (!buildResult.isSuccessful) { + val errorMessages = + buildResult + .getMessages(BuildMessage.Kind.ERROR) + .map { it.messageText } + .map { it.replace("^.+:\\d+:\\s+".toRegex(), "").trim() } + .joinToString("\n") + return MakeResult( + log = logger.log + "$COMPILATION_FAILED\n" + errorMessages + "\n", + makeFailed = true, + mappingsDump = null, + name = name + ) + } else { + return MakeResult( + log = logger.log, + makeFailed = false, + mappingsDump = createMappingsDump(projectDescriptor, kotlinCompileContext, lookupsDuringTest), + name = name + ) + } + } finally { + projectDescriptor.dataManager.flush(false) + projectDescriptor.release() + } + } + + private fun initialMake(): MakeResult { + val makeResult = build(null) + + val initBuildLogFile = File(testDataDir, "init-build.log") + if (initBuildLogFile.exists()) { + UsefulTestCase.assertSameLinesWithFile(initBuildLogFile.absolutePath, makeResult.log) + } else { + assertFalse("Initial make failed:\n$makeResult", makeResult.makeFailed) + } + + return makeResult + } + + private fun make(name: String?): MakeResult { + return build(name) + } + + private fun rebuild(): MakeResult { + return build(null, CompileScopeTestBuilder.rebuild().allModules()) + } + + private fun rebuildAndCheckOutput(makeOverallResult: MakeResult) { + val outDir = File(getAbsolutePath("out")) + val outAfterMake = File(getAbsolutePath("out-after-make")) + + if (outDir.exists()) { + FileUtil.copyDir(outDir, outAfterMake) + } + + val rebuildResult = rebuild() + assertEquals( + "Rebuild failed: ${rebuildResult.makeFailed}, last make failed: ${makeOverallResult.makeFailed}. Rebuild result: $rebuildResult", + rebuildResult.makeFailed, makeOverallResult.makeFailed + ) + + if (!outAfterMake.exists()) { + assertFalse(outDir.exists()) + } else { + assertEqualDirectories(outDir, outAfterMake, makeOverallResult.makeFailed) + } + + if (!makeOverallResult.makeFailed) { + if (checkDumpsCaseInsensitively && rebuildResult.mappingsDump?.toLowerCase() == makeOverallResult.mappingsDump?.toLowerCase()) { + // do nothing + } else { + TestCase.assertEquals(rebuildResult.mappingsDump, makeOverallResult.mappingsDump) + } + } + + FileUtil.delete(outAfterMake) + } + + private fun clearCachesRebuildAndCheckOutput(makeOverallResult: MakeResult) { + FileUtil.delete(BuildDataPathsImpl(myDataStorageRoot).dataStorageRoot!!) + + rebuildAndCheckOutput(makeOverallResult) + } + + open val modulesTxtFile + get() = File(testDataDir, "dependencies.txt") + + private fun readModulesTxt(): ModulesTxt? { + var actualModulesTxtFile = modulesTxtFile + + if (!actualModulesTxtFile.exists()) { + // also try `"_${fileName}.txt"`. Useful for sorting files in IDE. + actualModulesTxtFile = modulesTxtFile.parentFile.resolve("_" + modulesTxtFile.name) + if (!actualModulesTxtFile.exists()) return null + } + + return ModulesTxtBuilder().readFile(actualModulesTxtFile) + } + + protected open fun createBuildLog(incrementalMakeResults: List): String = + buildString { + incrementalMakeResults.forEachIndexed { i, makeResult -> + if (i > 0) append("\n") + if (makeResult.name != null) { + append("================ Step #${i + 1} ${makeResult.name} =================\n\n") + } else { + append("================ Step #${i + 1} =================\n\n") + } + append(makeResult.log) + } + } + + protected open fun doTest(testDataPath: String) { + testDataDir = File(testDataPath) + workDir = FileUtilRt.createTempDirectory(TEMP_DIRECTORY_TO_USE, "jps-build", null) + val buildLogFile = buildLogFinder.findBuildLog(testDataDir) + Disposer.register(testRootDisposable, Disposable { FileUtilRt.delete(workDir) }) + + val modulesTxt = configureModules() + if (modulesTxt?.muted == true) return + + initialMake() + + val otherMakeResults = performModificationsAndMake( + modulesTxt?.modules?.map { it.name }, + hasBuildLog = buildLogFile != null + ) + + buildLogFile?.let { + val logs = createBuildLog(otherMakeResults) + UsefulTestCase.assertSameLinesWithFile(buildLogFile.absolutePath, logs) + + val lastMakeResult = otherMakeResults.last() + clearCachesRebuildAndCheckOutput(lastMakeResult) + } + } + + protected data class MakeResult( + val log: String, + val makeFailed: Boolean, + val mappingsDump: String?, + val name: String? = null + ) + + open val testDataSrc: File + get() = testDataDir + + private fun performModificationsAndMake( + moduleNames: Collection?, + hasBuildLog: Boolean + ): List { + val results = arrayListOf() + val modifications = getModificationsToPerform( + testDataSrc, + moduleNames, + allowNoFilesWithSuffixInTestData = allowNoFilesWithSuffixInTestData || !hasBuildLog, + touchPolicy = TouchPolicy.TIMESTAMP + ) + + if (!hasBuildLog) { + check(modifications.size == 1 && modifications.single().isEmpty()) { + "Bad test data: build steps are provided, but there is no `build.log` file" + } + return results + } + + val stepsTxt = File(testDataSrc, "_steps.txt") + val modificationNames = if (stepsTxt.exists()) stepsTxt.readLines() else null + + modifications.forEachIndexed { index, step -> + step.forEach { it.perform(workDir, mapWorkingToOriginalFile) } + performAdditionalModifications(step) + if (moduleNames == null) { + preProcessSources(File(workDir, "src")) + } else { + moduleNames.forEach { preProcessSources(File(workDir, "$it/src")) } + } + + val name = modificationNames?.getOrNull(index) + val makeResult = make(name) + results.add(makeResult) + } + return results + } + + protected open fun performAdditionalModifications(modifications: List) { + } + + protected open fun generateModuleSources(modulesTxt: ModulesTxt) = Unit + + // null means one module + private fun configureModules(): ModulesTxt? { + JpsJavaExtensionService.getInstance().getOrCreateProjectExtension(myProject).outputUrl = + JpsPathUtil.pathToUrl(getAbsolutePath("out")) + + val jdk = addJdk("my jdk") + val modulesTxt = readModulesTxt() + mapWorkingToOriginalFile = hashMapOf() + + if (modulesTxt == null) configureSingleModuleProject(jdk) + else configureMultiModuleProject(modulesTxt, jdk) + + overrideModuleSettings() + configureRequiredLibraries() + + return modulesTxt + } + + open fun overrideModuleSettings() { + } + + private fun configureSingleModuleProject(jdk: JpsSdk?) { + addModule("module", arrayOf(getAbsolutePath("src")), null, null, jdk) + + val sourceDestinationDir = File(workDir, "src") + val sourcesMapping = copyTestSources(testDataDir, File(workDir, "src"), "") + mapWorkingToOriginalFile.putAll(sourcesMapping) + + preProcessSources(sourceDestinationDir) + } + + protected open val ModulesTxt.Module.sourceFilePrefix: String + get() = "${name}_" + + private fun configureMultiModuleProject( + modulesTxt: ModulesTxt, + jdk: JpsSdk? + ) { + modulesTxt.modules.forEach { module -> + module.jpsModule = addModule( + module.name, + arrayOf(getAbsolutePath("${module.name}/src")), + null, + null, + jdk + )!! + + val kotlinFacetSettings = module.kotlinFacetSettings + if (kotlinFacetSettings != null) { + val compilerArguments = kotlinFacetSettings.compilerArguments + if (compilerArguments is K2MetadataCompilerArguments) { + val out = getAbsolutePath("${module.name}/out") + File(out).mkdirs() + compilerArguments.destination = out + } else if (compilerArguments is K2JVMCompilerArguments) { + compilerArguments.disableDefaultScriptingPlugin = true + } + + module.jpsModule.container.setChild( + JpsKotlinFacetModuleExtension.KIND, + JpsKotlinFacetModuleExtension(kotlinFacetSettings) + ) + } + } + + modulesTxt.dependencies.forEach { + JpsModuleRootModificationUtil.addDependency( + it.from.jpsModule, it.to.jpsModule, + it.scope, it.exported + ) + } + + // configure module contents + generateModuleSources(modulesTxt) + modulesTxt.modules.forEach { module -> + val sourceDirName = "${module.name}/src" + val sourceDestinationDir = File(workDir, sourceDirName) + val sourcesMapping = copyTestSources(testDataSrc, sourceDestinationDir, module.sourceFilePrefix) + mapWorkingToOriginalFile.putAll(sourcesMapping) + + preProcessSources(sourceDestinationDir) + } + } + + private fun configureRequiredLibraries() { + myProject.modules.forEach { module -> + val platformKind = module.kotlinFacet?.settings?.targetPlatform?.idePlatformKind.orDefault() + + when { + platformKind.isJvm -> { + JpsModuleRootModificationUtil.addDependency(module, requireLibrary(KotlinJpsLibrary.JvmStdLib)) + JpsModuleRootModificationUtil.addDependency(module, requireLibrary(KotlinJpsLibrary.JvmTest)) + } + platformKind.isJavaScript -> { + JpsModuleRootModificationUtil.addDependency(module, requireLibrary(KotlinJpsLibrary.JsStdLib)) + JpsModuleRootModificationUtil.addDependency(module, requireLibrary(KotlinJpsLibrary.JsTest)) + } + } + } + } + + protected open fun preProcessSources(srcDir: File) { + } + + override fun doGetProjectDir(): File? = workDir + + internal class MyLogger(val rootPath: String) : ProjectBuilderLoggerBase(), TestingBuildLogger { + private val markedDirtyBeforeRound = ArrayList() + private val markedDirtyAfterRound = ArrayList() + private val customMessages = mutableListOf() + + override fun invalidOrUnusedCache( + chunk: KotlinChunk?, + target: KotlinModuleBuildTarget<*>?, + attributesDiff: CacheAttributesDiff<*> + ) { + val cacheManager = attributesDiff.manager + val cacheTitle = when (cacheManager) { + is CacheVersionManager -> "Local cache for ${chunk ?: target}" + is CompositeLookupsCacheAttributesManager -> "Lookups cache" + else -> error("Unknown cache manager $cacheManager") + } + + logLine("$cacheTitle are ${attributesDiff.status}") + } + + override fun markedAsDirtyBeforeRound(files: Iterable) { + markedDirtyBeforeRound.addAll(files) + } + + override fun markedAsDirtyAfterRound(files: Iterable) { + markedDirtyAfterRound.addAll(files) + } + + override fun chunkBuildStarted(context: CompileContext, chunk: ModuleChunk) { + logDirtyFiles(markedDirtyBeforeRound) // files can be marked as dirty during build start (KotlinCompileContext initialization) + + if (!chunk.isDummy(context) && context.projectDescriptor.project.modules.size > 1) { + logLine("Building ${chunk.modules.sortedBy { it.name }.joinToString { it.name }}") + } + } + + override fun afterChunkBuildStarted(context: CompileContext, chunk: ModuleChunk) { + logDirtyFiles(markedDirtyBeforeRound) + } + + override fun addCustomMessage(message: String) { + customMessages.add(message) + } + + override fun buildFinished(exitCode: ModuleLevelBuilder.ExitCode) { + customMessages.forEach { + logLine(it) + } + customMessages.clear() + logDirtyFiles(markedDirtyAfterRound) + logLine("Exit code: $exitCode") + logLine("------------------------------------------") + } + + private fun logDirtyFiles(files: MutableList) { + if (files.isEmpty()) return + + logLine("Marked as dirty by Kotlin:") + files.apply { + map { FileUtil.toSystemIndependentName(it.path) } + .sorted() + .forEach { logLine(it) } + + clear() + } + } + + private val logBuf = StringBuilder() + val log: String + get() = logBuf.toString() + + val compiledFiles = hashSetOf() + + override fun isEnabled(): Boolean = true + + override fun logCompiledFiles(files: MutableCollection?, builderName: String?, description: String?) { + super.logCompiledFiles(files, builderName, description) + + if (builderName == KotlinBuilder.KOTLIN_BUILDER_NAME) { + compiledFiles.addAll(files!!) + } + } + + override fun logLine(message: String?) { + logBuf.append(message!!.replace("^$rootPath/".toRegex(), " ")).append('\n') + } + } +} + +private fun createMappingsDump( + project: ProjectDescriptor, + kotlinContext: KotlinCompileContext, + lookupsDuringTest: Set +) = createKotlinCachesDump(project, kotlinContext, lookupsDuringTest) + "\n\n\n" + + createCommonMappingsDump(project) + "\n\n\n" + + createJavaMappingsDump(project) + +internal fun createKotlinCachesDump( + project: ProjectDescriptor, + kotlinContext: KotlinCompileContext, + lookupsDuringTest: Set +) = createKotlinIncrementalCacheDump(project, kotlinContext) + "\n\n\n" + + createLookupCacheDump(kotlinContext, lookupsDuringTest) + +private fun createKotlinIncrementalCacheDump( + project: ProjectDescriptor, + kotlinContext: KotlinCompileContext +): String { + return buildString { + for (target in project.allModuleTargets.sortedBy { it.presentableName }) { + val kotlinCache = project.dataManager.getKotlinCache(kotlinContext.targetsBinding[target]) + if (kotlinCache != null) { + append("\n") + append(kotlinCache.dump()) + append("\n\n\n") + } + } + } +} + +private fun createLookupCacheDump(kotlinContext: KotlinCompileContext, lookupsDuringTest: Set): String { + val sb = StringBuilder() + val p = Printer(sb) + p.println("Begin of Lookup Maps") + p.println() + + kotlinContext.lookupStorageManager.withLookupStorage { lookupStorage -> + lookupStorage.forceGC() + p.print(lookupStorage.dump(lookupsDuringTest)) + } + + p.println() + p.println("End of Lookup Maps") + return sb.toString() +} + +private fun createCommonMappingsDump(project: ProjectDescriptor): String { + val resultBuf = StringBuilder() + val result = Printer(resultBuf) + + result.println("Begin of SourceToOutputMap") + result.pushIndent() + + for (target in project.allModuleTargets) { + result.println(target) + result.pushIndent() + + val mapping = project.dataManager.getSourceToOutputMap(target) + mapping.sources.sorted().forEach { + val outputs = mapping.getOutputs(it)!!.sorted() + if (outputs.isNotEmpty()) { + result.println("source $it -> $outputs") + } + } + + result.popIndent() + } + + result.popIndent() + result.println("End of SourceToOutputMap") + + return resultBuf.toString() +} + +private fun createJavaMappingsDump(project: ProjectDescriptor): String { + val byteArrayOutputStream = ByteArrayOutputStream() + PrintStream(byteArrayOutputStream).use { + project.dataManager.mappings.toStream(it) + } + return byteArrayOutputStream.toString() +} + +internal val ProjectDescriptor.allModuleTargets: Collection + get() = buildTargetIndex.allTargets.filterIsInstance() + +private val EXPORTED_SUFFIX = "[exported]" diff --git a/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/KotlinJpsBuildTest.kt.202 b/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/KotlinJpsBuildTest.kt.202 new file mode 100644 index 00000000000..125ffde238d --- /dev/null +++ b/jps/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/build/KotlinJpsBuildTest.kt.202 @@ -0,0 +1,1131 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.jps.build + +import com.google.common.collect.Lists +import com.intellij.openapi.util.io.FileUtil +import com.intellij.openapi.util.io.FileUtil.toSystemIndependentName +import com.intellij.openapi.util.io.FileUtilRt +import com.intellij.openapi.util.text.StringUtil +import com.intellij.openapi.vfs.StandardFileSystems +import com.intellij.testFramework.LightVirtualFile +import com.intellij.testFramework.UsefulTestCase +import com.intellij.util.io.URLUtil +import com.intellij.util.io.ZipUtil +import org.jetbrains.jps.ModuleChunk +import org.jetbrains.jps.api.CanceledStatus +import org.jetbrains.jps.builders.BuildResult +import org.jetbrains.jps.builders.CompileScopeTestBuilder +import org.jetbrains.jps.builders.JpsBuildTestCase +import org.jetbrains.jps.builders.TestProjectBuilderLogger +import org.jetbrains.jps.builders.impl.BuildDataPathsImpl +import org.jetbrains.jps.builders.logging.BuildLoggingManager +import org.jetbrains.jps.cmdline.ProjectDescriptor +import org.jetbrains.jps.devkit.model.JpsPluginModuleType +import org.jetbrains.jps.incremental.BuilderRegistry +import org.jetbrains.jps.incremental.CompileContext +import org.jetbrains.jps.incremental.IncProjectBuilder +import org.jetbrains.jps.incremental.ModuleLevelBuilder +import org.jetbrains.jps.incremental.messages.BuildMessage +import org.jetbrains.jps.incremental.messages.CompilerMessage +import org.jetbrains.jps.model.JpsModuleRootModificationUtil +import org.jetbrains.jps.model.JpsProject +import org.jetbrains.jps.model.java.JavaSourceRootType +import org.jetbrains.jps.model.java.JpsJavaDependencyScope +import org.jetbrains.jps.model.java.JpsJavaExtensionService +import org.jetbrains.jps.model.java.JpsJavaSdkType +import org.jetbrains.jps.model.library.JpsOrderRootType +import org.jetbrains.jps.model.module.JpsModule +import org.jetbrains.jps.util.JpsPathUtil +import org.jetbrains.kotlin.cli.common.Usage +import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments +import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler +import org.jetbrains.kotlin.codegen.AsmUtil +import org.jetbrains.kotlin.codegen.JvmCodegenUtil +import org.jetbrains.kotlin.config.IncrementalCompilation +import org.jetbrains.kotlin.config.KotlinCompilerVersion.TEST_IS_PRE_RELEASE_SYSTEM_PROPERTY +import org.jetbrains.kotlin.config.LanguageVersion +import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.incremental.withIC +import org.jetbrains.kotlin.jps.build.KotlinJpsBuildTestBase.LibraryDependency.* +import org.jetbrains.kotlin.jps.incremental.CacheAttributesDiff +import org.jetbrains.kotlin.jps.model.kotlinCommonCompilerArguments +import org.jetbrains.kotlin.jps.model.kotlinCompilerArguments +import org.jetbrains.kotlin.jps.targets.KotlinModuleBuildTarget +import org.jetbrains.kotlin.load.kotlin.PackagePartClassUtils +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.test.KotlinTestUtils +import org.jetbrains.kotlin.test.MockLibraryUtil +import org.jetbrains.kotlin.utils.PathUtil +import org.jetbrains.kotlin.utils.Printer +import org.jetbrains.org.objectweb.asm.ClassReader +import org.jetbrains.org.objectweb.asm.ClassVisitor +import org.jetbrains.org.objectweb.asm.MethodVisitor +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.Assert +import java.io.File +import java.io.FileNotFoundException +import java.io.FileOutputStream +import java.io.IOException +import java.net.URLClassLoader +import java.util.* +import java.util.zip.ZipOutputStream + +open class KotlinJpsBuildTest : KotlinJpsBuildTestBase() { + companion object { + private val ADDITIONAL_MODULE_NAME = "module2" + + private val EXCLUDE_FILES = arrayOf("Excluded.class", "YetAnotherExcluded.class") + private val NOTHING = arrayOf() + private val KOTLIN_JS_LIBRARY = "jslib-example" + private val PATH_TO_KOTLIN_JS_LIBRARY = AbstractKotlinJpsBuildTestCase.TEST_DATA_PATH + "general/KotlinJavaScriptProjectWithDirectoryAsLibrary/" + KOTLIN_JS_LIBRARY + private val KOTLIN_JS_LIBRARY_JAR = "$KOTLIN_JS_LIBRARY.jar" + + private fun getMethodsOfClass(classFile: File): Set { + val result = TreeSet() + ClassReader(FileUtil.loadFileBytes(classFile)).accept(object : ClassVisitor(Opcodes.API_VERSION) { + override fun visitMethod(access: Int, name: String, desc: String, signature: String?, exceptions: Array?): MethodVisitor? { + result.add(name) + return null + } + }, 0) + return result + } + + @JvmStatic + protected fun klass(moduleName: String, classFqName: String): String { + val outputDirPrefix = "out/production/$moduleName/" + return outputDirPrefix + classFqName.replace('.', '/') + ".class" + } + + @JvmStatic + protected fun module(moduleName: String): String { + return "out/production/$moduleName/${JvmCodegenUtil.getMappingFileName(moduleName)}" + } + } + + fun doTest() { + initProject(JVM_MOCK_RUNTIME) + buildAllModules().assertSuccessful() + } + + fun doTestWithRuntime() { + initProject(JVM_FULL_RUNTIME) + buildAllModules().assertSuccessful() + } + + fun doTestWithKotlinJavaScriptLibrary() { + initProject(JS_STDLIB) + createKotlinJavaScriptLibraryArchive() + addDependency(KOTLIN_JS_LIBRARY, File(workDir, KOTLIN_JS_LIBRARY_JAR)) + buildAllModules().assertSuccessful() + } + + fun testKotlinProject() { + doTest() + + checkWhen(touch("src/test1.kt"), null, packageClasses("kotlinProject", "src/test1.kt", "Test1Kt")) + } + + fun testSourcePackagePrefix() { + doTest() + } + + fun testSourcePackageLongPrefix() { + initProject(JVM_MOCK_RUNTIME) + val buildResult = buildAllModules() + buildResult.assertSuccessful() + val warnings = buildResult.getMessages(BuildMessage.Kind.WARNING) + assertEquals("Warning about invalid package prefix in module 2 is expected: $warnings", 1, warnings.size) + assertEquals("Invalid package prefix name is ignored: invalid-prefix.test", warnings.first().messageText) + } + + fun testSourcePackagePrefixWithInnerClasses() { + initProject(JVM_MOCK_RUNTIME) + buildAllModules().assertSuccessful() + } + + fun testKotlinJavaScriptProject() { + initProject(JS_STDLIB) + buildAllModules().assertSuccessful() + + checkOutputFilesList() + checkWhen(touch("src/test1.kt"), null, pathsToDelete = k2jsOutput(PROJECT_NAME)) + } + + private fun k2jsOutput(vararg moduleNames: String): Array { + val moduleNamesSet = moduleNames.toSet() + val list = mutableListOf() + + myProject.modules.forEach { + if (it.name in moduleNamesSet) { + val outputDir = it.productionBuildTarget.outputDir!! + list.add(toSystemIndependentName(File("$outputDir/${it.name}.js").relativeTo(workDir).path)) + list.add(toSystemIndependentName(File("$outputDir/${it.name}.meta.js").relativeTo(workDir).path)) + + val kjsmFiles = outputDir.walk() + .filter { it.isFile && it.extension.equals("kjsm", ignoreCase = true) } + + list.addAll(kjsmFiles.map { toSystemIndependentName(it.relativeTo(workDir).path) }) + } + } + + return list.toTypedArray() + } + + fun testKotlinJavaScriptProjectNewSourceRootTypes() { + initProject(JS_STDLIB) + buildAllModules().assertSuccessful() + + checkOutputFilesList() + } + + fun testKotlinJavaScriptProjectWithCustomOutputPaths() { + initProject(JS_STDLIB) + buildAllModules().assertSuccessful() + + checkOutputFilesList(File(workDir, "target")) + } + + fun testKotlinJavaScriptProjectWithSourceMap() { + initProject(JS_STDLIB) + buildAllModules().assertSuccessful() + + val sourceMapContent = File(getOutputDir(PROJECT_NAME), "$PROJECT_NAME.js.map").readText() + val expectedPath = "prefix-dir/src/pkg/test1.kt" + assertTrue("Source map file should contain relative path ($expectedPath)", sourceMapContent.contains("\"$expectedPath\"")) + + val librarySourceMapFile = File(getOutputDir(PROJECT_NAME), "lib/kotlin.js.map") + assertTrue("Source map for stdlib should be copied to $librarySourceMapFile", librarySourceMapFile.exists()) + } + + fun testKotlinJavaScriptProjectWithSourceMapRelativePaths() { + initProject(JS_STDLIB) + buildAllModules().assertSuccessful() + + val sourceMapContent = File(getOutputDir(PROJECT_NAME), "$PROJECT_NAME.js.map").readText() + val expectedPath = "../../../src/pkg/test1.kt" + assertTrue("Source map file should contain relative path ($expectedPath)", sourceMapContent.contains("\"$expectedPath\"")) + + val librarySourceMapFile = File(getOutputDir(PROJECT_NAME), "lib/kotlin.js.map") + assertTrue("Source map for stdlib should be copied to $librarySourceMapFile", librarySourceMapFile.exists()) + } + + fun testKotlinJavaScriptProjectWithTwoModules() { + initProject(JS_STDLIB) + buildAllModules().assertSuccessful() + + checkOutputFilesList() + checkWhen(touch("src/test1.kt"), null, k2jsOutput(PROJECT_NAME)) + checkWhen(touch("module2/src/module2.kt"), null, k2jsOutput(ADDITIONAL_MODULE_NAME)) + checkWhen(arrayOf(touch("src/test1.kt"), touch("module2/src/module2.kt")), null, k2jsOutput(PROJECT_NAME, ADDITIONAL_MODULE_NAME)) + } + + @WorkingDir("KotlinJavaScriptProjectWithTwoModules") + fun testKotlinJavaScriptProjectWithTwoModulesAndWithLibrary() { + initProject() + createKotlinJavaScriptLibraryArchive() + addDependency(KOTLIN_JS_LIBRARY, File(workDir, KOTLIN_JS_LIBRARY_JAR)) + addKotlinJavaScriptStdlibDependency() + buildAllModules().assertSuccessful() + } + + fun testKotlinJavaScriptProjectWithDirectoryAsStdlib() { + initProject() + val jslibJar = PathUtil.kotlinPathsForDistDirectory.jsStdLibJarPath + val jslibDir = File(workDir, "KotlinJavaScript") + try { + ZipUtil.extract(jslibJar, jslibDir, null) + } + catch (ex: IOException) { + throw IllegalStateException(ex.message) + } + + addDependency("KotlinJavaScript", jslibDir) + buildAllModules().assertSuccessful() + + checkOutputFilesList() + checkWhen(touch("src/test1.kt"), null, k2jsOutput(PROJECT_NAME)) + } + + fun testKotlinJavaScriptProjectWithDirectoryAsLibrary() { + initProject(JS_STDLIB) + addDependency(KOTLIN_JS_LIBRARY, File(workDir, KOTLIN_JS_LIBRARY)) + buildAllModules().assertSuccessful() + + checkOutputFilesList() + checkWhen(touch("src/test1.kt"), null, k2jsOutput(PROJECT_NAME)) + } + + fun testKotlinJavaScriptProjectWithLibrary() { + doTestWithKotlinJavaScriptLibrary() + + checkOutputFilesList() + checkWhen(touch("src/test1.kt"), null, k2jsOutput(PROJECT_NAME)) + } + + fun testKotlinJavaScriptProjectWithLibraryCustomOutputDir() { + doTestWithKotlinJavaScriptLibrary() + + checkOutputFilesList() + checkWhen(touch("src/test1.kt"), null, k2jsOutput(PROJECT_NAME)) + } + + fun testKotlinJavaScriptProjectWithLibraryNoCopy() { + doTestWithKotlinJavaScriptLibrary() + + checkOutputFilesList() + checkWhen(touch("src/test1.kt"), null, k2jsOutput(PROJECT_NAME)) + } + + fun testKotlinJavaScriptProjectWithLibraryAndErrors() { + initProject(JS_STDLIB) + createKotlinJavaScriptLibraryArchive() + addDependency(KOTLIN_JS_LIBRARY, File(workDir, KOTLIN_JS_LIBRARY_JAR)) + buildAllModules().assertFailed() + + checkOutputFilesList() + } + + fun testKotlinJavaScriptProjectWithEmptyDependencies() { + initProject(JS_STDLIB) + buildAllModules().assertSuccessful() + } + + fun testKotlinJavaScriptInternalFromSpecialRelatedModule() { + initProject(JS_STDLIB) + buildAllModules().assertSuccessful() + } + + fun testKotlinJavaScriptProjectWithTests() { + initProject(JS_STDLIB) + buildAllModules().assertSuccessful() + } + + fun testKotlinJavaScriptProjectWithTestsAndSeparateTestAndSrcModuleDependencies() { + initProject(JS_STDLIB) + buildAllModules().assertSuccessful() + } + + fun testKotlinJavaScriptProjectWithTestsAndTestAndSrcModuleDependency() { + initProject(JS_STDLIB) + val buildResult = buildAllModules() + buildResult.assertSuccessful() + + val warnings = buildResult.getMessages(BuildMessage.Kind.WARNING) + assertEquals("Warning about duplicate module definition: $warnings", 0, warnings.size) + } + + fun testKotlinJavaScriptProjectWithTwoSrcModuleDependency() { + initProject(JS_STDLIB) + val buildResult = buildAllModules() + buildResult.assertSuccessful() + + val warnings = buildResult.getMessages(BuildMessage.Kind.WARNING) + assertEquals("Warning about duplicate module definition: $warnings", 0, warnings.size) + } + + fun testExcludeFolderInSourceRoot() { + doTest() + + val module = myProject.modules.get(0) + assertFilesExistInOutput(module, "Foo.class") + assertFilesNotExistInOutput(module, *EXCLUDE_FILES) + + checkWhen( + touch("src/foo.kt"), null, + arrayOf(klass("kotlinProject", "Foo"), module("kotlinProject")) + ) + } + + fun testExcludeModuleFolderInSourceRootOfAnotherModule() { + doTest() + + for (module in myProject.modules) { + assertFilesExistInOutput(module, "Foo.class") + } + + checkWhen( + touch("src/foo.kt"), null, + arrayOf(klass("kotlinProject", "Foo"), module("kotlinProject")) + ) + checkWhen( + touch("src/module2/src/foo.kt"), null, + arrayOf(klass("module2", "Foo"), module("module2")) + ) + } + + fun testExcludeFileUsingCompilerSettings() { + doTest() + + val module = myProject.modules.get(0) + assertFilesExistInOutput(module, "Foo.class", "Bar.class") + assertFilesNotExistInOutput(module, *EXCLUDE_FILES) + + if (IncrementalCompilation.isEnabledForJvm()) { + checkWhen(touch("src/foo.kt"), null, arrayOf(module("kotlinProject"), klass("kotlinProject", "Foo"))) + } + else { + val allClasses = myProject.outputPaths() + checkWhen(touch("src/foo.kt"), null, allClasses) + } + + checkWhen(touch("src/Excluded.kt"), null, NOTHING) + checkWhen(touch("src/dir/YetAnotherExcluded.kt"), null, NOTHING) + } + + fun testExcludeFolderNonRecursivelyUsingCompilerSettings() { + doTest() + + val module = myProject.modules.get(0) + assertFilesExistInOutput(module, "Foo.class", "Bar.class") + assertFilesNotExistInOutput(module, *EXCLUDE_FILES) + + if (IncrementalCompilation.isEnabledForJvm()) { + checkWhen(touch("src/foo.kt"), null, arrayOf(module("kotlinProject"), klass("kotlinProject", "Foo"))) + checkWhen(touch("src/dir/subdir/bar.kt"), null, arrayOf(module("kotlinProject"), klass("kotlinProject", "Bar"))) + } + else { + val allClasses = myProject.outputPaths() + checkWhen(touch("src/foo.kt"), null, allClasses) + checkWhen(touch("src/dir/subdir/bar.kt"), null, allClasses) + } + + checkWhen(touch("src/dir/Excluded.kt"), null, NOTHING) + checkWhen(touch("src/dir/subdir/YetAnotherExcluded.kt"), null, NOTHING) + } + + fun testExcludeFolderRecursivelyUsingCompilerSettings() { + doTest() + + val module = myProject.modules.get(0) + assertFilesExistInOutput(module, "Foo.class", "Bar.class") + assertFilesNotExistInOutput(module, *EXCLUDE_FILES) + + if (IncrementalCompilation.isEnabledForJvm()) { + checkWhen(touch("src/foo.kt"), null, arrayOf(module("kotlinProject"), klass("kotlinProject", "Foo"))) + } + else { + val allClasses = myProject.outputPaths() + checkWhen(touch("src/foo.kt"), null, allClasses) + } + + checkWhen(touch("src/exclude/Excluded.kt"), null, NOTHING) + checkWhen(touch("src/exclude/YetAnotherExcluded.kt"), null, NOTHING) + checkWhen(touch("src/exclude/subdir/Excluded.kt"), null, NOTHING) + checkWhen(touch("src/exclude/subdir/YetAnotherExcluded.kt"), null, NOTHING) + } + + fun testKotlinProjectTwoFilesInOnePackage() { + doTest() + + if (IncrementalCompilation.isEnabledForJvm()) { + checkWhen(touch("src/test1.kt"), null, packageClasses("kotlinProject", "src/test1.kt", "_DefaultPackage")) + checkWhen(touch("src/test2.kt"), null, packageClasses("kotlinProject", "src/test2.kt", "_DefaultPackage")) + } + else { + val allClasses = myProject.outputPaths() + checkWhen(touch("src/test1.kt"), null, allClasses) + checkWhen(touch("src/test2.kt"), null, allClasses) + } + + checkWhen(arrayOf(del("src/test1.kt"), del("src/test2.kt")), NOTHING, + arrayOf(packagePartClass("kotlinProject", "src/test1.kt", "_DefaultPackage"), + packagePartClass("kotlinProject", "src/test2.kt", "_DefaultPackage"), + module("kotlinProject"))) + + assertFilesNotExistInOutput(myProject.modules.get(0), "_DefaultPackage.class") + } + + fun testDefaultLanguageVersionCustomApiVersion() { + initProject(JVM_FULL_RUNTIME) + buildAllModules().assertFailed() + + assertEquals(1, myProject.modules.size) + val module = myProject.modules.first() + val args = module.kotlinCompilerArguments + args.apiVersion = "1.2" + myProject.kotlinCommonCompilerArguments = args + + buildAllModules().assertSuccessful() + } + + fun testKotlinJavaProject() { + doTestWithRuntime() + } + + fun testJKJProject() { + doTestWithRuntime() + } + + fun testKJKProject() { + doTestWithRuntime() + } + + fun testKJCircularProject() { + doTestWithRuntime() + } + + fun testJKJInheritanceProject() { + doTestWithRuntime() + } + + fun testKJKInheritanceProject() { + doTestWithRuntime() + } + + fun testCircularDependenciesNoKotlinFiles() { + doTest() + } + + fun testCircularDependenciesDifferentPackages() { + initProject(JVM_MOCK_RUNTIME) + val result = buildAllModules() + + // Check that outputs are located properly + assertFilesExistInOutput(findModule("module2"), "kt1/Kt1Kt.class") + assertFilesExistInOutput(findModule("kotlinProject"), "kt2/Kt2Kt.class") + + result.assertSuccessful() + + if (IncrementalCompilation.isEnabledForJvm()) { + checkWhen(touch("src/kt2.kt"), null, packageClasses("kotlinProject", "src/kt2.kt", "kt2.Kt2Kt")) + checkWhen(touch("module2/src/kt1.kt"), null, packageClasses("module2", "module2/src/kt1.kt", "kt1.Kt1Kt")) + } + else { + val allClasses = myProject.outputPaths() + checkWhen(touch("src/kt2.kt"), null, allClasses) + checkWhen(touch("module2/src/kt1.kt"), null, allClasses) + } + } + + fun testCircularDependenciesSamePackage() { + initProject(JVM_MOCK_RUNTIME) + val result = buildAllModules() + result.assertSuccessful() + + // Check that outputs are located properly + val facadeWithA = findFileInOutputDir(findModule("module1"), "test/AKt.class") + val facadeWithB = findFileInOutputDir(findModule("module2"), "test/BKt.class") + UsefulTestCase.assertSameElements(getMethodsOfClass(facadeWithA), "", "a", "getA") + UsefulTestCase.assertSameElements(getMethodsOfClass(facadeWithB), "", "b", "getB", "setB") + + + if (IncrementalCompilation.isEnabledForJvm()) { + checkWhen(touch("module1/src/a.kt"), null, packageClasses("module1", "module1/src/a.kt", "test.TestPackage")) + checkWhen(touch("module2/src/b.kt"), null, packageClasses("module2", "module2/src/b.kt", "test.TestPackage")) + } + else { + val allClasses = myProject.outputPaths() + checkWhen(touch("module1/src/a.kt"), null, allClasses) + checkWhen(touch("module2/src/b.kt"), null, allClasses) + } + } + + fun testCircularDependenciesSamePackageWithTests() { + initProject(JVM_MOCK_RUNTIME) + val result = buildAllModules() + result.assertSuccessful() + + // Check that outputs are located properly + val facadeWithA = findFileInOutputDir(findModule("module1"), "test/AKt.class") + val facadeWithB = findFileInOutputDir(findModule("module2"), "test/BKt.class") + UsefulTestCase.assertSameElements(getMethodsOfClass(facadeWithA), "", "a", "funA", "getA") + UsefulTestCase.assertSameElements(getMethodsOfClass(facadeWithB), "", "b", "funB", "getB", "setB") + + if (IncrementalCompilation.isEnabledForJvm()) { + checkWhen(touch("module1/src/a.kt"), null, packageClasses("module1", "module1/src/a.kt", "test.TestPackage")) + checkWhen(touch("module2/src/b.kt"), null, packageClasses("module2", "module2/src/b.kt", "test.TestPackage")) + } + else { + val allProductionClasses = myProject.outputPaths(tests = false) + checkWhen(touch("module1/src/a.kt"), null, allProductionClasses) + checkWhen(touch("module2/src/b.kt"), null, allProductionClasses) + } + } + + fun testInternalFromAnotherModule() { + initProject(JVM_MOCK_RUNTIME) + val result = buildAllModules() + result.assertFailed() + result.checkErrors() + } + + fun testInternalFromSpecialRelatedModule() { + initProject(JVM_MOCK_RUNTIME) + buildAllModules().assertSuccessful() + + val classpath = listOf("out/production/module1", "out/test/module2").map { File(workDir, it).toURI().toURL() }.toTypedArray() + val clazz = URLClassLoader(classpath).loadClass("test2.BarKt") + clazz.getMethod("box").invoke(null) + } + + fun testCircularDependenciesInternalFromAnotherModule() { + initProject(JVM_MOCK_RUNTIME) + val result = buildAllModules() + result.assertFailed() + result.checkErrors() + } + + fun testCircularDependenciesWrongInternalFromTests() { + initProject(JVM_MOCK_RUNTIME) + val result = buildAllModules() + result.assertFailed() + result.checkErrors() + } + + fun testCircularDependencyWithReferenceToOldVersionLib() { + initProject(JVM_MOCK_RUNTIME) + + val libraryJar = MockLibraryUtil.compileJvmLibraryToJar(workDir.absolutePath + File.separator + "oldModuleLib/src", "module-lib") + + AbstractKotlinJpsBuildTestCase.addDependency(JpsJavaDependencyScope.COMPILE, Lists.newArrayList(findModule("module1"), findModule("module2")), false, "module-lib", libraryJar) + + val result = buildAllModules() + result.assertSuccessful() + } + + fun testDependencyToOldKotlinLib() { + initProject() + + val libraryJar = MockLibraryUtil.compileJvmLibraryToJar(workDir.absolutePath + File.separator + "oldModuleLib/src", "module-lib") + + AbstractKotlinJpsBuildTestCase.addDependency(JpsJavaDependencyScope.COMPILE, Lists.newArrayList(findModule("module")), false, "module-lib", libraryJar) + + addKotlinStdlibDependency() + + val result = buildAllModules() + result.assertSuccessful() + } + + fun testDevKitProject() { + initProject(JVM_MOCK_RUNTIME) + val module = myProject.modules.single() + assertEquals(module.moduleType, JpsPluginModuleType.INSTANCE) + buildAllModules().assertSuccessful() + assertFilesExistInOutput(module, "TestKt.class") + } + + fun testAccessToInternalInProductionFromTests() { + initProject(JVM_MOCK_RUNTIME) + val result = buildAllModules() + result.assertSuccessful() + } + + /* + * Here we're checking that enabling inference in IDE doesn't affect compilation via JPS + * + * the following two tests are connected: + * - testKotlinProjectWithEnabledNewInferenceInIDE checks that project is compiled when new inference is enabled only in IDE + * - this is done via project component + * - testKotlinProjectWithErrorsBecauseOfNewInference checks that project isn't compiled when new inference is enabled in the compiler + * + * So, if the former will fail => option affects JPS compilation, it's bad. Also, if the latter test fails => test is useless as it's + * compiled with new and old inference. + * + */ + fun testKotlinProjectWithEnabledNewInferenceInIDE() { + initProject(JVM_MOCK_RUNTIME) + val module = myProject.modules.single() + val args = module.kotlinCompilerArguments + args.languageVersion = LanguageVersion.KOTLIN_1_3.versionString + myProject.kotlinCommonCompilerArguments = args + + buildAllModules().assertSuccessful() + } + + fun testKotlinProjectWithErrorsBecauseOfNewInference() { + initProject(JVM_MOCK_RUNTIME) + val module = myProject.modules.single() + val args = module.kotlinCompilerArguments + args.newInference = true + myProject.kotlinCommonCompilerArguments = args + + val result = buildAllModules() + result.assertFailed() + result.checkErrors() + } + + private fun createKotlinJavaScriptLibraryArchive() { + val jarFile = File(workDir, KOTLIN_JS_LIBRARY_JAR) + try { + val zip = ZipOutputStream(FileOutputStream(jarFile)) + ZipUtil.addDirToZipRecursively(zip, jarFile, File(PATH_TO_KOTLIN_JS_LIBRARY), "", null, null) + zip.close() + } + catch (ex: FileNotFoundException) { + throw IllegalStateException(ex.message) + } + catch (ex: IOException) { + throw IllegalStateException(ex.message) + } + + } + + protected fun checkOutputFilesList(outputDir: File = productionOutputDir) { + if (!expectedOutputFile.exists()) { + expectedOutputFile.writeText("") + throw IllegalStateException("$expectedOutputFile did not exist. Created empty file.") + } + + val sb = StringBuilder() + val p = Printer(sb, " ") + outputDir.printFilesRecursively(p) + + UsefulTestCase.assertSameLinesWithFile(expectedOutputFile.canonicalPath, sb.toString(), true) + } + + private fun File.printFilesRecursively(p: Printer) { + val files = listFiles() ?: return + + for (file in files.sortedBy { it.name }) { + when { + file.isFile -> { + p.println(file.name) + } + file.isDirectory -> { + p.println(file.name + "/") + p.pushIndent() + file.printFilesRecursively(p) + p.popIndent() + } + } + } + } + + private val productionOutputDir + get() = File(workDir, "out/production") + + private fun getOutputDir(moduleName: String): File = File(productionOutputDir, moduleName) + + fun testReexportedDependency() { + initProject() + AbstractKotlinJpsBuildTestCase.addKotlinStdlibDependency(myProject.modules.filter { module -> module.name == "module2" }, true) + buildAllModules().assertSuccessful() + } + + fun testCheckIsCancelledIsCalledOftenEnough() { + val classCount = 30 + val methodCount = 30 + + fun generateFiles() { + val srcDir = File(workDir, "src") + srcDir.mkdirs() + + for (i in 0..classCount) { + val code = buildString { + appendln("package foo") + appendln("class Foo$i {") + for (j in 0..methodCount) { + appendln(" fun get${j*j}(): Int = square($j)") + } + appendln("}") + + } + File(srcDir, "Foo$i.kt").writeText(code) + } + } + + generateFiles() + initProject(JVM_MOCK_RUNTIME) + + var checkCancelledCalledCount = 0 + val countingCancelledStatus = CanceledStatus { + checkCancelledCalledCount++ + false + } + + val logger = TestProjectBuilderLogger() + val buildResult = BuildResult() + + buildCustom(countingCancelledStatus, logger, buildResult) + + buildResult.assertSuccessful() + assert(checkCancelledCalledCount > classCount) { + "isCancelled should be called at least once per class. Expected $classCount, but got $checkCancelledCalledCount" + } + } + + fun testCancelKotlinCompilation() { + initProject(JVM_MOCK_RUNTIME) + buildAllModules().assertSuccessful() + + val module = myProject.modules.get(0) + assertFilesExistInOutput(module, "foo/Bar.class") + + val buildResult = BuildResult() + val canceledStatus = object : CanceledStatus { + var checkFromIndex = 0 + + override fun isCanceled(): Boolean { + val messages = buildResult.getMessages(BuildMessage.Kind.INFO) + for (i in checkFromIndex..messages.size - 1) { + if (messages[i].messageText.matches("kotlinc-jvm .+ \\(JRE .+\\)".toRegex())) { + return true + } + } + + checkFromIndex = messages.size + return false + } + } + + touch("src/Bar.kt").apply() + buildCustom(canceledStatus, TestProjectBuilderLogger(), buildResult) + assertCanceled(buildResult) + } + + fun testFileDoesNotExistWarning() { + fun absoluteFiles(vararg paths: String): Array = + paths.map { File(it).absoluteFile }.toTypedArray() + + initProject(JVM_MOCK_RUNTIME) + + val filesToBeReported = absoluteFiles("badroot.jar", "some/test.class") + val otherFiles = absoluteFiles("test/other/file.xml", "some/other/baddir") + + AbstractKotlinJpsBuildTestCase.addDependency( + JpsJavaDependencyScope.COMPILE, + Lists.newArrayList(findModule("module")), + false, + "LibraryWithBadRoots", + *(filesToBeReported + otherFiles) + ) + + val result = buildAllModules() + result.assertSuccessful() + + val actualWarnings = result.getMessages(BuildMessage.Kind.WARNING).map { it.messageText } + val expectedWarnings = filesToBeReported.map { "Classpath entry points to a non-existent location: $it" } + + val expectedText = expectedWarnings.sorted().joinToString("\n") + val actualText = actualWarnings.sorted().joinToString("\n") + + Assert.assertEquals(expectedText, actualText) + } + + fun testHelp() { + initProject() + + val result = buildAllModules() + result.assertSuccessful() + val warning = result.getMessages(BuildMessage.Kind.WARNING).single() + + val expectedText = StringUtil.convertLineSeparators(Usage.render(K2JVMCompiler(), K2JVMCompilerArguments())) + Assert.assertEquals(expectedText, warning.messageText) + } + + fun testWrongArgument() { + initProject() + + val result = buildAllModules() + result.assertFailed() + val errors = result.getMessages(BuildMessage.Kind.ERROR).joinToString("\n\n") { it.messageText } + + Assert.assertEquals("Invalid argument: -abcdefghij-invalid-argument", errors) + } + + fun testCodeInKotlinPackage() { + initProject(JVM_MOCK_RUNTIME) + + val result = buildAllModules() + result.assertFailed() + val errors = result.getMessages(BuildMessage.Kind.ERROR) + + Assert.assertEquals("Only the Kotlin standard library is allowed to use the 'kotlin' package", errors.single().messageText) + } + + fun testDoNotCreateUselessKotlinIncrementalCaches() { + initProject(JVM_MOCK_RUNTIME) + buildAllModules().assertSuccessful() + + val storageRoot = BuildDataPathsImpl(myDataStorageRoot).dataStorageRoot + assertFalse(File(storageRoot, "targets/java-test/kotlinProject/kotlin").exists()) + assertFalse(File(storageRoot, "targets/java-production/kotlinProject/kotlin").exists()) + } + + fun testDoNotCreateUselessKotlinIncrementalCachesForDependentTargets() { + initProject(JVM_MOCK_RUNTIME) + buildAllModules().assertSuccessful() + + if (IncrementalCompilation.isEnabledForJvm()) { + checkWhen(touch("src/utils.kt"), null, packageClasses("kotlinProject", "src/utils.kt", "_DefaultPackage")) + } + else { + val allClasses = findModule("kotlinProject").outputFilesPaths() + checkWhen(touch("src/utils.kt"), null, allClasses.toTypedArray()) + } + + val storageRoot = BuildDataPathsImpl(myDataStorageRoot).dataStorageRoot + assertFalse(File(storageRoot, "targets/java-production/kotlinProject/kotlin").exists()) + assertFalse(File(storageRoot, "targets/java-production/module2/kotlin").exists()) + } + + fun testKotlinProjectWithEmptyProductionOutputDir() { + initProject(JVM_MOCK_RUNTIME) + val result = buildAllModules() + result.assertFailed() + result.checkErrors() + } + + fun testKotlinProjectWithEmptyTestOutputDir() { + doTest() + } + + fun testKotlinProjectWithEmptyProductionOutputDirWithoutSrcDir() { + doTest() + } + + fun testKotlinProjectWithEmptyOutputDirInSomeModules() { + doTest() + } + + fun testEAPToReleaseIC() { + fun setPreRelease(value: Boolean) { + System.setProperty(TEST_IS_PRE_RELEASE_SYSTEM_PROPERTY, value.toString()) + } + + try { + withIC { + initProject(JVM_MOCK_RUNTIME) + + setPreRelease(true) + buildAllModules().assertSuccessful() + assertCompiled(KotlinBuilder.KOTLIN_BUILDER_NAME, "src/Bar.kt", "src/Foo.kt") + + touch("src/Foo.kt").apply() + buildAllModules() + assertCompiled(KotlinBuilder.KOTLIN_BUILDER_NAME, "src/Foo.kt") + + setPreRelease(false) + touch("src/Foo.kt").apply() + buildAllModules().assertSuccessful() + assertCompiled(KotlinBuilder.KOTLIN_BUILDER_NAME, "src/Bar.kt", "src/Foo.kt") + } + } + finally { + System.clearProperty(TEST_IS_PRE_RELEASE_SYSTEM_PROPERTY) + } + } + + fun testGetDependentTargets() { + fun addModuleWithSourceAndTestRoot(name: String): JpsModule { + return addModule(name, "src/").apply { + contentRootsList.addUrl(JpsPathUtil.pathToUrl("test/")) + addSourceRoot(JpsPathUtil.pathToUrl("test/"), JavaSourceRootType.TEST_SOURCE) + } + } + + // c -> b -exported-> a + // c2 -> b2 ------------^ + + val a = addModuleWithSourceAndTestRoot("a") + val b = addModuleWithSourceAndTestRoot("b") + val c = addModuleWithSourceAndTestRoot("c") + val b2 = addModuleWithSourceAndTestRoot("b2") + val c2 = addModuleWithSourceAndTestRoot("c2") + + JpsModuleRootModificationUtil.addDependency(b, a, JpsJavaDependencyScope.COMPILE, /*exported =*/ true) + JpsModuleRootModificationUtil.addDependency(c, b, JpsJavaDependencyScope.COMPILE, /*exported =*/ false) + JpsModuleRootModificationUtil.addDependency(b2, a, JpsJavaDependencyScope.COMPILE, /*exported =*/ false) + JpsModuleRootModificationUtil.addDependency(c2, b2, JpsJavaDependencyScope.COMPILE, /*exported =*/ false) + + val actual = StringBuilder() + buildCustom(CanceledStatus.NULL, TestProjectBuilderLogger(), BuildResult()) { + project.setTestingContext(TestingContext(LookupTracker.DO_NOTHING, object : TestingBuildLogger { + override fun chunkBuildStarted(context: CompileContext, chunk: ModuleChunk) { + actual.append("Targets dependent on ${chunk.targets.joinToString()}:\n") + val dependentRecursively = mutableSetOf() + context.kotlin.getChunk(chunk)!!.collectDependentChunksRecursivelyExportedOnly(dependentRecursively) + dependentRecursively.asSequence().map { it.targets.joinToString() }.sorted().joinTo(actual, "\n") + actual.append("\n---------\n") + } + + override fun afterChunkBuildStarted(context: CompileContext, chunk: ModuleChunk) {} + override fun invalidOrUnusedCache( + chunk: KotlinChunk?, + target: KotlinModuleBuildTarget<*>?, + attributesDiff: CacheAttributesDiff<*> + ) {} + override fun addCustomMessage(message: String) {} + override fun buildFinished(exitCode: ModuleLevelBuilder.ExitCode) {} + override fun markedAsDirtyBeforeRound(files: Iterable) {} + override fun markedAsDirtyAfterRound(files: Iterable) {} + })) + } + + val expectedFile = File(getCurrentTestDataRoot(), "expected.txt") + + KotlinTestUtils.assertEqualsToFile(expectedFile, actual.toString()) + } + + fun testJre9() { + val jdk9Path = KotlinTestUtils.getJdk9Home().absolutePath + + val jdk = myModel.global.addSdk(JDK_NAME, jdk9Path, "9", JpsJavaSdkType.INSTANCE) + jdk.addRoot(StandardFileSystems.JRT_PROTOCOL_PREFIX + jdk9Path + URLUtil.JAR_SEPARATOR + "java.base", JpsOrderRootType.COMPILED) + + loadProject(workDir.absolutePath + File.separator + PROJECT_NAME + ".ipr") + addKotlinStdlibDependency() + + buildAllModules().assertSuccessful() + } + + fun testCustomDestination() { + loadProject(workDir.absolutePath + File.separator + PROJECT_NAME + ".ipr") + addKotlinStdlibDependency() + buildAllModules().apply { + assertSuccessful() + + val aClass = File(workDir, "customOut/A.class") + assert(aClass.exists()) { "$aClass does not exist!" } + + val warnings = getMessages(BuildMessage.Kind.WARNING) + assert(warnings.isEmpty()) { "Unexpected warnings: \n${warnings.joinToString("\n")}" } + } + } + + private fun BuildResult.checkErrors() { + val actualErrors = getMessages(BuildMessage.Kind.ERROR) + .map { it as CompilerMessage } + .map { "${it.messageText} at line ${it.line}, column ${it.column}" }.sorted().joinToString("\n") + val expectedFile = File(getCurrentTestDataRoot(), "errors.txt") + KotlinTestUtils.assertEqualsToFile(expectedFile, actualErrors) + } + + private fun getCurrentTestDataRoot() = File(AbstractKotlinJpsBuildTestCase.TEST_DATA_PATH + "general/" + getTestName(false)) + + private fun buildCustom( + canceledStatus: CanceledStatus, + logger: TestProjectBuilderLogger, + buildResult: BuildResult, + setupProject: ProjectDescriptor.() -> Unit = {} + ) { + val scopeBuilder = CompileScopeTestBuilder.make().allModules() + val descriptor = this.createProjectDescriptor(BuildLoggingManager(logger)) + + descriptor.setupProject() + + try { + val builder = IncProjectBuilder(descriptor, BuilderRegistry.getInstance(), this.myBuildParams, canceledStatus, true) + builder.addMessageHandler(buildResult) + builder.build(scopeBuilder.build(), false) + } + finally { + descriptor.dataManager.flush(false) + descriptor.release() + } + } + + private fun assertCanceled(buildResult: BuildResult) { + val list = buildResult.getMessages(BuildMessage.Kind.INFO) + assertTrue("The build has been canceled" == list.last().messageText) + } + + private fun findModule(name: String): JpsModule { + for (module in myProject.modules) { + if (module.name == name) { + return module + } + } + throw IllegalStateException("Couldn't find module $name") + } + + protected fun checkWhen(action: Action, pathsToCompile: Array?, pathsToDelete: Array?) { + checkWhen(arrayOf(action), pathsToCompile, pathsToDelete) + } + + protected fun checkWhen(actions: Array, pathsToCompile: Array?, pathsToDelete: Array?) { + for (action in actions) { + action.apply() + } + + buildAllModules().assertSuccessful() + + if (pathsToCompile != null) { + assertCompiled(KotlinBuilder.KOTLIN_BUILDER_NAME, *pathsToCompile) + } + + if (pathsToDelete != null) { + assertDeleted(*pathsToDelete) + } + } + + protected fun packageClasses(moduleName: String, fileName: String, packageClassFqName: String): Array { + return arrayOf(module(moduleName), packagePartClass(moduleName, fileName, packageClassFqName)) + } + + protected fun packagePartClass(moduleName: String, fileName: String, packageClassFqName: String): String { + val path = FileUtilRt.toSystemIndependentName(File(workDir, fileName).absolutePath) + val fakeVirtualFile = object : LightVirtualFile(path.substringAfterLast('/')) { + override fun getPath(): String { + // strip extra "/" from the beginning + return path.substring(1) + } + } + + val packagePartFqName = PackagePartClassUtils.getDefaultPartFqName(FqName(packageClassFqName), fakeVirtualFile) + return klass(moduleName, AsmUtil.internalNameByFqNameWithoutInnerClasses(packagePartFqName)) + } + + private fun JpsProject.outputPaths(production: Boolean = true, tests: Boolean = true) = + modules.flatMap { it.outputFilesPaths(production = production, tests = tests) }.toTypedArray() + + private fun JpsModule.outputFilesPaths(production: Boolean = true, tests: Boolean = true): List { + val outputFiles = arrayListOf() + if (production) { + prodOut.walk().filterTo(outputFiles) { it.isFile } + } + if (tests) { + testsOut.walk().filterTo(outputFiles) { it.isFile } + } + return outputFiles.map { FileUtilRt.toSystemIndependentName(it.relativeTo(workDir).path) } + } + + private val JpsModule.prodOut: File + get() = outDir(forTests = false) + + private val JpsModule.testsOut: File + get() = outDir(forTests = true) + + private fun JpsModule.outDir(forTests: Boolean) = + JpsJavaExtensionService.getInstance().getOutputDirectory(this, forTests)!! + + protected enum class Operation { + CHANGE, + DELETE + } + + protected fun touch(path: String): Action = Action(Operation.CHANGE, path) + + protected fun del(path: String): Action = Action(Operation.DELETE, path) + + // TODO inline after KT-3974 will be fixed + protected fun touch(file: File): Unit = JpsBuildTestCase.change(file.absolutePath) + + protected inner class Action constructor(private val operation: Operation, private val path: String) { + fun apply() { + val file = File(workDir, path) + when (operation) { + Operation.CHANGE -> + touch(file) + Operation.DELETE -> + assertTrue("Can not delete file \"" + file.absolutePath + "\"", file.delete()) + } + } + } +}